mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-31 16:22:06 +00:00
Compare commits
31 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 884388163d | |||
| 422e724051 | |||
| e0bf54197d | |||
| b4af43c884 | |||
| 1cb4c0ebe2 | |||
| e5563e6004 | |||
| 0a72b29a34 | |||
| 51275cb7d8 | |||
| 3bf270b49f | |||
| d8d064bd18 | |||
| e5653ca055 | |||
| 6413a11889 | |||
| 9e4ea9fdd1 | |||
| 1a841fb7c7 | |||
| b8ee0301e9 | |||
| 0e5fe8c658 | |||
| e46877dd1c | |||
| 3fc610c8a2 | |||
| 4992034cc2 | |||
| b4c9bf9abb | |||
| 7d45685d70 | |||
| 76bab81aa6 | |||
| 72ff5dbbad | |||
| 8ece50ff8c | |||
| 7f13ebe2a5 | |||
| 696dc1496e | |||
| 68ee16bbbe | |||
| bc0706b2e3 | |||
| c6152954f6 | |||
| 0d2bf69e2d | |||
| 7e536843ea |
@@ -18,6 +18,7 @@ These implementation, style, and quality rules apply to all contributions, regar
|
||||
- Do not include agent, tool, contributor, account, or organization names in branch names.
|
||||
- Push to the user-requested remote or the repository's configured push remote. Do not hard-code or infer a remote from an account name.
|
||||
- Check free space before artifact-heavy builds, tests, coverage, or downloads. Re-check before a broad gate when space is tight.
|
||||
- Before handoff, remove the Cargo target directories and temporary worktrees the task created unless they stay intentionally or hold unmerged work. Identify build output by Cargo's markers (`.rustc_info.json`, `CACHEDIR.TAG`), not by directory name: `target` is also a source directory here.
|
||||
- Remove only task-owned temporary/build artifacts. Never delete another task's worktree or uncommitted data.
|
||||
- At handoff, mention disk or cleanup details only when they affected execution or artifacts/worktrees remain intentionally.
|
||||
- For Docker verification, ensure the tested image contains the intended binary. `buildx build` without `--load` leaves the image in the build cache; use the Makefile's loaded image and rebuild without cache only when stale artifacts are evidenced.
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-linux=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
|
||||
sha256-darwin=eb33067d90ee66c0a97e8f3658e46c152e47f897d2fbfdc6cb5ad93cfa55414b
|
||||
sha256-linux=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
sha256-darwin=686dcc075e41a75dde3d37e5bd0393d016469f02f83aa3508f92bfc17f26a4d4
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=67251cb7d9978d38e899268cac4d7898297ff7e5b5c48c9c1546dc760b93568d
|
||||
sha256-linux=673da76c2e5710c4622fe2fa32d2b297730895769487fcf1152ee1c2366f32f4
|
||||
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
|
||||
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
|
||||
sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
|
||||
|
||||
@@ -960,6 +960,12 @@ test-group = 'e2e-inline-boundaries'
|
||||
filter = 'package(e2e_test) & test(/^heal_erasure_disk_rebuild_test::/)'
|
||||
test-group = 'e2e-cluster-nightly'
|
||||
|
||||
# The multi-pool crash fixture seeds three nodes and 36 drives. Reserve this
|
||||
# nextest run's workers so other multi-node tests do not starve its deadline.
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & test(=heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_ec84_shards_across_multi_pool_after_background_target_crash)'
|
||||
threads-required = "num-test-threads"
|
||||
|
||||
[[profile.e2e-full.overrides]]
|
||||
filter = 'package(e2e_test) & (test(/^kms::kms_vault_test::/) | test(/^kms::kms_rekey_sweep_test::/) | test(/^kms::configured_roundtrip_test::test_configured_vault_kms_admin_and_versioned_cleanup$/))'
|
||||
test-group = 'e2e-vault'
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"plan_id": "rustfs-release-validation",
|
||||
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"targets": [
|
||||
{"os": "linux", "arch": "x86_64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
|
||||
{"os": "macos", "arch": "aarch64", "abi": "none"},
|
||||
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
|
||||
],
|
||||
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
|
||||
"features": ["pyroscope"],
|
||||
"trusted_workflows": [
|
||||
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
|
||||
],
|
||||
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
|
||||
"lanes": [
|
||||
{
|
||||
"lane": "identity",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
|
||||
"timeout_seconds": 1800,
|
||||
"environment_capabilities": ["process-exec"],
|
||||
"cases": ["artifact-identity", "startup"],
|
||||
"applies_to": {
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"platforms": "all"
|
||||
},
|
||||
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
|
||||
},
|
||||
{
|
||||
"lane": "upgrade",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
|
||||
"timeout_seconds": 5400,
|
||||
"environment_capabilities": ["process-exec", "previous-release-artifact"],
|
||||
"cases": [
|
||||
"direct-upgrade",
|
||||
"mixed-version-upgrade",
|
||||
"bucket-config-upgrade",
|
||||
"bucket-config-rollback",
|
||||
"multipart-layout-upgrade",
|
||||
"multipart-layout-baseline"
|
||||
],
|
||||
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
|
||||
"applies_to": {
|
||||
"release_types": ["prerelease", "stable"],
|
||||
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
|
||||
},
|
||||
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
|
||||
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
|
||||
},
|
||||
{
|
||||
"version": "1.0.0-rc.5",
|
||||
"role": "N-2",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
|
||||
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
+311
-5
@@ -814,13 +814,18 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Determine upload path based on build type
|
||||
# Determine upload path based on build type. Release and prerelease
|
||||
# builds only stage their bytes here: the official
|
||||
# artifacts/rustfs/release/ path is written solely by the
|
||||
# certification-gated promote-release-artifacts job, from these same
|
||||
# staged objects. Development builds keep the uncertified dev channel
|
||||
# and never touch official or latest channels.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
|
||||
echo "📤 Uploading development build to R2 dev directory"
|
||||
else
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
echo "📤 Uploading release build to R2 release directory"
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
echo "📤 Staging release build for certification-gated promotion"
|
||||
fi
|
||||
|
||||
# Upload all rustfs zip files to R2 using S3-compatible API
|
||||
@@ -869,6 +874,221 @@ jobs:
|
||||
echo "This run does not trigger Docker publishing."
|
||||
fi
|
||||
|
||||
# Aggregate the parent candidate batch. This job always collects results and
|
||||
# retains its evidence, including on failure, and every publication job
|
||||
# below requires its success. Missing, stale, substituted or non-PASS
|
||||
# evidence fails closed here: only the trusted verification sources
|
||||
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
|
||||
# it, so a PR-uploaded JSON set never gains release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download all build artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Write candidate and collect trusted validation results
|
||||
env:
|
||||
# Cross-repo read of the trusted verification run. Falls back to the
|
||||
# workflow token, which cannot read rustfs/s3chaos; a missing token
|
||||
# therefore fail-closes instead of certifying.
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
PRODUCT_COMMIT: ${{ github.sha }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
|
||||
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p results release-gate-output
|
||||
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
|
||||
|
||||
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
|
||||
import hashlib, json, os, sys, zipfile
|
||||
|
||||
plan_sha256 = sys.argv[1]
|
||||
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
|
||||
if profile is None:
|
||||
raise SystemExit("release certification only covers preview, prerelease and stable builds")
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
deliverables = []
|
||||
for name in sorted(os.listdir("artifacts")):
|
||||
if not name.startswith("rustfs-") or not name.endswith(".zip"):
|
||||
continue
|
||||
path = os.path.join("artifacts", name)
|
||||
members = zipfile.ZipFile(path).namelist()
|
||||
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
binary_bytes = archive.read(binary_name)
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile() as handle:
|
||||
handle.write(binary_bytes)
|
||||
handle.flush()
|
||||
binary_sha = sha256(handle.name)
|
||||
deliverables.append({
|
||||
"kind": "zip",
|
||||
"name": name,
|
||||
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
|
||||
"sha256": sha256(path),
|
||||
"size": os.path.getsize(path),
|
||||
"binary_sha256": binary_sha,
|
||||
})
|
||||
if not deliverables:
|
||||
raise SystemExit("no built deliverables found to certify")
|
||||
candidate = {
|
||||
"schema_version": 1,
|
||||
"product_commit": os.environ["PRODUCT_COMMIT"],
|
||||
"version": os.environ["VERSION"],
|
||||
"release_profile": profile,
|
||||
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
|
||||
"plan_sha256": plan_sha256,
|
||||
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
|
||||
"deliverables": deliverables,
|
||||
}
|
||||
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
|
||||
print()
|
||||
PY
|
||||
|
||||
# Trusted verification evidence only: the release-gate envelopes of
|
||||
# rustfs/s3chaos for this exact product commit. Absence blocks below.
|
||||
TRUSTED_REPO="rustfs/s3chaos"
|
||||
TRUSTED_WORKFLOW="release-gate.yml"
|
||||
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
|
||||
if [[ -z "$TRUSTED_RUN" ]]; then
|
||||
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
|
||||
else
|
||||
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
|
||||
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
|
||||
fi
|
||||
|
||||
python3 scripts/release/release_gate.py validate \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate candidate.json \
|
||||
--results results \
|
||||
--output release-gate-output
|
||||
|
||||
- name: Upload certification evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: |
|
||||
candidate.json
|
||||
results/
|
||||
release-gate-output/
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
# The only writer of the official R2 release path. Promotion re-checks the
|
||||
# exact bytes about to be published against the passed certification list
|
||||
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
|
||||
promote-release-artifacts:
|
||||
name: Promote Release Artifacts
|
||||
needs: [ build-check, build-rustfs, release-certification ]
|
||||
# Target-channel lock for the official R2 release path.
|
||||
concurrency:
|
||||
group: release-channel-r2-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download build and certification artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged release artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$name"
|
||||
done
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged object ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
|
||||
# Create GitHub Release for every valid release tag, including previews
|
||||
create-release:
|
||||
name: Create GitHub Release
|
||||
@@ -1028,6 +1248,10 @@ jobs:
|
||||
update-latest-version:
|
||||
name: Update Latest Version
|
||||
needs: [ build-check, publish-release ]
|
||||
# Target-channel lock for the latest.json version pointer.
|
||||
concurrency:
|
||||
group: release-channel-oss-latest
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
@@ -1068,6 +1292,45 @@ jobs:
|
||||
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
|
||||
chmod +x "$OSSUTIL_BIN"
|
||||
|
||||
# Candidate/pointer ordering: an older candidate must never overwrite
|
||||
# a newer pointer, even if its promotion job finishes later.
|
||||
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
|
||||
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
|
||||
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
def key(version):
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
|
||||
if not match:
|
||||
raise SystemExit(3)
|
||||
core = tuple(int(part) for part in match.groups()[:3])
|
||||
prerelease = match.group(4)
|
||||
if prerelease is None:
|
||||
return core + (1,)
|
||||
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
|
||||
|
||||
try:
|
||||
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
|
||||
proposed = sys.argv[2]
|
||||
if current and key(current) > key(proposed):
|
||||
raise SystemExit(1)
|
||||
except SystemExit:
|
||||
raise
|
||||
except (OSError, ValueError, TypeError):
|
||||
raise SystemExit(3)
|
||||
PY
|
||||
guard_status=$?
|
||||
if [[ "$guard_status" -eq 1 ]]; then
|
||||
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
|
||||
exit 0
|
||||
elif [[ "$guard_status" -ne 0 ]]; then
|
||||
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create latest.json
|
||||
cat > latest.json << EOF
|
||||
{
|
||||
@@ -1084,16 +1347,59 @@ jobs:
|
||||
|
||||
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
|
||||
|
||||
# Publish release (remove draft status)
|
||||
# Publish release (remove draft status). Requires the release-certification
|
||||
# aggregate: without a complete legal success there is no public write.
|
||||
publish-release:
|
||||
name: Publish Release
|
||||
needs: [ build-check, create-release, upload-release-assets ]
|
||||
needs: [ build-check, create-release, upload-release-assets, release-certification ]
|
||||
# Target-channel lock: concurrent releases serialize on this channel so an
|
||||
# older candidate cannot interleave its publish with a newer one.
|
||||
concurrency:
|
||||
group: release-channel-github-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify release assets against certification
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${{ needs.build-check.outputs.version }}"
|
||||
mkdir -p "$RUNNER_TEMP/promotion-assets"
|
||||
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
|
||||
found=0
|
||||
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
found=1
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$(basename "$zip_file")"
|
||||
done
|
||||
if [[ "$found" -ne 1 ]]; then
|
||||
echo "No release zip assets to verify against the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Publish release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -96,7 +96,7 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
|
||||
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
|
||||
# e2e-tests, e2e-full.
|
||||
warm-ci-dev:
|
||||
name: Warm ci-dev
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
# --all-targets covers the test binaries nextest builds, including
|
||||
# e2e_test, which test-and-lint's own run excludes. The second build adds
|
||||
# e2e_test, which the workspace run excludes. The second build adds
|
||||
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
|
||||
# and that no lint lane enables.
|
||||
- name: Build ci-dev superset
|
||||
|
||||
+36
-34
@@ -113,7 +113,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^rehearse_shards=' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
test-and-lint:
|
||||
workspace-lint-and-build:
|
||||
name: Workspace Lint and Build
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
@@ -193,13 +193,13 @@ jobs:
|
||||
|
||||
- name: Prepare test evidence
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
{
|
||||
echo "run_id=${GITHUB_RUN_ID}"
|
||||
echo "job=${GITHUB_JOB}"
|
||||
echo "runner=${RUNNER_NAME}"
|
||||
echo "started_at=$(date --utc --iso-8601=seconds)"
|
||||
} > artifacts/test-and-lint/run-metadata.txt
|
||||
} > artifacts/workspace/run-metadata.txt
|
||||
|
||||
# Clippy gates every event, pull requests included. When it ran only after
|
||||
# merge, a lint regression merged freely and then failed every main run
|
||||
@@ -233,11 +233,11 @@ jobs:
|
||||
|
||||
- name: Run documentation tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
set +e
|
||||
timeout --verbose --signal=TERM --kill-after=30s 15m \
|
||||
python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
|
||||
2>&1 | tee artifacts/test-and-lint/doctest.log
|
||||
2>&1 | tee artifacts/workspace/doctest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=cargo test --all --exclude e2e_test --doc"
|
||||
@@ -246,7 +246,7 @@ jobs:
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
|
||||
} > artifacts/test-and-lint/doctest-diagnostics.txt
|
||||
} > artifacts/workspace/doctest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Upload test reports and diagnostics
|
||||
@@ -255,7 +255,7 @@ jobs:
|
||||
with:
|
||||
name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
@@ -268,7 +268,7 @@ jobs:
|
||||
|
||||
workspace-tests:
|
||||
name: Workspace tests
|
||||
needs: [ test-and-lint, classify-changes ]
|
||||
needs: [ workspace-lint-and-build, classify-changes ]
|
||||
# The archive retains compile-time fixture paths and the large test set
|
||||
# requires the same runner image until the hosted-runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
@@ -293,14 +293,14 @@ jobs:
|
||||
|
||||
- name: Extract and list workspace tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
|
||||
> artifacts/test-and-lint/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
|
||||
> artifacts/workspace/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
|
||||
|
||||
- name: Run nextest tests
|
||||
run: |
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
@@ -309,7 +309,7 @@ jobs:
|
||||
cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
|
||||
--workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=workspace archive execution"
|
||||
@@ -321,13 +321,13 @@ jobs:
|
||||
echo
|
||||
echo "Kernel OOM / kill events:"
|
||||
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
|
||||
} > artifacts/test-and-lint/nextest-diagnostics.txt
|
||||
} > artifacts/workspace/nextest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Verify workspace execution completeness
|
||||
run: |
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
|
||||
- name: Run ignored ILM integration tests serially
|
||||
if: needs.classify-changes.outputs.mode == 'full'
|
||||
@@ -358,7 +358,7 @@ jobs:
|
||||
name: workspace-test-evidence-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
artifacts/ilm-integration
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
@@ -368,7 +368,7 @@ jobs:
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
run: |
|
||||
python3 scripts/ci_test_shards.py prepare \
|
||||
--listing artifacts/test-and-lint/core-test-listing.json \
|
||||
--listing artifacts/workspace/core-test-listing.json \
|
||||
--junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
|
||||
- name: Upload same-run shard plan
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
@@ -449,7 +449,7 @@ jobs:
|
||||
|
||||
connect-feature-boundaries:
|
||||
name: Connect Feature Boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -499,11 +499,13 @@ jobs:
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
# Alternate feature configurations run on the nightly schedule, merge
|
||||
# groups and manual dispatch. The default workspace remains the PR gate and
|
||||
# the per-push main baseline: repeating four feature builds on every merge
|
||||
# bought nothing the nightly run does not report within a day.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
# Specialized boundary, hardware and alternate-feature lanes run only on
|
||||
# the nightly schedule and manual dispatch. They do not gate a merge: the
|
||||
# nightly run reports a regression within a day, and repeating these builds
|
||||
# on every PR revision and merge group bought no earlier signal worth the
|
||||
# runner occupancy. Workspace lint and tests, the core contract, the debug
|
||||
# build, E2E smoke and the S3 lanes remain the pull-request gate.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -540,12 +542,12 @@ jobs:
|
||||
# --profile ci so the quarantine list (and its junit flaky markers)
|
||||
# covers this leg too; the default profile is the local no-retry
|
||||
# profile and silently ignored quarantined flakes here (rustfs#6703).
|
||||
mkdir -p artifacts/test-and-lint
|
||||
mkdir -p artifacts/workspace
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest-rio-v2
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
cargo test -p rustfs --doc --features rio-v2
|
||||
|
||||
- name: Upload rio-v2 test reports and diagnostics
|
||||
@@ -555,14 +557,14 @@ jobs:
|
||||
name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
test-and-lint-protocols:
|
||||
name: "Test and Lint (${{ matrix.features.name }})"
|
||||
# Same cadence as test-and-lint-rio-v2 above.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -631,7 +633,7 @@ jobs:
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/protocol-tests
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
@@ -783,7 +785,7 @@ jobs:
|
||||
# job had neither, so each closed/merged PR really ran the whole io_uring
|
||||
# suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
|
||||
# 30662728539) and kept the cancellation run in progress for minutes.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
|
||||
# a container, applies no seccomp filter that would block io_uring_setup — so
|
||||
@@ -978,7 +980,7 @@ jobs:
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
artifacts/test-and-lint
|
||||
artifacts/workspace
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
@@ -1389,7 +1391,7 @@ jobs:
|
||||
- classify-changes
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- workspace-shards
|
||||
- workspace-shard-evidence
|
||||
@@ -1425,7 +1427,7 @@ jobs:
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
- test-and-lint
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
|
||||
@@ -329,13 +329,75 @@ jobs:
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Consume the trusted release certification for the exact version this
|
||||
# workflow wraps and verify the certified input digests. A manual dispatch
|
||||
# without that certification fails here and pushes no official tag; only
|
||||
# results from the trusted verification sources carry release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$run_id" ]]; then
|
||||
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$run_id" -n release-certification -D certification; then
|
||||
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Input digest binding: every zip this image wraps must be the
|
||||
# certified byte-for-byte content, never a mutable latest pointer.
|
||||
while IFS=$'\t' read -r name location expected; do
|
||||
echo "Verifying certified input ${name}"
|
||||
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "Certified input ${name} does not match its recorded digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
# Supports both release and dev channel binaries based on build context
|
||||
# Only runs when should_build is true (which includes workflow success check)
|
||||
build-docker:
|
||||
name: Build Docker Images
|
||||
needs: build-check
|
||||
needs: [ build-check, release-certification ]
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -443,6 +505,14 @@ jobs:
|
||||
# Output tags
|
||||
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Staging identity for this run's promotion: official tags are only
|
||||
# created after the staged image verifies against the certification,
|
||||
# and they point at this same digest (no rebuild-after-verify).
|
||||
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
|
||||
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
|
||||
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
|
||||
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Generate labels
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
@@ -467,7 +537,9 @@ jobs:
|
||||
file: ${{ matrix.file }}
|
||||
platforms: ${{ env.DOCKER_PLATFORMS }}
|
||||
push: ${{ needs.build-check.outputs.should_push == 'true' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
# Push to staging tags only: official tags are created by the
|
||||
# certification-gated promotion step below from this same digest.
|
||||
tags: ${{ steps.meta.outputs.staging_tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# No layer cache. This build compiles nothing — it downloads a
|
||||
# release zip and runs apk/apt — so the cache could only save the
|
||||
@@ -493,6 +565,114 @@ jobs:
|
||||
no-cache: false
|
||||
pull: true
|
||||
|
||||
# A later-built wrapper: the staged image must complete this platform's
|
||||
# own binary identity verification before any official tag is written.
|
||||
# It may not inherit an unverified package's conclusion.
|
||||
- name: Verify staged image against certification
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
case "${{ matrix.variant }}" in
|
||||
musl) libc="musl" ;;
|
||||
glibc) libc="gnu" ;;
|
||||
esac
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
case "$platform" in
|
||||
linux/amd64) arch_sub="x86_64" ;;
|
||||
linux/arm64) arch_sub="aarch64" ;;
|
||||
esac
|
||||
docker pull --platform "$platform" "$STAGING_REF"
|
||||
container="$(docker create --platform "$platform" "$STAGING_REF")"
|
||||
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
|
||||
docker rm "$container" >/dev/null
|
||||
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
|
||||
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
|
||||
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/staged-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Promote staged image to official tags
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
|
||||
for tag in "${official_tags[@]}"; do
|
||||
docker buildx imagetools create -t "$tag" "$STAGING_REF"
|
||||
echo "Promoted $tag -> $STAGING_REF"
|
||||
done
|
||||
|
||||
# Downstream consumers (Helm) verify this record instead of inheriting
|
||||
# an unverified wrapper's conclusion.
|
||||
- name: Record image promotion
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
jq -n \
|
||||
--arg version "$VERSION" \
|
||||
--arg digest "$digest" \
|
||||
--arg staging_ref "$STAGING_REF" \
|
||||
--arg candidate_sha256 "$candidate_sha" \
|
||||
--arg tags "${{ steps.meta.outputs.tags }}" \
|
||||
--slurpfile candidate certification/candidate.json \
|
||||
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
|
||||
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
|
||||
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
|
||||
> image-promotion-record.json
|
||||
|
||||
- name: Upload image promotion record
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: image-promotion-record${{ matrix.suffix }}
|
||||
path: image-promotion-record.json
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
# Note: Manifest creation is no longer needed as we only build one variant
|
||||
# Multi-arch manifests are automatically created by docker/build-push-action
|
||||
|
||||
|
||||
@@ -30,9 +30,102 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Consume the same trusted release certification as every other publication
|
||||
# exit and verify the image promotion record of the Docker workflow. The
|
||||
# chart wraps no binary of its own, so it may only reference an image whose
|
||||
# platform verification already passed; it must not inherit an unverified
|
||||
# package's conclusion. A run without that evidence fails closed here and
|
||||
# never reaches the chart repository.
|
||||
release-certification:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.raw_tag }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Normalize release version
|
||||
id: version
|
||||
env:
|
||||
RAW_INPUT: ${{ github.event.inputs.version }}
|
||||
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
RAW="$RAW_INPUT"
|
||||
else
|
||||
RAW="$RAW_BRANCH"
|
||||
fi
|
||||
case "$RAW" in
|
||||
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
|
||||
*) RAW_TAG="$RAW" ;;
|
||||
esac
|
||||
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download and verify release certification and image record
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification image-record
|
||||
|
||||
build_run="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$build_run" ]]; then
|
||||
build_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$build_run" ]]; then
|
||||
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$build_run" -n release-certification -D certification; then
|
||||
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
docker_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
|
||||
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
|
||||
exit 1
|
||||
fi
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
|
||||
if [[ "$record_sha" != "$candidate_sha" ]]; then
|
||||
echo "Image promotion record does not belong to this certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
|
||||
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
|
||||
image-record/image-promotion-record.json)"
|
||||
if [[ "$overlap" -ne 0 ]]; then
|
||||
echo "Image promotion record carries binary digests outside the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build-helm-package:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ release-certification ]
|
||||
if: |
|
||||
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
|
||||
(
|
||||
|
||||
+210
-14
@@ -225,10 +225,57 @@ jobs:
|
||||
echo " Build run ID: $BUILD_RUN_ID"
|
||||
echo " Build run number: $RUN_NUMBER"
|
||||
|
||||
# Consume the trusted release certification of the resolved build run and
|
||||
# verify the certified input digests before any package is wrapped. A run
|
||||
# without that certification fails closed here; packages built later must
|
||||
# complete their own verification and cannot inherit an unverified
|
||||
# conclusion.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: resolve
|
||||
if: needs.resolve.result == 'success'
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
|
||||
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
# Development artifacts are explicitly uncertified and never touch
|
||||
# official or latest channels; only the staging/dev paths remain.
|
||||
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
|
||||
> certification/development-uncertified.json
|
||||
exit 0
|
||||
fi
|
||||
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
|
||||
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Build DEB and RPM packages for each architecture and libc variant
|
||||
package:
|
||||
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
|
||||
needs: resolve
|
||||
needs: [ resolve, release-certification ]
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
@@ -494,6 +541,55 @@ jobs:
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
- name: Download certification evidence
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
# Later-built wrapper verification: the binary inside each package must
|
||||
# match the certified binary digest for this platform before anything is
|
||||
# uploaded, and the package itself stays on staging until promotion.
|
||||
- name: Verify wrapped binary against certification
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
shell: bash
|
||||
env:
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
ARTIFACT_NAME: ${{ matrix.artifact_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" ]]; then
|
||||
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y cpio
|
||||
fi
|
||||
for package_file in "$DEB_FILE" "$RPM_FILE"; do
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Upload packages to artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
@@ -531,10 +627,14 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Build-phase writes stay on staging for release types: the official
|
||||
# packages/release and packages/latest paths are written only by the
|
||||
# certification-gated promote-packages job, from these same staged
|
||||
# objects. Development packages keep the uncertified dev channel.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PREFIX="artifacts/rustfs/packages/dev"
|
||||
else
|
||||
R2_PREFIX="artifacts/rustfs/packages/release"
|
||||
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
|
||||
fi
|
||||
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
|
||||
|
||||
@@ -549,18 +649,6 @@ jobs:
|
||||
|
||||
echo "✅ Upload complete"
|
||||
|
||||
# Also upload as latest for release/prerelease
|
||||
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading latest: $(basename "$f")"
|
||||
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
@@ -614,6 +702,114 @@ jobs:
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# The only writer of the official packages/release and packages/latest R2
|
||||
# paths. Promotion re-checks each package's wrapped binary against the
|
||||
# passed certification list and copies the staged objects byte-for-byte:
|
||||
# no rebuild-after-verify, no mutable latest resolution downstream.
|
||||
promote-packages:
|
||||
name: Promote Packages
|
||||
needs: [ resolve, package, release-certification ]
|
||||
# Target-channel lock for the official package paths.
|
||||
concurrency:
|
||||
group: release-channel-r2-packages
|
||||
cancel-in-progress: false
|
||||
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download packages and certification
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./packages
|
||||
pattern: packages-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged packages
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y cpio
|
||||
fi
|
||||
|
||||
promoted=0
|
||||
for package_file in packages/*.deb packages/*.rpm; do
|
||||
[[ -f "$package_file" ]] || continue
|
||||
name="$(basename "$package_file")"
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged package ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
promoted=$((promoted + 1))
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
if [[ "$promoted" -eq 0 ]]; then
|
||||
echo "No staged packages found to promote" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
|
||||
Generated
+56
-45
@@ -1321,9 +1321,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-smithy-schema"
|
||||
version = "1.1.0"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bec8dae1015a6d959a5faf86fbdf77486cfe19e4ccb6ee009d8a46efc45028a4"
|
||||
checksum = "9543fc722ed4865ab29fb66bd7ed0bef7b029d265ba8c587144511e30006d431"
|
||||
dependencies = [
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-types",
|
||||
@@ -4198,9 +4198,9 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
|
||||
|
||||
[[package]]
|
||||
name = "encoding_rs"
|
||||
version = "0.8.42"
|
||||
version = "0.8.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679"
|
||||
checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"core_detect",
|
||||
@@ -5277,9 +5277,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.30.1"
|
||||
version = "0.30.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "502dc37a7c218b9b1d3a0ccfb904da81d10d6dda0e899c722c503092d55576ea"
|
||||
checksum = "b39e941010d86063ff233bf8325269bf68097fb24ba67000fbddc0a410e8056e"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5312,15 +5312,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-drain"
|
||||
version = "0.30.1"
|
||||
version = "0.30.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ea17d04c231b90b1f68a91f6f80fb0762c5dea07c611dce9ae157b56a50ff4c"
|
||||
checksum = "d0faae1a9e1fd8ce5debaaaf452dcbe833fc8a8a4f3779d2ec8828b116c3c16a"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.30.1"
|
||||
version = "0.30.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a76e0af911930fc97abc766b393b48cdca77fea9f5f6b388cf2123a6d9c30deb"
|
||||
checksum = "a24f5179ff680132003306354efd0ac8cde12e87b1a788719c836e91af35cb58"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -5899,9 +5899,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.106"
|
||||
version = "0.3.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5"
|
||||
checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"futures-util",
|
||||
@@ -6094,9 +6094,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "libbz2-rs-sys"
|
||||
version = "0.2.5"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c"
|
||||
checksum = "32f454fd6831a29c0e827ed075bca2a15be201bd57f4168f0fd1041124eab17e"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
@@ -8305,9 +8305,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pulldown-cmark"
|
||||
version = "0.13.5"
|
||||
version = "0.13.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "278e9268c93b9e498a2da46c34d90ba93a3255eb99009646398f34bc73ab36eb"
|
||||
checksum = "3db2be46835f013dd3f4fa7f5f7402c537787b54ebb6fe2b94bbdd7e2669bb7b"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"memchr",
|
||||
@@ -8823,9 +8823,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "1.8.0"
|
||||
version = "1.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7acd809e80afa754c63f586955699d8990efe1c4b6613183ee5f027a11fbc8b5"
|
||||
checksum = "db7f3d75b5a3301746bff39a767bf4c5c42506b08513527308b3875bfb64a031"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arcstr",
|
||||
@@ -10617,6 +10617,9 @@ dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
"metrics",
|
||||
"opentelemetry-proto",
|
||||
"opentelemetry_sdk",
|
||||
"prost 0.14.4",
|
||||
"rand 0.10.3",
|
||||
"rmp-serde",
|
||||
"rustfs-common",
|
||||
@@ -10630,6 +10633,7 @@ dependencies = [
|
||||
"rustfs-heal-contracts",
|
||||
"rustfs-lifecycle",
|
||||
"rustfs-lock",
|
||||
"rustfs-obs",
|
||||
"rustfs-s3-types",
|
||||
"rustfs-scanner-metrics",
|
||||
"rustfs-storage-api",
|
||||
@@ -10844,8 +10848,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-uring"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7deda770b7981ee3bfaa202ecd18625fc794c962b8d58cd4f436952b7e66dcf"
|
||||
source = "git+https://github.com/rustfs/uring.git?rev=eb8fc160e71d804427de1e905f5c226d580f71bf#eb8fc160e71d804427de1e905f5c226d580f71bf"
|
||||
dependencies = [
|
||||
"io-uring",
|
||||
"libc",
|
||||
@@ -12214,9 +12217,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.40.0"
|
||||
version = "0.40.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "110e37d98880fd804cd47509b90a8444f7aaaab655ae625e83cd442a8bc2ebf2"
|
||||
checksum = "0e8a3edbe0394ce115f3917a00349dd242be09eb5a091ed509eaf67b55cfb8a8"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
@@ -12707,9 +12710,9 @@ checksum = "06bdbd8cfc056b8d2e2e85f29b56a3bdbecb527cef81eb39e3e7b98af4652770"
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.14.8"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd1fa9b1a51188f3266a823208078315811269d06d272c3a61be0b9a194194ff"
|
||||
checksum = "e2cdfc673aabd30fe8fee92e65837b644c37c5d47f0f97ae6d5e0f1237d15e1b"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -12740,9 +12743,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic-build"
|
||||
version = "0.14.8"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c07bd4e9f99e2a3ed8bebad596ae02a94e53745b68acc6900e22c3475c8bc44"
|
||||
checksum = "ac6d916d206ed7bdda1778ddff9bda43303789c258877e7bc22fb44ee5c984b4"
|
||||
dependencies = [
|
||||
"prettyplease 0.2.37",
|
||||
"proc-macro2",
|
||||
@@ -12752,9 +12755,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic-prost"
|
||||
version = "0.14.8"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f63c616ffaa648f7c5dd5657e97d9a285fcb44ff3b70da761d436f5a13ff7be6"
|
||||
checksum = "eba464054e2226f182a34489b8b079bf4c6e3e7ab5fd7fc30e024f24aa92a484"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost 0.14.4",
|
||||
@@ -12763,9 +12766,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tonic-prost-build"
|
||||
version = "0.14.8"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a05d8957fac6b3542999796cb4449d45da790751962f974a457e5e00c43d78c"
|
||||
checksum = "2ea216ddc448e08e7839ab09e20e60b06b6a1c32745ccb212c0ebd161c199f1b"
|
||||
dependencies = [
|
||||
"prettyplease 0.2.37",
|
||||
"proc-macro2",
|
||||
@@ -13158,9 +13161,12 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0"
|
||||
dependencies = [
|
||||
"utf_types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf8parse"
|
||||
@@ -13168,6 +13174,12 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "utf_types"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6"
|
||||
|
||||
[[package]]
|
||||
name = "uuid"
|
||||
version = "1.29.0"
|
||||
@@ -13282,12 +13294,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen"
|
||||
version = "0.2.129"
|
||||
version = "0.2.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409"
|
||||
checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"rustversion",
|
||||
"wasm-bindgen-macro",
|
||||
"wasm-bindgen-shared",
|
||||
@@ -13295,9 +13306,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.79"
|
||||
version = "0.4.80"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e"
|
||||
checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"tokio",
|
||||
@@ -13306,9 +13317,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.129"
|
||||
version = "0.2.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535"
|
||||
checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"wasm-bindgen-macro-support",
|
||||
@@ -13316,9 +13327,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro-support"
|
||||
version = "0.2.129"
|
||||
version = "0.2.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7"
|
||||
checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"proc-macro2",
|
||||
@@ -13329,9 +13340,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-shared"
|
||||
version = "0.2.129"
|
||||
version = "0.2.130"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6"
|
||||
checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
@@ -13351,9 +13362,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.106"
|
||||
version = "0.3.107"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4"
|
||||
checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
|
||||
+6
-6
@@ -175,9 +175,9 @@ tokio-rustls = { default-features = false, version = "0.26.6" }
|
||||
tokio-stream = { version = "0.1.19" }
|
||||
tokio-test = "0.4.6"
|
||||
tokio-util = { version = "0.7.20" }
|
||||
tonic = { version = "0.14.8" }
|
||||
tonic-prost = { version = "0.14.8" }
|
||||
tonic-prost-build = { version = "0.14.8" }
|
||||
tonic = { version = "0.14.9" }
|
||||
tonic-prost = { version = "0.14.9" }
|
||||
tonic-prost-build = { version = "0.14.9" }
|
||||
tower = { version = "0.5.3" }
|
||||
tower-http = { version = "0.7.1" }
|
||||
h3 = "0.0.8"
|
||||
@@ -319,7 +319,7 @@ rustfs-erasure-codec = { version = "9.0.0" }
|
||||
reed-solomon-simd = "3.1.0"
|
||||
regex = { version = "1.13.1" }
|
||||
rumqttc = { package = "rumqttc-next", version = "0.34.0" }
|
||||
redis = { version = "1.8.0" }
|
||||
redis = { version = "1.8.1" }
|
||||
rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.6" }
|
||||
rust-embed = { version = "8.13.0" }
|
||||
@@ -334,7 +334,7 @@ compact_str = "0.10.0"
|
||||
snap = "1.1.2"
|
||||
starshard = { version = "2.4.0" }
|
||||
strum = { version = "0.28.0" }
|
||||
sysinfo = "0.40.0"
|
||||
sysinfo = "0.40.1"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.4.0"
|
||||
@@ -386,7 +386,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.2" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.30.1", default-features = false }
|
||||
hotpath = { version = "0.30.2", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
|
||||
@@ -10,6 +10,10 @@ use std::time::Duration;
|
||||
use tokio::sync::{Notify, Semaphore};
|
||||
use tokio::time::Instant;
|
||||
|
||||
pub mod permit_set;
|
||||
|
||||
const MAX_PERMIT_REFERENCES: usize = usize::MAX / 2;
|
||||
|
||||
/// A resource owner publishes a capacity; running units retain their permits
|
||||
/// after a reduction. Lease slots are reserved once, with no per-unit Arc
|
||||
/// allocation. Tokio wait queues may lazily initialize platform mutex storage
|
||||
@@ -36,6 +40,7 @@ pub struct AdaptiveAdmission {
|
||||
maximum: u32,
|
||||
slots: Box<[PermitSlots]>,
|
||||
next_chunk: AtomicUsize,
|
||||
completed: AtomicU64,
|
||||
}
|
||||
|
||||
struct PermitSlots {
|
||||
@@ -61,10 +66,12 @@ pub struct AdaptivePermit {
|
||||
|
||||
impl Clone for AdaptivePermit {
|
||||
fn clone(&self) -> Self {
|
||||
// Every slot reference owns a strong owner reference. Clone the Arc
|
||||
// first, so its overflow protection also bounds the slot refcount.
|
||||
// Subset carriers can retain slot references through a shared owner,
|
||||
// so the slot count needs its own bound in addition to Arc's guard.
|
||||
let owner = Arc::clone(&self.owner);
|
||||
owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed);
|
||||
if owner.slots[self.slot / 64].references[self.slot % 64].fetch_add(1, Ordering::Relaxed) >= MAX_PERMIT_REFERENCES {
|
||||
std::process::abort();
|
||||
}
|
||||
Self { owner, slot: self.slot }
|
||||
}
|
||||
}
|
||||
@@ -114,11 +121,22 @@ impl NodeAdmission {
|
||||
|
||||
/// Publish feedback with this owner's actual completed-work counter.
|
||||
pub fn update(&self, controller: &mut FeedbackController, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
self.update_with_probe(controller, observation, true)
|
||||
}
|
||||
|
||||
/// Coordinate growth with other resource controllers in the same window.
|
||||
/// Protective reductions never depend on permission to probe.
|
||||
pub fn update_with_probe(
|
||||
&self,
|
||||
controller: &mut FeedbackController,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
let observation = observation.map(|mut observation| {
|
||||
observation.completed_units = self.completed();
|
||||
observation
|
||||
});
|
||||
controller.update(&self.capacity, observation)
|
||||
controller.update_with_probe(&self.capacity, observation, allow_probe)
|
||||
}
|
||||
|
||||
/// Current leased scheduling cap and held async units.
|
||||
@@ -249,16 +267,31 @@ impl FeedbackController {
|
||||
/// Consume one sample and publish a three-second lease to the same gate.
|
||||
/// If this function stops being called, admission expires independently.
|
||||
pub fn update(&mut self, gate: &AdaptiveAdmission, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
self.update_with_probe(gate, observation, true)
|
||||
}
|
||||
|
||||
/// Allow at most the sampling owner's selected controller to probe upward.
|
||||
pub fn update_with_probe(
|
||||
&mut self,
|
||||
gate: &AdaptiveAdmission,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
let observation = observation.map(|mut observation| {
|
||||
observation.max_concurrency = observation.max_concurrency.min(gate.maximum);
|
||||
observation
|
||||
});
|
||||
let decision = self.decide(Instant::now(), observation);
|
||||
let decision = self.decide_with_probe(Instant::now(), observation, allow_probe);
|
||||
gate.set_sampled_limit(decision.limit, Duration::from_secs(3));
|
||||
decision
|
||||
}
|
||||
|
||||
fn decide(&mut self, now: Instant, observation: Option<FeedbackObservation>) -> FeedbackDecision {
|
||||
fn decide_with_probe(
|
||||
&mut self,
|
||||
now: Instant,
|
||||
observation: Option<FeedbackObservation>,
|
||||
allow_probe: bool,
|
||||
) -> FeedbackDecision {
|
||||
let Some(observation) = observation else {
|
||||
self.previous = None;
|
||||
self.stable_since = None;
|
||||
@@ -303,7 +336,8 @@ impl FeedbackController {
|
||||
FeedbackReason::NewBaseline
|
||||
} else {
|
||||
let since = *self.stable_since.get_or_insert(now);
|
||||
if advanced
|
||||
if allow_probe
|
||||
&& advanced
|
||||
&& now.saturating_duration_since(since) >= Duration::from_secs(8)
|
||||
&& now.saturating_duration_since(self.last_change) >= Duration::from_secs(8)
|
||||
&& self.limit < observation.max_concurrency
|
||||
@@ -325,6 +359,13 @@ impl FeedbackController {
|
||||
}
|
||||
|
||||
impl AdaptiveAdmission {
|
||||
/// Successful async operations using this physical owner. Native clones
|
||||
/// retain capacity but never count another completion.
|
||||
pub fn completed(&self) -> Option<u64> {
|
||||
let value = self.completed.load(Ordering::Acquire);
|
||||
(value != u64::MAX).then_some(value)
|
||||
}
|
||||
|
||||
/// Construct a bounded owner, with zero initial capacity allowed for unknown
|
||||
/// input. The immutable maximum reserves at most 65,535 lease slots once;
|
||||
/// acquiring or cloning a permit does not allocate a per-unit Arc.
|
||||
@@ -351,6 +392,7 @@ impl AdaptiveAdmission {
|
||||
maximum: maximum_u32,
|
||||
slots,
|
||||
next_chunk: AtomicUsize::new(0),
|
||||
completed: AtomicU64::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -466,6 +508,21 @@ impl AdaptiveAdmission {
|
||||
self.slots[slot / 64].free.fetch_or(1_u64 << (slot % 64), Ordering::Release);
|
||||
}
|
||||
|
||||
fn release_reference(&self, slot: usize) {
|
||||
self.release_reference_with(slot, || {});
|
||||
}
|
||||
|
||||
fn release_reference_with(&self, slot: usize, before_release: impl FnOnce()) {
|
||||
if self.slots[slot / 64].references[slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
|
||||
return;
|
||||
}
|
||||
before_release();
|
||||
// Publish the free slot before exposing another admission capacity.
|
||||
self.release_slot(slot);
|
||||
self.state.fetch_sub(1, Ordering::AcqRel);
|
||||
self.changed.notify_one();
|
||||
}
|
||||
|
||||
/// Wait for capacity. The future owns no permit until it returns; dropping
|
||||
/// a pending future leaves capacity untouched. Wrap in the caller's existing
|
||||
/// cancellation/deadline select before acquiring any storage commit locks.
|
||||
@@ -493,13 +550,7 @@ impl AdaptiveAdmission {
|
||||
|
||||
impl Drop for AdaptivePermit {
|
||||
fn drop(&mut self) {
|
||||
if self.owner.slots[self.slot / 64].references[self.slot % 64].fetch_sub(1, Ordering::AcqRel) != 1 {
|
||||
return;
|
||||
}
|
||||
// Publish the free slot before exposing another admission capacity.
|
||||
self.owner.release_slot(self.slot);
|
||||
self.owner.state.fetch_sub(1, Ordering::AcqRel);
|
||||
self.owner.changed.notify_one();
|
||||
self.owner.release_reference(self.slot);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -861,6 +912,24 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shared_probe_permission_blocks_growth_but_never_protective_cuts() {
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bound"), 0);
|
||||
let mut controller = FeedbackController::new();
|
||||
controller.update_with_probe(&gate, Some(healthy(0)), false);
|
||||
for completed in 1..=20 {
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(healthy(completed)), false).limit, 1);
|
||||
}
|
||||
tokio::time::advance(Duration::from_secs(1)).await;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(healthy(21)), true).reason, FeedbackReason::Probe);
|
||||
assert_eq!(gate.snapshot().limit, 2);
|
||||
let mut busy = healthy(22);
|
||||
busy.pressure = ResourcePressure::Busy;
|
||||
assert_eq!(controller.update_with_probe(&gate, Some(busy), false).limit, 1);
|
||||
assert_eq!(controller.update_with_probe(&gate, None, false).limit, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn controller_grows_one_unit_per_cooldown_and_cuts_pressure_immediately() {
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(8).expect("bounded test owner"), 0);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,6 +30,16 @@ pub const COMMENT_KEY: &str = "comment";
|
||||
pub const ENV_AUDIT_ENABLE: &str = "RUSTFS_AUDIT_ENABLE";
|
||||
/// Global switch for enabling the notify module.
|
||||
pub const ENV_NOTIFY_ENABLE: &str = "RUSTFS_NOTIFY_ENABLE";
|
||||
/// Maximum audit entries per second for authorization denials of authenticated callers.
|
||||
/// `0` disables denial auditing for that class of caller.
|
||||
pub const ENV_AUDIT_DENIAL_RATE: &str = "RUSTFS_AUDIT_DENIAL_RATE";
|
||||
/// Maximum audit entries per second for authorization denials of anonymous callers.
|
||||
/// `0` disables denial auditing for that class of caller.
|
||||
pub const ENV_AUDIT_DENIAL_ANONYMOUS_RATE: &str = "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE";
|
||||
/// Default denial audit rate for authenticated callers (entries per second).
|
||||
pub const DEFAULT_AUDIT_DENIAL_RATE: u64 = 50;
|
||||
/// Default denial audit rate for anonymous callers (entries per second).
|
||||
pub const DEFAULT_AUDIT_DENIAL_ANONYMOUS_RATE: u64 = 5;
|
||||
/// Default global audit switch (disabled by default).
|
||||
pub const DEFAULT_AUDIT_ENABLE: bool = false;
|
||||
/// Default global notify switch (disabled by default).
|
||||
@@ -328,5 +338,7 @@ mod tests {
|
||||
fn test_global_audit_notify_switch_constants() {
|
||||
assert_eq!(ENV_AUDIT_ENABLE, "RUSTFS_AUDIT_ENABLE");
|
||||
assert_eq!(ENV_NOTIFY_ENABLE, "RUSTFS_NOTIFY_ENABLE");
|
||||
assert_eq!(ENV_AUDIT_DENIAL_RATE, "RUSTFS_AUDIT_DENIAL_RATE");
|
||||
assert_eq!(ENV_AUDIT_DENIAL_ANONYMOUS_RATE, "RUSTFS_AUDIT_DENIAL_ANONYMOUS_RATE");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,14 @@ use std::{
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
/// Copy an existing cache-model value without including surrounding work.
|
||||
#[inline]
|
||||
pub fn clone_cache_value<T: ToOwned + ?Sized>(source: &T) -> T::Owned {
|
||||
#[cfg(feature = "hotpath-alloc")]
|
||||
let _copy_scope = hotpath::MeasurementGuardSync::new("cache_model.owned_copy", false, false);
|
||||
source.to_owned()
|
||||
}
|
||||
|
||||
/// Maximum amount a persisted `last_update` may lead the local wall clock before the
|
||||
/// persisted timestamp is treated as untrustworthy.
|
||||
///
|
||||
@@ -342,7 +350,7 @@ impl UnknownTierStats {
|
||||
self.diagnostics_dropped = checked_saturating_add(1, self.diagnostics_dropped, &mut self.counter_overflowed);
|
||||
continue;
|
||||
}
|
||||
self.diagnostics.push(diagnostic.clone());
|
||||
self.diagnostics.push(clone_cache_value(diagnostic));
|
||||
}
|
||||
}
|
||||
if !self.diagnostics.is_empty() {
|
||||
@@ -367,7 +375,7 @@ impl UnknownTierStats {
|
||||
if !self.fits_add(other) {
|
||||
return None;
|
||||
}
|
||||
let mut merged = self.clone();
|
||||
let mut merged = clone_cache_value(self);
|
||||
merged.merge(other);
|
||||
Some(merged)
|
||||
}
|
||||
@@ -407,7 +415,7 @@ impl AllTierStats {
|
||||
if st.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let entry = self.tiers.entry(tier.clone()).or_default();
|
||||
let entry = self.tiers.entry(clone_cache_value(tier)).or_default();
|
||||
*entry = entry.add(st);
|
||||
}
|
||||
}
|
||||
@@ -437,7 +445,7 @@ impl AllTierStats {
|
||||
.tiers
|
||||
.keys()
|
||||
.filter(|tier| tier.as_str() != "STANDARD" && tier.as_str() != "REDUCED_REDUNDANCY" && !known.contains(tier.as_str()))
|
||||
.cloned()
|
||||
.map(clone_cache_value)
|
||||
.collect();
|
||||
for tier in retired_tiers {
|
||||
if let Some(stats) = self.tiers.remove(&tier) {
|
||||
@@ -1237,10 +1245,10 @@ impl DataUsageEntry {
|
||||
}
|
||||
|
||||
pub fn add_child(&mut self, hash: &DataUsageHash) {
|
||||
if self.children.contains(&hash.key()) {
|
||||
if self.children.contains(&clone_cache_value(&hash.0)) {
|
||||
return;
|
||||
}
|
||||
self.children.insert(hash.key());
|
||||
self.children.insert(clone_cache_value(&hash.0));
|
||||
}
|
||||
|
||||
pub fn merge(&mut self, other: &DataUsageEntry) {
|
||||
@@ -1257,7 +1265,7 @@ impl DataUsageEntry {
|
||||
s_rep.replica_size += o_rep.replica_size;
|
||||
s_rep.replica_count += o_rep.replica_count;
|
||||
for (arn, stat) in o_rep.targets.iter() {
|
||||
let st = s_rep.targets.entry(arn.clone()).or_default();
|
||||
let st = s_rep.targets.entry(clone_cache_value(arn)).or_default();
|
||||
st.pending_size += stat.pending_size;
|
||||
st.replicated_size += stat.replicated_size;
|
||||
st.failed_size += stat.failed_size;
|
||||
@@ -1451,7 +1459,7 @@ pub fn prefix_usage_in_cache(
|
||||
} else {
|
||||
format!("{bucket}/{prefix}")
|
||||
};
|
||||
let entry = cache.get(&hash_path(&root).key())?.clone();
|
||||
let entry = clone_cache_value(cache.get(&clone_cache_value(&hash_path(&root).0))?);
|
||||
|
||||
let usage = PrefixUsageSummary::from_entry(&flatten_entry(cache, &entry, 0)?);
|
||||
|
||||
@@ -1594,38 +1602,38 @@ pub struct DataUsageCache {
|
||||
impl DataUsageCache {
|
||||
pub fn replace(&mut self, path: &str, parent: &str, e: DataUsageEntry) {
|
||||
let hash = hash_path(path);
|
||||
self.cache.insert(hash.key(), e);
|
||||
self.cache.insert(clone_cache_value(&hash.0), e);
|
||||
if !parent.is_empty() {
|
||||
let phash = hash_path(parent);
|
||||
let p = {
|
||||
let p = self.cache.entry(phash.key()).or_default();
|
||||
let p = self.cache.entry(clone_cache_value(&phash.0)).or_default();
|
||||
p.add_child(&hash);
|
||||
p.clone()
|
||||
clone_cache_value(p)
|
||||
};
|
||||
self.cache.insert(phash.key(), p);
|
||||
self.cache.insert(clone_cache_value(&phash.0), p);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn replace_hashed(&mut self, hash: &DataUsageHash, parent: &Option<DataUsageHash>, e: &DataUsageEntry) {
|
||||
self.cache.insert(hash.key(), e.clone());
|
||||
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
|
||||
if let Some(parent) = parent {
|
||||
self.cache.entry(parent.key()).or_default().add_child(hash);
|
||||
self.cache.entry(clone_cache_value(&parent.0)).or_default().add_child(hash);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn find(&self, path: &str) -> Option<DataUsageEntry> {
|
||||
self.cache.get(&hash_path(path).key()).cloned()
|
||||
self.cache.get(&clone_cache_value(&hash_path(path).0)).map(clone_cache_value)
|
||||
}
|
||||
|
||||
pub fn find_children_copy(&mut self, h: DataUsageHash) -> DataUsageHashMap {
|
||||
self.cache.entry(h.string()).or_default().children.clone()
|
||||
clone_cache_value(&self.cache.entry(clone_cache_value(&h.0)).or_default().children)
|
||||
}
|
||||
|
||||
pub fn flatten(&self, root: &DataUsageEntry) -> DataUsageEntry {
|
||||
let mut root = root.clone();
|
||||
for id in root.children.clone().iter() {
|
||||
let mut root = clone_cache_value(root);
|
||||
for id in clone_cache_value(&root.children).iter() {
|
||||
if let Some(e) = self.cache.get(id) {
|
||||
let mut e = e.clone();
|
||||
let mut e = clone_cache_value(e);
|
||||
if !e.children.is_empty() {
|
||||
e = self.flatten(&e);
|
||||
}
|
||||
@@ -1637,16 +1645,16 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn copy_with_children(&mut self, src: &DataUsageCache, hash: &DataUsageHash, parent: &Option<DataUsageHash>) {
|
||||
if let Some(e) = src.cache.get(&hash.string()) {
|
||||
self.cache.insert(hash.key(), e.clone());
|
||||
if let Some(e) = src.cache.get(&clone_cache_value(&hash.0)) {
|
||||
self.cache.insert(clone_cache_value(&hash.0), clone_cache_value(e));
|
||||
for ch in e.children.iter() {
|
||||
if *ch == hash.key() {
|
||||
if *ch == clone_cache_value(&hash.0) {
|
||||
return;
|
||||
}
|
||||
self.copy_with_children(src, &DataUsageHash(ch.to_string()), &Some(hash.clone()));
|
||||
self.copy_with_children(src, &DataUsageHash(clone_cache_value(ch)), &Some(clone_cache_value(hash)));
|
||||
}
|
||||
if let Some(parent) = parent {
|
||||
let p = self.cache.entry(parent.key()).or_default();
|
||||
let p = self.cache.entry(clone_cache_value(&parent.0)).or_default();
|
||||
p.add_child(hash);
|
||||
}
|
||||
}
|
||||
@@ -1654,14 +1662,14 @@ impl DataUsageCache {
|
||||
|
||||
pub fn delete_recursive(&mut self, hash: &DataUsageHash) {
|
||||
let mut need_remove = Vec::new();
|
||||
if let Some(v) = self.cache.get(&hash.string()) {
|
||||
if let Some(v) = self.cache.get(&clone_cache_value(&hash.0)) {
|
||||
for child in v.children.iter() {
|
||||
need_remove.push(child.clone());
|
||||
need_remove.push(clone_cache_value(child));
|
||||
}
|
||||
}
|
||||
self.cache.remove(&hash.string());
|
||||
self.cache.remove(&clone_cache_value(&hash.0));
|
||||
need_remove.iter().for_each(|child| {
|
||||
self.delete_recursive(&DataUsageHash(child.to_string()));
|
||||
self.delete_recursive(&DataUsageHash(clone_cache_value(child)));
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1682,7 +1690,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn search_parent(&self, hash: &DataUsageHash) -> Option<DataUsageHash> {
|
||||
let want = hash.key();
|
||||
let want = clone_cache_value(&hash.0);
|
||||
if let Some(last_index) = want.rfind('/')
|
||||
&& let Some(v) = self.find(&want[0..last_index])
|
||||
&& v.children.contains(&want)
|
||||
@@ -1693,7 +1701,7 @@ impl DataUsageCache {
|
||||
|
||||
for (k, v) in self.cache.iter() {
|
||||
if v.children.contains(&want) {
|
||||
let found = DataUsageHash(k.clone());
|
||||
let found = DataUsageHash(clone_cache_value(k));
|
||||
return Some(found);
|
||||
}
|
||||
}
|
||||
@@ -1701,7 +1709,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn is_compacted(&self, hash: &DataUsageHash) -> bool {
|
||||
match self.cache.get(&hash.key()) {
|
||||
match self.cache.get(&clone_cache_value(&hash.0)) {
|
||||
Some(due) => due.compacted,
|
||||
None => false,
|
||||
}
|
||||
@@ -1726,7 +1734,7 @@ impl DataUsageCache {
|
||||
if self.cache.len() < limit {
|
||||
return;
|
||||
}
|
||||
let top = hash_path(&self.info.name).key();
|
||||
let top = clone_cache_value(&hash_path(&self.info.name).0);
|
||||
let top_e = match self.find(&top) {
|
||||
Some(e) => e,
|
||||
None => return,
|
||||
@@ -1753,7 +1761,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
|
||||
let e = match self.cache.get(&path.key()) {
|
||||
let e = match self.cache.get(&clone_cache_value(&path.0)) {
|
||||
Some(e) => e,
|
||||
None => return,
|
||||
};
|
||||
@@ -1763,13 +1771,13 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
if e.children.len() > limit && compact_self {
|
||||
let mut flat = self.size_recursive(&path.key()).unwrap_or_default();
|
||||
let mut flat = self.size_recursive(&clone_cache_value(&path.0)).unwrap_or_default();
|
||||
flat.compacted = true;
|
||||
self.delete_recursive(path);
|
||||
self.replace_hashed(path, &None, &flat);
|
||||
return;
|
||||
}
|
||||
let total = self.total_children_rec(&path.key());
|
||||
let total = self.total_children_rec(&clone_cache_value(&path.0));
|
||||
if total < limit {
|
||||
return;
|
||||
}
|
||||
@@ -1782,12 +1790,12 @@ impl DataUsageCache {
|
||||
let mut candidate_index = 0;
|
||||
while remove > 0 && candidate_index < candidates.len() {
|
||||
let e = &candidates[candidate_index];
|
||||
let candidate = e.path.clone();
|
||||
let candidate = clone_cache_value(&e.path);
|
||||
if candidate == *path && !compact_self {
|
||||
break;
|
||||
}
|
||||
let removing = self.total_children_rec(&candidate.key());
|
||||
let mut flat = match self.size_recursive(&candidate.key()) {
|
||||
let removing = self.total_children_rec(&clone_cache_value(&candidate.0));
|
||||
let mut flat = match self.size_recursive(&clone_cache_value(&candidate.0)) {
|
||||
Some(flat) => flat,
|
||||
None => {
|
||||
candidate_index += 1;
|
||||
@@ -1824,7 +1832,7 @@ impl DataUsageCache {
|
||||
if o.root().is_none() {
|
||||
return;
|
||||
}
|
||||
*self = o.clone();
|
||||
*self = clone_cache_value(o);
|
||||
return;
|
||||
};
|
||||
|
||||
@@ -1837,7 +1845,8 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
existing_root.merge(&other_root);
|
||||
self.cache.insert(hash_path(&self.info.name).key(), existing_root);
|
||||
self.cache
|
||||
.insert(clone_cache_value(&hash_path(&self.info.name).0), existing_root);
|
||||
|
||||
let root_hash = self.root_hash();
|
||||
for key in other_root.children.iter() {
|
||||
@@ -1848,7 +1857,7 @@ impl DataUsageCache {
|
||||
if let Some(existing) = self.cache.get_mut(key) {
|
||||
existing.merge(&flat);
|
||||
} else {
|
||||
self.replace_hashed(&DataUsageHash(key.clone()), &Some(root_hash.clone()), &flat);
|
||||
self.replace_hashed(&DataUsageHash(clone_cache_value(key)), &Some(clone_cache_value(&root_hash)), &flat);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1892,7 +1901,7 @@ impl DataUsageCache {
|
||||
|
||||
for (arn, stat) in rs.targets.iter() {
|
||||
bui.replication_info.insert(
|
||||
arn.clone(),
|
||||
clone_cache_value(arn),
|
||||
BucketTargetUsageInfo {
|
||||
replication_pending_size: stat.pending_size,
|
||||
replicated_size: stat.replicated_size,
|
||||
@@ -1905,7 +1914,7 @@ impl DataUsageCache {
|
||||
);
|
||||
}
|
||||
}
|
||||
buckets_usage.insert(bucket_name.clone(), bui);
|
||||
buckets_usage.insert(clone_cache_value(bucket_name), bui);
|
||||
}
|
||||
|
||||
DataUsageInfo {
|
||||
@@ -1937,20 +1946,20 @@ struct Inner {
|
||||
}
|
||||
|
||||
fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut Vec<Inner>) -> usize {
|
||||
let e = match data_usage_cache.cache.get(&path.key()) {
|
||||
let e = match data_usage_cache.cache.get(&clone_cache_value(&path.0)) {
|
||||
Some(e) => e,
|
||||
None => return 0,
|
||||
};
|
||||
let mut objects = e.objects;
|
||||
for ch in e.children.iter() {
|
||||
objects += add(data_usage_cache, &DataUsageHash(ch.clone()), candidates);
|
||||
objects += add(data_usage_cache, &DataUsageHash(clone_cache_value(ch)), candidates);
|
||||
}
|
||||
// Collect internal nodes (with children) as compaction candidates.
|
||||
// Leaf nodes have no children to remove, so compacting them is a no-op.
|
||||
if !e.children.is_empty() {
|
||||
candidates.push(Inner {
|
||||
objects,
|
||||
path: path.clone(),
|
||||
path: clone_cache_value(path),
|
||||
});
|
||||
}
|
||||
objects
|
||||
@@ -1958,7 +1967,7 @@ fn add(data_usage_cache: &DataUsageCache, path: &DataUsageHash, candidates: &mut
|
||||
|
||||
fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String>) {
|
||||
for k in entry.children.iter() {
|
||||
found.insert(k.to_string());
|
||||
found.insert(clone_cache_value(k));
|
||||
if let Some(ch) = duc.cache.get(k) {
|
||||
mark(duc, ch, found);
|
||||
}
|
||||
@@ -2408,11 +2417,11 @@ impl SizeSummary {
|
||||
}
|
||||
|
||||
for entry in &other.size_reconciliation {
|
||||
self.record_size_reconciliation(entry.clone());
|
||||
self.record_size_reconciliation(clone_cache_value(entry));
|
||||
}
|
||||
self.size_reconciliation_truncated |= other.size_reconciliation_truncated;
|
||||
for scope in &other.reconciliation_scopes {
|
||||
self.record_reconciliation_scope(&scope.bucket, &scope.object);
|
||||
self.record_reconciliation_scope_with(&scope.bucket, &scope.object, || clone_cache_value(scope));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2437,6 +2446,13 @@ impl SizeSummary {
|
||||
/// Mark one object scope as refreshed. Duplicate scopes are suppressed so
|
||||
/// merging summaries remains bounded and deterministic.
|
||||
pub fn record_reconciliation_scope(&mut self, bucket: &str, object: &str) {
|
||||
self.record_reconciliation_scope_with(bucket, object, || SizeReconciliationScope {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
fn record_reconciliation_scope_with(&mut self, bucket: &str, object: &str, build: impl FnOnce() -> SizeReconciliationScope) {
|
||||
if !self
|
||||
.reconciliation_scopes
|
||||
.iter()
|
||||
@@ -2446,10 +2462,7 @@ impl SizeSummary {
|
||||
self.size_reconciliation_truncated = true;
|
||||
return;
|
||||
}
|
||||
self.reconciliation_scopes.push(SizeReconciliationScope {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
});
|
||||
self.reconciliation_scopes.push(build());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3404,6 +3417,42 @@ mod tests {
|
||||
assert_eq!(summary1.versions, 15);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reconciliation_scope_builds_only_after_admission() {
|
||||
let mut summary = SizeSummary::new();
|
||||
let builds = std::cell::Cell::new(0);
|
||||
let build = || {
|
||||
builds.set(builds.get() + 1);
|
||||
SizeReconciliationScope {
|
||||
bucket: "bucket".to_string(),
|
||||
object: "object".to_string(),
|
||||
}
|
||||
};
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "an admitted scope is copied once");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1);
|
||||
assert_eq!(summary.reconciliation_scopes[0].bucket, "bucket");
|
||||
assert_eq!(summary.reconciliation_scopes[0].object, "object");
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "duplicate scopes must not be copied");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1);
|
||||
assert!(!summary.size_reconciliation_truncated);
|
||||
|
||||
for index in 1..1024 {
|
||||
summary.record_reconciliation_scope("bucket", &index.to_string());
|
||||
}
|
||||
summary.record_reconciliation_scope_with("bucket", "object", build);
|
||||
assert_eq!(builds.get(), 1, "a duplicate at capacity must not be copied");
|
||||
assert!(!summary.size_reconciliation_truncated);
|
||||
|
||||
summary.record_reconciliation_scope_with("bucket", "overflow", build);
|
||||
assert_eq!(builds.get(), 1, "a rejected scope must not be copied");
|
||||
assert_eq!(summary.reconciliation_scopes.len(), 1024);
|
||||
assert!(summary.size_reconciliation_truncated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_size_histogram_compat_rollup_sums_all_sub_buckets() {
|
||||
let mut hist = SizeHistogram::default();
|
||||
|
||||
@@ -25,6 +25,7 @@ workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
scanner-peer-mixed-version = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
|
||||
@@ -64,6 +64,25 @@ Each build/run holds an exclusive `rustfs.e2e.lock` marker beside the binary; co
|
||||
|
||||
The protocols suite has its own fixed-port and single-worker contract in [`src/protocols/README.md`](src/protocols/README.md). Use its command under [Troubleshooting](#troubleshooting).
|
||||
|
||||
### Scanner mixed-version gate
|
||||
|
||||
The Linux N/O scanner gate needs a separately qualified `33482bb` server, which supports scoped ACKs. Set `RUSTFS_SCANNER_COMPAT_OLD_BINARY` to that executable and `RUSTFS_SCANNER_COMPAT_OLD_SHA256` to its independently verified digest. The release binary used by ordinary upgrade CI is a different baseline. Missing or mismatched inputs fail the test.
|
||||
|
||||
Enable the harness-only `scanner-peer-mixed-version` feature explicitly; keep the server wrapper features identical between build and run:
|
||||
|
||||
```bash
|
||||
python3 scripts/e2e_binary.py build --features e2e-test-hooks
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
|
||||
cargo nextest run --profile e2e-distributed -p e2e_test \
|
||||
--features scanner-peer-mixed-version --ignore-default-filter \
|
||||
-E 'test(=distributed::scanner_peer_bucket_test::scanner_peer_bucket_new_coordinator_certifies_old_peers_and_scopes_next_cycle)' \
|
||||
--test-threads 1 --retries 0 --no-tests=fail
|
||||
```
|
||||
|
||||
The separate `scanner_peer_bucket_new_coordinator_certifies_old_worker_peers_and_scopes_next_cycle` case uses the same feature and qualified old binary inputs. Select its exact name in the command above. It keeps the original N process enabled and starts the three official O replacements with `RUSTFS_SCANNER_ENABLED=false`: their autonomous scanner loops are disabled, while real S3 dirty tracking, authenticated snapshots, publication leases, scoped ACKs, and N-dispatched RemoteScanner workers remain active. It records each node's actual scanner status and retains the same identity, durability, root, ACK, data-conservation, and physical-walk requirements. This fixed-role case does not validate autonomous O scanning or all-enabled election behavior. The original all-enabled N/O test remains separate and has not passed acceptance; a fixed-role result cannot satisfy that gate.
|
||||
|
||||
For this fixed-role case, the `e2e-test-hooks` server also records the N client's complete bounded worker inventory. Requests are observed before opening the normal stream and paired only after authenticated source/plan validation and terminal EOF. Closed manifests and final health receipts must bind both cycles to their actual candidate/root; the next cycle needs exactly four sets of hot-only execution and at least one positive-progress worker mapped to a measured official O process. A zero-progress Current response, missing/extra request, unhealthy recorder, or missing physical coverage fails the gate. The recorder does not add O hooks or change any protocol or timeout. Final health reads check immutable closed-cycle evidence, not current N liveness.
|
||||
|
||||
### `#[ignore]` semantics
|
||||
|
||||
Ignored tests are excluded from the default `cargo nextest run` pass because
|
||||
|
||||
@@ -16,8 +16,193 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::http::HttpResponse;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
|
||||
fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
|
||||
assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"{error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("start RustFS");
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "conditional-delete-marker-proof";
|
||||
create_versioned_bucket(&client, bucket).await;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("source")
|
||||
.body(ByteStream::from_static(b"source"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write copy source");
|
||||
|
||||
for operation in ["put", "copy", "multipart"] {
|
||||
let old = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.body(ByteStream::from_static(b"old"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write historical version");
|
||||
let old_etag = old.e_tag().expect("historical ETag");
|
||||
let marker = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("create delete marker");
|
||||
let marker_id = marker.version_id().expect("delete marker version");
|
||||
|
||||
let (upload_id, completed) = if operation == "multipart" {
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let upload_id = upload.upload_id().expect("upload ID").to_owned();
|
||||
let part = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upload part");
|
||||
let completed = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.set_e_tag(part.e_tag().map(str::to_owned))
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
(Some(upload_id), Some(completed))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
for etag in [old_etag, "\"00000000000000000000000000000000\""] {
|
||||
match operation {
|
||||
"put" => {
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_match(etag)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
"copy" => {
|
||||
let err = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match copy must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
_ => {
|
||||
let err = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.clone().expect("completed parts"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match completion must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
}
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions after rejected write");
|
||||
assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
|
||||
assert_eq!(versions.delete_markers().len(), 1);
|
||||
assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
|
||||
assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
|
||||
}
|
||||
|
||||
match operation {
|
||||
"put" => {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_none_match("*")
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match PUT over marker");
|
||||
}
|
||||
"copy" => {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match copy over marker");
|
||||
}
|
||||
_ => {
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.expect("completed parts"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match completion over marker");
|
||||
}
|
||||
}
|
||||
let current = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("read successful conditional write")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("read object body")
|
||||
.into_bytes();
|
||||
let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
|
||||
assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
|
||||
}
|
||||
}
|
||||
|
||||
async fn create_versioned_bucket(client: &Client, bucket: &str) {
|
||||
client
|
||||
|
||||
@@ -39,6 +39,8 @@ mod s3_during_data_movement_test;
|
||||
mod scanner_heal_topology_fixture;
|
||||
mod scanner_heal_topology_test;
|
||||
mod scanner_heal_topology_tier_test;
|
||||
mod scanner_peer_bucket_fixture;
|
||||
mod scanner_peer_bucket_test;
|
||||
mod site_replication_test;
|
||||
mod table_catalog_test;
|
||||
mod upgrade_test;
|
||||
|
||||
@@ -18,7 +18,7 @@ use super::harness::{
|
||||
};
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
|
||||
use crate::rpc_fault_proxy::{RpcFaultProxy, RpcPause};
|
||||
use crate::storage_api::multipart_fence_acceptance::RUSTFS_META_MULTIPART_BUCKET;
|
||||
use crate::storage_api::multipart_fence_acceptance::{LOCK_DIAGNOSTICS_FILTER, RUSTFS_META_MULTIPART_BUCKET};
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
config::retry::RetryConfig,
|
||||
@@ -38,14 +38,16 @@ const DEADLINE: Duration = Duration::from_secs(45);
|
||||
async fn fixture(
|
||||
drives_per_node: usize,
|
||||
storage_class: &str,
|
||||
lock_diagnostics: bool,
|
||||
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
|
||||
fixture_from_binary(drives_per_node, storage_class, None).await
|
||||
fixture_from_binary(drives_per_node, storage_class, None, lock_diagnostics).await
|
||||
}
|
||||
|
||||
async fn fixture_from_binary(
|
||||
drives_per_node: usize,
|
||||
storage_class: &str,
|
||||
binary: Option<&std::path::Path>,
|
||||
lock_diagnostics: bool,
|
||||
) -> TestResult<(RustFSTestClusterEnvironment, Vec<RpcFaultProxy>, Vec<Client>, String)> {
|
||||
let mut cluster =
|
||||
RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, drives_per_node)).await?;
|
||||
@@ -90,6 +92,10 @@ async fn fixture_from_binary(
|
||||
] {
|
||||
cluster.set_env(name, value);
|
||||
}
|
||||
if lock_diagnostics {
|
||||
cluster.set_env("RUST_LOG", LOCK_DIAGNOSTICS_FILTER);
|
||||
cluster.set_env("RUSTFS_OBJECT_LOCK_DIAG_ENABLE", "true");
|
||||
}
|
||||
let mut proxies = Vec::new();
|
||||
if binary.is_none() {
|
||||
for node in 0..4 {
|
||||
@@ -363,26 +369,120 @@ fn upload_path_scope(bucket: &str, key: &str, upload_id: &str) -> TestResult<Str
|
||||
))
|
||||
}
|
||||
|
||||
async fn with_paused_peers<T: Send + 'static>(
|
||||
async fn with_paused_peers<T, E>(
|
||||
proxies: &[RpcFaultProxy],
|
||||
methods: &[&'static str],
|
||||
limits: &[usize],
|
||||
scope: &str,
|
||||
request: impl std::future::Future<Output = T> + Send + 'static,
|
||||
) -> TestResult<T> {
|
||||
request: impl std::future::Future<Output = Result<T, SdkError<E>>> + Send + 'static,
|
||||
) -> TestResult<Result<T, SdkError<E>>>
|
||||
where
|
||||
T: Send + 'static,
|
||||
E: std::error::Error + ProvideErrorMetadata + Send + Sync + 'static,
|
||||
{
|
||||
let deadline = tokio::time::Instant::now() + DEADLINE;
|
||||
let pauses: Vec<_> = proxies[1..=limits.len()]
|
||||
.iter()
|
||||
.zip(limits)
|
||||
.map(|(proxy, limit)| proxy.pause(methods, *limit, Some(scope)))
|
||||
.collect::<Result<_, _>>()?;
|
||||
let task = tokio::spawn(request);
|
||||
wait_for_faults(&pauses, methods).await?;
|
||||
let result = tokio::time::timeout(DEADLINE, task).await??;
|
||||
let mut task = tokio::spawn(request);
|
||||
let mut joined = None;
|
||||
let mut deadline_expired = false;
|
||||
let mut barrier_result = {
|
||||
let faults = wait_for_faults(&pauses, methods);
|
||||
tokio::pin!(faults);
|
||||
match tokio::time::timeout_at(deadline, async {
|
||||
tokio::select! {
|
||||
result = &mut faults => result,
|
||||
result = &mut task => {
|
||||
joined = Some(result);
|
||||
// A ready request is not evidence that every selected fault
|
||||
// was reached. Poll the original barrier once for that proof.
|
||||
match futures::poll!(faults.as_mut()) {
|
||||
std::task::Poll::Ready(result) => result,
|
||||
std::task::Poll::Pending => Err("SDK request completed before the selected RPC fault barrier".into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
deadline_expired = true;
|
||||
Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into())
|
||||
}
|
||||
}
|
||||
};
|
||||
// Tokio's timeout polls an already-ready inner future before its timer.
|
||||
// A late observation cannot certify the original absolute deadline.
|
||||
deadline_expired |= tokio::time::Instant::now() >= deadline;
|
||||
if deadline_expired {
|
||||
barrier_result = Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"selected RPC fault proof observed after the absolute MPU deadline",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if barrier_result.is_err() {
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
}
|
||||
let mut client_task_aborted = false;
|
||||
let request_result = match joined {
|
||||
Some(result) => result,
|
||||
None => match tokio::time::timeout_at(deadline, &mut task).await {
|
||||
Ok(result) => result,
|
||||
Err(error) => {
|
||||
deadline_expired = true;
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
barrier_result = Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into());
|
||||
// This owns only the test's SDK request, not server/native work.
|
||||
// Retire the client task without granting another operation budget.
|
||||
client_task_aborted = true;
|
||||
task.abort();
|
||||
task.await
|
||||
}
|
||||
},
|
||||
};
|
||||
deadline_expired |= tokio::time::Instant::now() >= deadline;
|
||||
if deadline_expired {
|
||||
barrier_result = Err(std::io::Error::new(
|
||||
std::io::ErrorKind::TimedOut,
|
||||
"SDK request result observed after the absolute MPU deadline",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let hits: Vec<_> = pauses.iter().map(RpcPause::hits).collect();
|
||||
for pause in &pauses {
|
||||
pause.discard_pending();
|
||||
}
|
||||
drop(pauses);
|
||||
Ok(result)
|
||||
if barrier_result.is_err() {
|
||||
let sdk_error = request_result.as_ref().ok().and_then(|result| result.as_ref().err());
|
||||
let response = sdk_error.and_then(SdkError::raw_response);
|
||||
eprintln!(
|
||||
"MPU fault barrier failed: methods={methods:?}, hits={hits:?}, client_task_aborted={client_task_aborted}, deadline_expired={deadline_expired}, sdk_code={:?}, cause={:?}, request_id={:?}",
|
||||
sdk_error
|
||||
.and_then(|error| error.as_service_error())
|
||||
.and_then(|error| error.meta().code()),
|
||||
response.and_then(|response| response.headers().get("x-rustfs-error-cause")),
|
||||
response.and_then(|response| response.headers().get("x-amz-request-id")),
|
||||
);
|
||||
}
|
||||
match (barrier_result, request_result) {
|
||||
// Preserve the actual SDK source instead of replacing an early service
|
||||
// failure with an unrelated fault-wait timeout. The outer Err stays red.
|
||||
(Err(_), Ok(Err(error))) => Err(Box::new(error)),
|
||||
(Err(_), Err(error)) if !client_task_aborted => Err(Box::new(error)),
|
||||
(Err(error), _) => Err(error),
|
||||
(Ok(()), Ok(result)) => Ok(result),
|
||||
(Ok(()), Err(error)) => Err(Box::new(error)),
|
||||
}
|
||||
}
|
||||
|
||||
fn quorum_outcome<T: std::fmt::Debug, E: std::fmt::Debug + ProvideErrorMetadata>(
|
||||
@@ -1318,7 +1418,7 @@ async fn concurrent_capped_parts_respect_limit_on_distinct_nodes(clients: &[Clie
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", true).await?;
|
||||
list_parts_disk_loss_after_metadata_read(&cluster, &proxies, &clients, &bucket).await?;
|
||||
concurrent_capped_parts_respect_limit_on_distinct_nodes(&clients, &bucket).await?;
|
||||
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, false, 1, 3).await
|
||||
@@ -1326,13 +1426,13 @@ async fn metadata_delay_preserves_write_quorum_and_rejects_quorum_minus_one() ->
|
||||
|
||||
#[tokio::test]
|
||||
async fn lock_delay_preserves_admission_quorum_and_reports_timeout() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
phase_quorum_matrix(&cluster, &proxies, &clients, &bucket, true, 1, 3).await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult {
|
||||
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
|
||||
let storage = cluster_admin_ok(&cluster, http::Method::GET, "/rustfs/admin/v3/storageinfo", None).await?;
|
||||
let storage: serde_json::Value = serde_json::from_str(&storage)?;
|
||||
let storage: rustfs_madmin::StorageInfo = serde_json::from_value(storage["info"].clone())?;
|
||||
@@ -1346,7 +1446,7 @@ async fn sixteen_shard_multipart_metadata_requires_twelve_votes() -> TestResult
|
||||
#[tokio::test]
|
||||
async fn mixed_version_failed_complete_requires_current_coordinator_for_abort() -> TestResult {
|
||||
let previous = super::upgrade_test::source_binary()?;
|
||||
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous)).await?;
|
||||
let (mut cluster, mut proxies, clients, bucket) = fixture_from_binary(4, "EC:4", Some(&previous), false).await?;
|
||||
assert!(
|
||||
proxies.is_empty(),
|
||||
"the genuine previous fleet boots and seeds through its normal direct transport"
|
||||
@@ -1549,7 +1649,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
|
||||
capability_sha256: Option<[u8; 32]>,
|
||||
abort: Option<AbortRef>,
|
||||
}
|
||||
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4").await?;
|
||||
let (mut cluster, proxies, clients, bucket) = fixture(4, "EC:4", false).await?;
|
||||
let key = "partial-abort";
|
||||
let committed = payload_for("committed-before-abort", 4096);
|
||||
clients[0]
|
||||
@@ -1765,7 +1865,7 @@ async fn native_certified_partial_abort_retry_reclaims_survivors_below_metadata_
|
||||
#[tokio::test]
|
||||
async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation() -> TestResult {
|
||||
for size in [0, 8 * 1024 * 1024] {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let key = "late-complete";
|
||||
let upload = clients[0].create_multipart_upload().bucket(&bucket).key(key).send().await?;
|
||||
let upload_id = upload.upload_id().ok_or("missing upload ID")?;
|
||||
@@ -1892,7 +1992,7 @@ async fn late_complete_packets_cannot_publish_after_abort_and_bucket_recreation(
|
||||
async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
|
||||
for (method, settles) in [("PreparePartTransaction", false), ("SettlePartTransaction", true)] {
|
||||
for recreate_bucket in [false, true] {
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let key = "late-part-transaction";
|
||||
let previous = payload_for("committed-before-late-part", 4096);
|
||||
clients[0]
|
||||
@@ -2075,7 +2175,7 @@ async fn late_part_packets_preserve_retired_and_fresh_uploads() -> TestResult {
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestResult {
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
let pauses: Vec<_> = proxies[1..]
|
||||
.iter()
|
||||
.map(|proxy| proxy.pause(&["WriteMetadata"], usize::MAX, Some(&bucket)))
|
||||
@@ -2098,7 +2198,7 @@ async fn cancelled_metadata_wait_releases_fence_for_bucket_recreation() -> TestR
|
||||
async fn part_resend_owner_serializes_complete_and_abort_after_client_cancellation() -> TestResult {
|
||||
use md5::{Digest as _, Md5};
|
||||
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
for cancel_waiter in [false, true] {
|
||||
for complete in [false, true] {
|
||||
let key = format!("resend-{cancel_waiter}-{complete}");
|
||||
@@ -2586,7 +2686,7 @@ async fn resume_at_rpc<T: Send + 'static>(
|
||||
|
||||
#[tokio::test]
|
||||
async fn each_multipart_phase_resumes_after_metadata_and_lock_delays() -> TestResult {
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2").await?;
|
||||
let (_cluster, proxies, clients, bucket) = fixture(1, "EC:2", false).await?;
|
||||
for locks in [false, true] {
|
||||
let key = format!("phase-{locks}");
|
||||
let client = clients[0].clone();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1700,6 +1700,15 @@ mod tests {
|
||||
);
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(topology.cluster_topology()).await?;
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
let mrf_fallback_scope = if !background_enabled {
|
||||
let path = PathBuf::from(&cluster.temp_dir).join("skip-outage-mrf-fallback");
|
||||
std::fs::write(&path, format!("{bucket}/cluster/written-while-node-down-"))?;
|
||||
cluster.set_env("RUSTFS_E2E_SKIP_MRF_FALLBACK_PATH", path.to_string_lossy().into_owned());
|
||||
Some(path)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||
if let Some(storage_class) = evidence_case.and_then(|case| case.storage_class_standard) {
|
||||
@@ -1952,6 +1961,21 @@ mod tests {
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(path) = &mrf_fallback_scope {
|
||||
let observed = path.with_extension("observed");
|
||||
let expected = format!("{bucket}/{outage_key}");
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
if tokio::fs::read_to_string(&observed).await.ok().as_deref() == Some(expected.as_str()) {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err("outage repair did not reach the isolated best-effort fallback before target rejoin".into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
!outage_target_manifest_required
|
||||
|| target_set_peer_drives
|
||||
@@ -2045,7 +2069,12 @@ mod tests {
|
||||
} else {
|
||||
cluster_heal_is_idle(&recovered)
|
||||
};
|
||||
if ready {
|
||||
let pool_ready = if ready && background_enabled && !outage_write_deferred_until_rejoin {
|
||||
all_nodes_pool_metadata_readiness(&cluster, recovery_deadline).await.0
|
||||
} else {
|
||||
true
|
||||
};
|
||||
if ready && pool_ready {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= recovery_deadline {
|
||||
@@ -2542,6 +2571,10 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(path) = &mrf_fallback_scope {
|
||||
std::fs::remove_file(path)?;
|
||||
}
|
||||
|
||||
if let Some(cycle_end) = scanner_cycle_floor {
|
||||
wait_for_scanner_cycle_after(&cluster, cycle_end).await?;
|
||||
}
|
||||
|
||||
@@ -28,7 +28,8 @@ use hyper_util::{
|
||||
use prost::Message as _;
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, DeleteRequest, GenerallyLockRequest, PreparePartTransactionResponse, ReadVersionRequest,
|
||||
RecoverMultipartSourceRequest, RenameDataResponse, SettlePartTransactionRequest, SettlePartTransactionResponse,
|
||||
RecoverMultipartSourceRequest, RenameDataResponse, ScannerScopedDirtyUsageAckRequest, SettlePartTransactionRequest,
|
||||
SettlePartTransactionResponse,
|
||||
};
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
@@ -49,6 +50,9 @@ use tokio::{
|
||||
type ProxyBody = BoxBody<Bytes, io::Error>;
|
||||
type ProxyClient = Client<HttpConnector, ProxyBody>;
|
||||
|
||||
mod scoped_ack;
|
||||
pub(crate) use scoped_ack::{ScopedAckFault, ScopedAckQueue, ScopedAckScope};
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
enum Disposition {
|
||||
Hold,
|
||||
@@ -121,7 +125,8 @@ impl RpcPause {
|
||||
self.state.blocked.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Selected RPCs with successful HTTP/gRPC headers, excluding discarded packets.
|
||||
/// Selected RPCs with successful HTTP/gRPC response headers, excluding
|
||||
/// discarded packets. This does not prove a successful gRPC trailer or ACK apply.
|
||||
pub(crate) fn forwarded(&self) -> usize {
|
||||
self.state.forwarded.load(Ordering::SeqCst)
|
||||
}
|
||||
@@ -233,6 +238,7 @@ pub(crate) struct RpcFaultProxy {
|
||||
address: SocketAddr,
|
||||
slot: PauseSlot,
|
||||
lock_slot: PauseSlot,
|
||||
scoped_ack: scoped_ack::QueueSlot,
|
||||
task: JoinHandle<()>,
|
||||
late_requests: LateRequests,
|
||||
}
|
||||
@@ -245,6 +251,8 @@ impl RpcFaultProxy {
|
||||
let active = slot.clone();
|
||||
let lock_slot: PauseSlot = Arc::new(Mutex::new(None));
|
||||
let active_lock = lock_slot.clone();
|
||||
let scoped_ack = scoped_ack::empty_slot();
|
||||
let active_ack = scoped_ack.clone();
|
||||
let late_requests: LateRequests = Arc::new(Mutex::new(Some(Vec::new())));
|
||||
let late = late_requests.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
@@ -259,6 +267,7 @@ impl RpcFaultProxy {
|
||||
let h2 = h2.clone();
|
||||
let active = active.clone();
|
||||
let active_lock = active_lock.clone();
|
||||
let active_ack = active_ack.clone();
|
||||
let late = late.clone();
|
||||
connections.spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
@@ -272,6 +281,7 @@ impl RpcFaultProxy {
|
||||
None
|
||||
};
|
||||
let pause = owner_pause.or_else(|| active.lock().expect("RPC pause slot").clone());
|
||||
let ack = active_ack.lock().expect("scoped ACK queue slot").clone();
|
||||
let retain = pause.as_ref().is_some_and(|pause| {
|
||||
pause.retain_after_disconnect
|
||||
&& request.uri().path().strip_prefix("/node_service.NodeService/")
|
||||
@@ -280,11 +290,11 @@ impl RpcFaultProxy {
|
||||
let late = late.clone();
|
||||
async move {
|
||||
if !retain {
|
||||
return forward_request(request, client, target, pause).await;
|
||||
return forward_request(request, client, target, pause, ack).await;
|
||||
}
|
||||
// This opt-in fault owns the packet after client cancellation.
|
||||
// Proxy teardown still aborts every retained request.
|
||||
let task = tokio::spawn(forward_request(request, client, target, pause));
|
||||
let task = tokio::spawn(forward_request(request, client, target, pause, ack));
|
||||
{
|
||||
let mut requests = late.lock().expect("late RPC ownership");
|
||||
if let Some(requests) = requests.as_mut() {
|
||||
@@ -309,6 +319,7 @@ impl RpcFaultProxy {
|
||||
address,
|
||||
slot,
|
||||
lock_slot,
|
||||
scoped_ack,
|
||||
task,
|
||||
late_requests,
|
||||
})
|
||||
@@ -318,6 +329,11 @@ impl RpcFaultProxy {
|
||||
self.address
|
||||
}
|
||||
|
||||
/// Retain only genuine scoped ACKs for one receiver/session and bucket set.
|
||||
pub(crate) fn scoped_ack_queue(&self, scope: ScopedAckScope, capacity: usize) -> io::Result<ScopedAckQueue> {
|
||||
ScopedAckQueue::arm(self.scoped_ack.clone(), scope, capacity)
|
||||
}
|
||||
|
||||
/// Pause at most limit requests to the named unary methods; other streams
|
||||
/// on the same HTTP/2 connection continue forwarding.
|
||||
pub(crate) fn pause(&self, methods: &[&'static str], limit: usize, scope: Option<&str>) -> io::Result<RpcPause> {
|
||||
@@ -469,6 +485,9 @@ fn unary_payload(body: &[u8]) -> io::Result<&[u8]> {
|
||||
}
|
||||
|
||||
fn scope_matches(method: &str, body: &[u8], scope: &[u8]) -> io::Result<bool> {
|
||||
if method == "ScannerScopedDirtyUsageAck" {
|
||||
return scoped_ack_matches(body, Some(scope));
|
||||
}
|
||||
if method == "RecoverMultipartSource" {
|
||||
let request = RecoverMultipartSourceRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
|
||||
if request.protocol_version != u32::from(MULTIPART_SOURCE_PROTOCOL_VERSION) || request.path.as_bytes() != scope {
|
||||
@@ -525,7 +544,9 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
|
||||
let args = match method {
|
||||
"Lock" => vec![GenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args],
|
||||
"LockBatch" => BatchGenerallyLockRequest::decode(payload).map_err(io::Error::other)?.args,
|
||||
_ => return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method")),
|
||||
_ => {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "lock observer requires a lock method"));
|
||||
}
|
||||
};
|
||||
args.into_iter()
|
||||
.filter(|args| args.as_bytes().windows(scope.len()).any(|window| window == scope))
|
||||
@@ -540,18 +561,39 @@ fn scoped_lock_owners(method: &str, body: &[u8], scope: &[u8]) -> io::Result<Vec
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn scoped_ack_matches(body: &[u8], scope: Option<&[u8]>) -> io::Result<bool> {
|
||||
let request = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(body)?).map_err(io::Error::other)?;
|
||||
Ok(!request.probe_only && scope.is_none_or(|scope| request.entries.iter().any(|entry| entry.bucket.as_bytes() == scope)))
|
||||
}
|
||||
|
||||
async fn forward_request(
|
||||
request: Request<Incoming>,
|
||||
client: ProxyClient,
|
||||
target: SocketAddr,
|
||||
pause: Option<Arc<PauseState>>,
|
||||
ack: Option<Arc<scoped_ack::QueueState>>,
|
||||
) -> io::Result<hyper::Response<ProxyBody>> {
|
||||
let (mut parts, body) = request.into_parts();
|
||||
let mut body = body.map_err(io::Error::other).boxed();
|
||||
let request = request.map(|body| body.map_err(io::Error::other).boxed());
|
||||
let request = if let Some(ack) = ack
|
||||
&& request.uri().path() == "/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"
|
||||
{
|
||||
match scoped_ack::intercept(request, client.clone(), target, ack).await? {
|
||||
scoped_ack::Interception::Pass(request) => request,
|
||||
scoped_ack::Interception::Handled(response) => return Ok(response),
|
||||
}
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let (parts, body) = request.into_parts();
|
||||
let mut body = body;
|
||||
let mut observed = None;
|
||||
let mut inspection = None;
|
||||
if let Some(pause) = pause {
|
||||
let method = parts.uri.path().strip_prefix("/node_service.NodeService/");
|
||||
let method = parts
|
||||
.uri
|
||||
.path()
|
||||
.strip_prefix("/node_service.NodeService/")
|
||||
.or_else(|| parts.uri.path().strip_prefix("/node_service.ScannerControlService/"));
|
||||
if let Some(method) = method.filter(|name| pause.methods.contains(name)) {
|
||||
let mut route = None;
|
||||
let mut lock_owners = Vec::new();
|
||||
@@ -599,6 +641,16 @@ async fn forward_request(
|
||||
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
|
||||
.boxed();
|
||||
matches
|
||||
} else if method == "ScannerScopedDirtyUsageAck" {
|
||||
let collected = Limited::new(body, 1024 * 1024).collect().await.map_err(io::Error::other)?;
|
||||
let trailers = collected.trailers().cloned();
|
||||
let bytes = collected.to_bytes();
|
||||
let matches = scoped_ack_matches(&bytes, None)?;
|
||||
body = Full::new(bytes)
|
||||
.map_err(|never| match never {})
|
||||
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
|
||||
.boxed();
|
||||
matches
|
||||
} else {
|
||||
true
|
||||
};
|
||||
@@ -642,24 +694,7 @@ async fn forward_request(
|
||||
}
|
||||
}
|
||||
}
|
||||
if !parts.headers.contains_key(http::header::HOST)
|
||||
&& let Some(authority) = parts.uri.authority()
|
||||
{
|
||||
parts.headers.insert(
|
||||
http::header::HOST,
|
||||
http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
|
||||
);
|
||||
}
|
||||
parts.uri = http::Uri::builder()
|
||||
.scheme("http")
|
||||
.authority(target.to_string())
|
||||
.path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
|
||||
.build()
|
||||
.map_err(io::Error::other)?;
|
||||
let response = client
|
||||
.request(Request::from_parts(parts, body))
|
||||
.await
|
||||
.map_err(io::Error::other)?;
|
||||
let response = forward_to_backend(Request::from_parts(parts, body), client, target).await?;
|
||||
if let Some((pause, disk)) = inspection
|
||||
&& response.status().is_success()
|
||||
&& response.headers().get("grpc-status").is_some_and(|status| status == "12")
|
||||
@@ -712,6 +747,32 @@ async fn forward_request(
|
||||
Ok(response.map(|body| body.map_err(io::Error::other).boxed()))
|
||||
}
|
||||
|
||||
async fn forward_to_backend(
|
||||
request: Request<ProxyBody>,
|
||||
client: ProxyClient,
|
||||
target: SocketAddr,
|
||||
) -> io::Result<hyper::Response<Incoming>> {
|
||||
let (mut parts, body) = request.into_parts();
|
||||
if !parts.headers.contains_key(http::header::HOST)
|
||||
&& let Some(authority) = parts.uri.authority()
|
||||
{
|
||||
parts.headers.insert(
|
||||
http::header::HOST,
|
||||
http::HeaderValue::from_str(authority.as_str()).map_err(io::Error::other)?,
|
||||
);
|
||||
}
|
||||
parts.uri = http::Uri::builder()
|
||||
.scheme("http")
|
||||
.authority(target.to_string())
|
||||
.path_and_query(parts.uri.path_and_query().map_or("/", |value| value.as_str()))
|
||||
.build()
|
||||
.map_err(io::Error::other)?;
|
||||
client
|
||||
.request(Request::from_parts(parts, body))
|
||||
.await
|
||||
.map_err(io::Error::other)
|
||||
}
|
||||
|
||||
impl Drop for RpcFaultProxy {
|
||||
fn drop(&mut self) {
|
||||
if let Some(requests) = self.late_requests.lock().expect("late RPC ownership").take() {
|
||||
@@ -731,6 +792,7 @@ mod tests {
|
||||
|
||||
struct Backend {
|
||||
received: Arc<AtomicUsize>,
|
||||
auth_headers: Arc<Mutex<Vec<(String, String)>>>,
|
||||
address: SocketAddr,
|
||||
task: JoinHandle<()>,
|
||||
}
|
||||
@@ -754,6 +816,8 @@ mod tests {
|
||||
let address = listener.local_addr()?;
|
||||
let received = Arc::new(AtomicUsize::new(0));
|
||||
let received_task = received.clone();
|
||||
let auth_headers = Arc::new(Mutex::new(Vec::new()));
|
||||
let backend_auth = auth_headers.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut children = JoinSet::new();
|
||||
loop {
|
||||
@@ -761,10 +825,12 @@ mod tests {
|
||||
accepted = listener.accept() => {
|
||||
let Ok((socket, _)) = accepted else { break };
|
||||
let received = received_task.clone();
|
||||
let backend_auth = backend_auth.clone();
|
||||
let reply = reply.clone();
|
||||
children.spawn(async move {
|
||||
let service = service_fn(move |request: Request<Incoming>| {
|
||||
let received = received.clone();
|
||||
let backend_auth = backend_auth.clone();
|
||||
let reply = reply.clone();
|
||||
async move {
|
||||
received.fetch_add(1, Ordering::SeqCst);
|
||||
@@ -773,6 +839,10 @@ mod tests {
|
||||
http::HeaderValue::from_str(authority.as_str()).expect("valid HTTP authority")
|
||||
})
|
||||
});
|
||||
if let (Some(nonce), Some(signature)) = (request.headers().get("x-rustfs-rpc-nonce"), request.headers().get("x-rustfs-rpc-signature-v2")) {
|
||||
backend_auth.lock().expect("backend auth header observations").push((
|
||||
nonce.to_str().expect("test nonce").to_string(), signature.to_str().expect("test signature").to_string()));
|
||||
}
|
||||
let body = request.into_body().collect().await?.to_bytes();
|
||||
let body = reply.unwrap_or(body);
|
||||
let mut trailers = http::HeaderMap::new();
|
||||
@@ -795,7 +865,12 @@ mod tests {
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(Backend { received, address, task })
|
||||
Ok(Backend {
|
||||
received,
|
||||
auth_headers,
|
||||
address,
|
||||
task,
|
||||
})
|
||||
}
|
||||
|
||||
async fn call(client: &ProxyClient, address: SocketAddr, method: &str) -> io::Result<hyper::Response<Incoming>> {
|
||||
@@ -809,9 +884,14 @@ mod tests {
|
||||
bytes: Bytes,
|
||||
) -> io::Result<hyper::Response<Incoming>> {
|
||||
let body: ProxyBody = Full::new(bytes).map_err(|never| match never {}).boxed();
|
||||
let service = if method == "ScannerScopedDirtyUsageAck" {
|
||||
"ScannerControlService"
|
||||
} else {
|
||||
"NodeService"
|
||||
};
|
||||
let request = Request::builder()
|
||||
.method("POST")
|
||||
.uri(format!("http://{address}/node_service.NodeService/{method}"))
|
||||
.uri(format!("http://{address}/node_service.{service}/{method}"))
|
||||
.header(http::header::TE, "trailers")
|
||||
.body(body)
|
||||
.map_err(io::Error::other)?;
|
||||
@@ -826,6 +906,353 @@ mod tests {
|
||||
frame.into()
|
||||
}
|
||||
|
||||
const ACK_OWNER: &str = "11111111-1111-1111-1111-111111111111";
|
||||
const ACK_INSTANCE: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||
|
||||
fn ack_scope() -> ScopedAckScope {
|
||||
ScopedAckScope {
|
||||
nonce: uuid::Uuid::new_v4().to_string(),
|
||||
owner_id: ACK_OWNER.to_string(),
|
||||
instance_id: ACK_INSTANCE.to_string(),
|
||||
buckets: std::collections::BTreeSet::from(["cold-bucket".to_string()]),
|
||||
}
|
||||
}
|
||||
|
||||
fn ack_request(bucket: &str, generation: u64) -> ScannerScopedDirtyUsageAckRequest {
|
||||
ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: vec![7; 16].into(),
|
||||
protocol_version: 1,
|
||||
owner_id: ACK_OWNER.to_string(),
|
||||
instance_id: ACK_INSTANCE.to_string(),
|
||||
scope: 1,
|
||||
probe_only: false,
|
||||
entries: vec![rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry {
|
||||
bucket: bucket.to_string(),
|
||||
bucket_incarnation: vec![4; 16].into(),
|
||||
generation,
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
fn ack_frame(request: &ScannerScopedDirtyUsageAckRequest) -> Bytes {
|
||||
let payload = request.encode_to_vec();
|
||||
let mut frame = vec![0];
|
||||
frame.extend_from_slice(&u32::try_from(payload.len()).expect("bounded test frame").to_be_bytes());
|
||||
frame.extend_from_slice(&payload);
|
||||
Bytes::from(frame)
|
||||
}
|
||||
|
||||
fn ack_hash(bytes: &[u8]) -> String {
|
||||
use sha2::Digest as _;
|
||||
hex_simd::encode_to_string(&sha2::Sha256::digest(bytes)[..], hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
// Transport-only fixtures: the distributed tests supply production signatures.
|
||||
async fn call_ack_frame(
|
||||
client: &ProxyClient,
|
||||
address: SocketAddr,
|
||||
frame: Bytes,
|
||||
nonce: &str,
|
||||
) -> io::Result<hyper::Response<Incoming>> {
|
||||
let request = Request::builder()
|
||||
.method("POST")
|
||||
.uri(format!("http://{address}/node_service.ScannerControlService/ScannerScopedDirtyUsageAck"))
|
||||
.header(http::header::TE, "trailers")
|
||||
.header("x-rustfs-rpc-nonce", nonce)
|
||||
.header("x-rustfs-rpc-signature-v2", format!("opaque-signature-{nonce}"))
|
||||
.body(Full::new(frame).map_err(|never| match never {}).boxed())
|
||||
.map_err(io::Error::other)?;
|
||||
client.request(request).await.map_err(io::Error::other)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_loss_preserves_backend_boundary() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
for fault in [ScopedAckFault::RequestLoss, ScopedAckFault::ResponseLoss] {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let frame = ack_frame(&ack_request("cold-bucket", 7));
|
||||
let address = proxy.local_addr();
|
||||
let body = frame.clone();
|
||||
let caller = tokio::spawn(async move { call_ack_frame(&client, address, body, "loss").await });
|
||||
queue.wait_for_packets(1).await.expect("captured ACK");
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 0, "retention must precede backend dispatch");
|
||||
queue.settle(0, fault).expect("settle actual retained packet");
|
||||
assert!(
|
||||
caller.await.expect("caller task").is_err(),
|
||||
"the caller must observe the injected transport failure"
|
||||
);
|
||||
let receipt = queue.wait_for_completion(0).await.expect("complete fault receipt");
|
||||
let response_loss = fault == ScopedAckFault::ResponseLoss;
|
||||
assert_eq!(receipt["forwarded"], response_loss);
|
||||
assert_eq!(receipt["backend_complete"], response_loss);
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), usize::from(response_loss));
|
||||
if response_loss {
|
||||
assert_eq!(receipt["outcome"], "response_lost");
|
||||
assert_eq!(receipt["response"]["grpc_status"], "0", "status exists only in real backend trailers");
|
||||
assert_eq!(receipt["response"]["bytes"], frame.len());
|
||||
assert_eq!(receipt["response"]["sha256"], ack_hash(&frame));
|
||||
assert!(
|
||||
receipt["response"]["trailers_sha256"]
|
||||
.as_str()
|
||||
.is_some_and(|value| value.len() == 64)
|
||||
);
|
||||
} else {
|
||||
assert_eq!(receipt["outcome"], "request_lost");
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded ACK loss checks");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_keeps_cancelled_packets_and_releases_out_of_order() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let queue = proxy.scoped_ack_queue(ack_scope(), 2).expect("two-packet ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let first = ack_frame(&ack_request("cold-bucket", 7));
|
||||
let second = ack_frame(&ack_request("cold-bucket", 9));
|
||||
let address = proxy.local_addr();
|
||||
let old_client = client.clone();
|
||||
let old_body = first.clone();
|
||||
let old_caller = tokio::spawn(async move { call_ack_frame(&old_client, address, old_body, "first").await });
|
||||
queue.wait_for_packets(1).await.expect("first retained packet");
|
||||
old_caller.abort();
|
||||
assert!(old_caller.await.expect_err("cancel original caller").is_cancelled());
|
||||
let new_body = second.clone();
|
||||
let new_caller = tokio::spawn(async move { call_ack_frame(&client, address, new_body, "second").await });
|
||||
queue.wait_for_packets(2).await.expect("second retained packet");
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 0);
|
||||
queue.settle(1, ScopedAckFault::Forward).expect("release newer packet first");
|
||||
let response = new_caller
|
||||
.await
|
||||
.expect("new caller task")
|
||||
.expect("new response")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("complete body");
|
||||
assert_eq!(response.trailers().expect("response trailers")["x-proof"], "unchanged-proof");
|
||||
assert_eq!(response.to_bytes(), second);
|
||||
queue.wait_for_completion(1).await.expect("new packet complete");
|
||||
queue
|
||||
.settle(0, ScopedAckFault::Forward)
|
||||
.expect("release old packet after its caller disappeared");
|
||||
let old = queue.wait_for_completion(0).await.expect("detached old packet complete");
|
||||
assert_eq!(old["backend_complete"], true);
|
||||
assert_eq!(old["request_sha256"], ack_hash(&first));
|
||||
assert_eq!(old["response"]["sha256"], ack_hash(&first));
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(
|
||||
*backend.auth_headers.lock().expect("backend header observations"),
|
||||
vec![
|
||||
("second".to_string(), "opaque-signature-second".to_string()),
|
||||
("first".to_string(), "opaque-signature-first".to_string())
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
queue.settle(0, ScopedAckFault::Forward).is_err(),
|
||||
"one original packet may only be forwarded once"
|
||||
);
|
||||
})
|
||||
.await
|
||||
.expect("bounded detached retention check");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_bounds_capacity_and_excludes_other_scopes() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
assert!(proxy.scoped_ack_queue(ack_scope(), 0).is_err());
|
||||
assert!(proxy.scoped_ack_queue(ack_scope(), 5).is_err());
|
||||
let mut invalid = ack_scope();
|
||||
invalid.nonce.clear();
|
||||
assert!(proxy.scoped_ack_queue(invalid, 1).is_err());
|
||||
let mut scope = ack_scope();
|
||||
scope.buckets.insert("cold-bucket-z".to_string());
|
||||
let queue = proxy.scoped_ack_queue(scope, 1).expect("bounded ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let mut probe = ack_request("cold-bucket", 7);
|
||||
probe.probe_only = true;
|
||||
let mut other_peer = ack_request("cold-bucket", 7);
|
||||
other_peer.owner_id = uuid::Uuid::new_v4().to_string();
|
||||
let mut other_instance = ack_request("cold-bucket", 7);
|
||||
other_instance.instance_id = "b".repeat(32);
|
||||
let mut invalid_challenge = ack_request("cold-bucket", 7);
|
||||
invalid_challenge.challenge = vec![7; 15].into();
|
||||
let mut noncanonical_entries = ack_request("cold-bucket-z", 7);
|
||||
noncanonical_entries.entries.extend(ack_request("cold-bucket", 7).entries);
|
||||
let unselected = [
|
||||
probe,
|
||||
other_peer,
|
||||
other_instance,
|
||||
ack_request("cold-bucket-other", 7),
|
||||
ack_request("cold-bucket", u64::MAX),
|
||||
invalid_challenge,
|
||||
noncanonical_entries,
|
||||
];
|
||||
let unselected_count = unselected.len();
|
||||
for (index, request) in unselected.into_iter().enumerate() {
|
||||
let body = ack_frame(&request);
|
||||
let response = call_ack_frame(&client, proxy.local_addr(), body.clone(), "unselected")
|
||||
.await
|
||||
.expect("unselected ACK progresses");
|
||||
assert_eq!(response.into_body().collect().await.expect("unselected body").to_bytes(), body);
|
||||
assert!(queue.receipts().is_empty(), "unselected or protocol-invalid ACK must not consume a slot");
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), index + 1);
|
||||
}
|
||||
assert!(queue.receipts().is_empty());
|
||||
let address = proxy.local_addr();
|
||||
let held_client = client.clone();
|
||||
let valid_frame = ack_frame(&ack_request("cold-bucket", 7));
|
||||
let held_body = valid_frame.clone();
|
||||
let held = tokio::spawn(async move { call_ack_frame(&held_client, address, held_body, "held").await });
|
||||
queue.wait_for_packets(1).await.expect("one matching packet");
|
||||
assert!(!held.is_finished(), "the subsequent valid ACK must be retained");
|
||||
assert_eq!(queue.receipts()[0]["request_sha256"], ack_hash(&valid_frame));
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count);
|
||||
let extra = call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 8)), "beyond-capacity")
|
||||
.await
|
||||
.expect("capacity overflow passes without retention");
|
||||
extra.into_body().collect().await.expect("overflow response");
|
||||
assert_eq!(queue.receipts().len(), 1);
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), unselected_count + 1);
|
||||
drop(queue);
|
||||
assert!(
|
||||
held.await.expect("held caller").is_err(),
|
||||
"dropping the session must cancel its retained packet"
|
||||
);
|
||||
assert_eq!(
|
||||
backend.received.load(Ordering::SeqCst),
|
||||
unselected_count + 1,
|
||||
"session drop must never forward the held packet"
|
||||
);
|
||||
})
|
||||
.await
|
||||
.expect("bounded scope and lifecycle checks");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_ack_queue_expiry_never_forwards_retained_packet() {
|
||||
let (backend, _proxy, queue, caller) = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy");
|
||||
let queue = proxy.scoped_ack_queue(ack_scope(), 1).expect("ACK queue");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let address = proxy.local_addr();
|
||||
let caller = tokio::spawn(async move {
|
||||
call_ack_frame(&client, address, ack_frame(&ack_request("cold-bucket", 7)), "expires").await
|
||||
});
|
||||
queue.wait_for_packets(1).await.expect("retained packet");
|
||||
(backend, proxy, queue, caller)
|
||||
})
|
||||
.await
|
||||
.expect("bounded queue setup");
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(Duration::from_secs(120)).await;
|
||||
tokio::time::resume();
|
||||
assert!(
|
||||
tokio::time::timeout(Duration::from_secs(10), caller)
|
||||
.await
|
||||
.expect("expiry completes caller")
|
||||
.expect("caller task")
|
||||
.is_err()
|
||||
);
|
||||
let receipt = queue.receipts().into_iter().next().expect("expired receipt");
|
||||
assert_eq!(receipt["completed"], true);
|
||||
assert_eq!(receipt["forwarded"], false);
|
||||
assert_eq!(receipt["backend_complete"], false);
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(
|
||||
queue
|
||||
.settle(0, ScopedAckFault::Forward)
|
||||
.expect_err("expired packet cannot be released")
|
||||
.kind(),
|
||||
io::ErrorKind::TimedOut
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_ack_fault_matches_exact_bucket_and_excludes_probes() {
|
||||
use rustfs_protos::proto_gen::node_service::ScannerScopedDirtyUsageEntry;
|
||||
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let backend = backend().await.expect("backend listener");
|
||||
let proxy = RpcFaultProxy::start_on("127.0.0.1:0".parse().expect("loopback"), backend.address)
|
||||
.await
|
||||
.expect("proxy listener");
|
||||
let client: ProxyClient = Client::builder(TokioExecutor::new()).http2_only(true).build_http();
|
||||
let pause = proxy
|
||||
.pause(&["ScannerScopedDirtyUsageAck"], 1, Some("cold-bucket"))
|
||||
.expect("scoped ACK fault");
|
||||
let frame = |bucket: &str, probe_only: bool| {
|
||||
let payload = ScannerScopedDirtyUsageAckRequest {
|
||||
challenge: vec![3; 16].into(),
|
||||
protocol_version: 1,
|
||||
owner_id: "11111111-1111-1111-1111-111111111111".to_string(),
|
||||
instance_id: "a".repeat(32),
|
||||
scope: 1,
|
||||
probe_only,
|
||||
entries: vec![ScannerScopedDirtyUsageEntry {
|
||||
bucket: bucket.to_string(),
|
||||
bucket_incarnation: vec![4; 16].into(),
|
||||
generation: 17,
|
||||
}],
|
||||
}
|
||||
.encode_to_vec();
|
||||
let mut frame = vec![0];
|
||||
frame.extend_from_slice(&u32::try_from(payload.len()).expect("small ACK frame").to_be_bytes());
|
||||
frame.extend_from_slice(&payload);
|
||||
Bytes::from(frame)
|
||||
};
|
||||
for bytes in [frame("cold-bucket", true), frame("cold-bucket-other", false)] {
|
||||
let response = call_body(&client, proxy.local_addr(), "ScannerScopedDirtyUsageAck", bytes.clone())
|
||||
.await
|
||||
.expect("probe and other bucket must progress");
|
||||
assert_eq!(response.into_body().collect().await.expect("unselected response").to_bytes(), bytes);
|
||||
}
|
||||
assert_eq!(pause.hits(), 0);
|
||||
let signed_bytes = frame("cold-bucket", false);
|
||||
let task_bytes = signed_bytes.clone();
|
||||
let address = proxy.local_addr();
|
||||
let blocked =
|
||||
tokio::spawn(async move { call_body(&client, address, "ScannerScopedDirtyUsageAck", task_bytes).await });
|
||||
pause
|
||||
.wait_for_hits(1)
|
||||
.await
|
||||
.expect("real ScannerControlService method must reach the fault");
|
||||
assert!(!blocked.is_finished());
|
||||
assert_eq!(backend.received.load(Ordering::SeqCst), 2, "held ACK must not apply at the backend");
|
||||
pause.release();
|
||||
let collected = blocked
|
||||
.await
|
||||
.expect("ACK task")
|
||||
.expect("released ACK")
|
||||
.into_body()
|
||||
.collect()
|
||||
.await
|
||||
.expect("ACK body and trailers");
|
||||
assert_eq!(collected.trailers().expect("ACK trailers")["x-proof"], "unchanged-proof");
|
||||
assert_eq!(collected.to_bytes(), signed_bytes, "proxy must preserve the complete signed ACK frame");
|
||||
})
|
||||
.await
|
||||
.expect("scanner ACK fault deadline");
|
||||
}
|
||||
|
||||
fn native_abort_request(path: &str) -> RecoverMultipartSourceRequest {
|
||||
use crate::storage_api::multipart_source_rpc::{
|
||||
MultipartAbortCertificate, MultipartAbortProof, MultipartAbortProofKind, MultipartAbortRound, MultipartSourcePhase,
|
||||
@@ -1719,3 +2146,4 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
// event-drill: isolated harness-only change
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Bounded retention of original scoped ACK frames, independent of caller cancellation.
|
||||
|
||||
use super::{ProxyBody, ProxyClient, forward_to_backend, unary_payload};
|
||||
use bytes::Bytes;
|
||||
use http_body_util::{BodyExt as _, Full, Limited};
|
||||
use hyper::{Request, Response};
|
||||
use prost::Message as _;
|
||||
use rustfs_protos::proto_gen::node_service::{ScannerScopedDirtyUsageAckRequest, ScannerScopedDirtyUsageAckResponse};
|
||||
use rustfs_protos::scoped_dirty_usage::validate_scoped_dirty_usage_request;
|
||||
use serde_json::{Value, json};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
io,
|
||||
net::SocketAddr,
|
||||
sync::{
|
||||
Arc, Mutex,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
sync::{Notify, oneshot, watch},
|
||||
task::AbortHandle,
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
const MAX_RETAINED: usize = 4;
|
||||
const MAX_BYTES: usize = 1024 * 1024;
|
||||
const RETENTION: Duration = Duration::from_secs(120);
|
||||
|
||||
pub(crate) struct ScopedAckScope {
|
||||
pub nonce: String,
|
||||
pub owner_id: String,
|
||||
pub instance_id: String,
|
||||
pub buckets: BTreeSet<String>,
|
||||
}
|
||||
|
||||
impl ScopedAckScope {
|
||||
fn validate(&self) -> io::Result<()> {
|
||||
let valid_uuid = |value: &str| uuid::Uuid::parse_str(value).is_ok_and(|value| !value.is_nil());
|
||||
if !valid_uuid(&self.nonce)
|
||||
|| !valid_uuid(&self.owner_id)
|
||||
|| self.instance_id.is_empty()
|
||||
|| self.instance_id.len() > 128
|
||||
|| self.buckets.is_empty()
|
||||
|| self.buckets.len() > 16
|
||||
|| self.buckets.iter().any(|bucket| bucket.is_empty() || bucket.len() > 63)
|
||||
{
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"scoped ACK queue needs a bounded session, receiver and bucket scope",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn matches(&self, request: &ScannerScopedDirtyUsageAckRequest) -> bool {
|
||||
!request.probe_only
|
||||
&& validate_scoped_dirty_usage_request(request).is_ok()
|
||||
&& request.owner_id == self.owner_id
|
||||
&& request.instance_id == self.instance_id
|
||||
&& request.entries.iter().all(|entry| self.buckets.contains(&entry.bucket))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum ScopedAckFault {
|
||||
Forward,
|
||||
RequestLoss,
|
||||
ResponseLoss,
|
||||
}
|
||||
|
||||
struct Packet {
|
||||
action: watch::Sender<Option<ScopedAckFault>>,
|
||||
receipt: Mutex<Value>,
|
||||
task: Mutex<Option<AbortHandle>>,
|
||||
}
|
||||
|
||||
pub(super) struct QueueState {
|
||||
scope: ScopedAckScope,
|
||||
capacity: usize,
|
||||
started: Instant,
|
||||
deadline: Instant,
|
||||
closed: AtomicBool,
|
||||
// The packet inventory lock is released before any packet lock or await.
|
||||
packets: Mutex<Vec<Arc<Packet>>>,
|
||||
changed: Notify,
|
||||
}
|
||||
|
||||
pub(super) type QueueSlot = Arc<Mutex<Option<Arc<QueueState>>>>;
|
||||
|
||||
pub(super) fn empty_slot() -> QueueSlot {
|
||||
Arc::new(Mutex::new(None))
|
||||
}
|
||||
|
||||
pub(crate) struct ScopedAckQueue {
|
||||
state: Arc<QueueState>,
|
||||
slot: QueueSlot,
|
||||
}
|
||||
|
||||
impl ScopedAckQueue {
|
||||
pub(super) fn arm(slot: QueueSlot, scope: ScopedAckScope, capacity: usize) -> io::Result<Self> {
|
||||
scope.validate()?;
|
||||
if !(1..=MAX_RETAINED).contains(&capacity) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "scoped ACK capacity must be 1..=4"));
|
||||
}
|
||||
let mut active = slot.lock().expect("scoped ACK queue slot");
|
||||
if active.is_some() {
|
||||
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK queue is already armed"));
|
||||
}
|
||||
let started = Instant::now();
|
||||
let state = Arc::new(QueueState {
|
||||
scope,
|
||||
capacity,
|
||||
started,
|
||||
deadline: started + RETENTION,
|
||||
closed: AtomicBool::new(false),
|
||||
packets: Mutex::new(Vec::new()),
|
||||
changed: Notify::new(),
|
||||
});
|
||||
*active = Some(state.clone());
|
||||
drop(active);
|
||||
Ok(Self { state, slot })
|
||||
}
|
||||
|
||||
pub(crate) fn receipts(&self) -> Vec<Value> {
|
||||
let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
|
||||
packets
|
||||
.iter()
|
||||
.map(|packet| packet.receipt.lock().expect("scoped ACK receipt").clone())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_packets(&self, count: usize) -> io::Result<()> {
|
||||
self.wait_until(|| self.receipts().len() >= count).await
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_completion(&self, id: usize) -> io::Result<Value> {
|
||||
self.wait_until(|| {
|
||||
self.receipts()
|
||||
.get(id)
|
||||
.is_some_and(|receipt| receipt["completed"].as_bool() == Some(true))
|
||||
})
|
||||
.await?;
|
||||
self.receipts()
|
||||
.get(id)
|
||||
.cloned()
|
||||
.ok_or_else(|| io::Error::other("scoped ACK receipt disappeared"))
|
||||
}
|
||||
|
||||
async fn wait_until(&self, ready: impl Fn() -> bool) -> io::Result<()> {
|
||||
tokio::time::timeout_at(self.state.deadline, async {
|
||||
loop {
|
||||
let changed = self.state.changed.notified();
|
||||
if ready() {
|
||||
return;
|
||||
}
|
||||
changed.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK queue lifetime elapsed"))
|
||||
}
|
||||
|
||||
pub(crate) fn settle(&self, id: usize, fault: ScopedAckFault) -> io::Result<()> {
|
||||
if Instant::now() >= self.state.deadline {
|
||||
return Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired"));
|
||||
}
|
||||
let packet = self
|
||||
.state
|
||||
.packets
|
||||
.lock()
|
||||
.expect("scoped ACK packets")
|
||||
.get(id)
|
||||
.cloned()
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::NotFound, "scoped ACK packet is not retained"))?;
|
||||
if !packet.action.send_if_modified(|current| {
|
||||
if current.is_none() {
|
||||
*current = Some(fault);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}) {
|
||||
return Err(io::Error::new(io::ErrorKind::AlreadyExists, "scoped ACK packet was already settled"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScopedAckQueue {
|
||||
fn drop(&mut self) {
|
||||
self.state.closed.store(true, Ordering::SeqCst);
|
||||
let mut slot = self.slot.lock().expect("scoped ACK queue slot");
|
||||
if slot.as_ref().is_some_and(|current| Arc::ptr_eq(current, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
drop(slot);
|
||||
let packets = self.state.packets.lock().expect("scoped ACK packets").clone();
|
||||
for packet in packets {
|
||||
if let Some(task) = packet.task.lock().expect("scoped ACK task").take() {
|
||||
task.abort();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) enum Interception {
|
||||
Pass(Request<ProxyBody>),
|
||||
Handled(Response<ProxyBody>),
|
||||
}
|
||||
|
||||
fn restored_body(bytes: Bytes, trailers: Option<http::HeaderMap>) -> ProxyBody {
|
||||
Full::new(bytes)
|
||||
.map_err(|never| match never {})
|
||||
.with_trailers(async move { trailers.map(Ok::<_, io::Error>) })
|
||||
.boxed()
|
||||
}
|
||||
|
||||
fn hash(bytes: &[u8]) -> String {
|
||||
hex(Sha256::digest(bytes).as_ref())
|
||||
}
|
||||
|
||||
fn hex(bytes: &[u8]) -> String {
|
||||
hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
fn trailer_receipt(trailers: Option<&http::HeaderMap>) -> Value {
|
||||
trailers.map_or(Value::Null, |headers| {
|
||||
json!(
|
||||
headers
|
||||
.iter()
|
||||
.map(|(name, value)| { json!({"name": name.as_str(), "value_hex": hex(value.as_bytes())}) })
|
||||
.collect::<Vec<_>>()
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn header_hash(headers: &http::HeaderMap) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
for (name, value) in headers {
|
||||
hasher.update(name.as_str().as_bytes());
|
||||
hasher.update([0]);
|
||||
hasher.update(value.as_bytes());
|
||||
hasher.update([0]);
|
||||
}
|
||||
hex(hasher.finalize().as_ref())
|
||||
}
|
||||
|
||||
fn millis(state: &QueueState) -> u64 {
|
||||
u64::try_from(state.started.elapsed().as_millis()).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
pub(super) async fn intercept(
|
||||
request: Request<ProxyBody>,
|
||||
client: ProxyClient,
|
||||
target: SocketAddr,
|
||||
state: Arc<QueueState>,
|
||||
) -> io::Result<Interception> {
|
||||
if Instant::now() >= state.deadline || state.closed.load(Ordering::SeqCst) {
|
||||
return Ok(Interception::Pass(request));
|
||||
}
|
||||
let (parts, body) = request.into_parts();
|
||||
let collected = tokio::time::timeout_at(state.deadline, Limited::new(body, MAX_BYTES).collect())
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "scoped ACK frame collection expired"))?
|
||||
.map_err(io::Error::other)?;
|
||||
let trailers = collected.trailers().cloned();
|
||||
let bytes = collected.to_bytes();
|
||||
let decoded = ScannerScopedDirtyUsageAckRequest::decode(unary_payload(&bytes)?).map_err(io::Error::other)?;
|
||||
let request = Request::from_parts(parts, restored_body(bytes.clone(), trailers.clone()));
|
||||
if Instant::now() >= state.deadline || !state.scope.matches(&decoded) {
|
||||
return Ok(Interception::Pass(request));
|
||||
}
|
||||
let header = |name| request.headers().get(name).and_then(|value| value.to_str().ok());
|
||||
let (action, mut selected) = watch::channel(None);
|
||||
let packet = {
|
||||
let mut packets = state.packets.lock().expect("scoped ACK packets");
|
||||
if packets.len() >= state.capacity || state.closed.load(Ordering::SeqCst) {
|
||||
return Ok(Interception::Pass(request));
|
||||
}
|
||||
let id = packets.len();
|
||||
let packet = Arc::new(Packet {
|
||||
action,
|
||||
task: Mutex::new(None),
|
||||
receipt: Mutex::new(json!({
|
||||
"id": id, "session_nonce": state.scope.nonce, "receiver": target.to_string(),
|
||||
"authority": request.uri().authority().map(|value| value.as_str()),
|
||||
"captured_at_ms": millis(&state), "completed": false, "forwarded": false,
|
||||
"backend_complete": false, "outcome": "held", "request_bytes": bytes.len(),
|
||||
"request_sha256": hash(&bytes), "request_frame_hex": hex(&bytes), "request_headers_sha256": header_hash(request.headers()),
|
||||
"request_trailers_sha256": trailers.as_ref().map(header_hash),
|
||||
"request_trailers": trailer_receipt(trailers.as_ref()),
|
||||
"auth_nonce": header("x-rustfs-rpc-nonce"), "replay_nonce": header("x-rustfs-rpc-replay-nonce"),
|
||||
"signature_sha256": header("x-rustfs-rpc-signature-v2").map(|value| hash(value.as_bytes())),
|
||||
"replay_signature_sha256": header("x-rustfs-rpc-signature-v3").map(|value| hash(value.as_bytes())),
|
||||
"request": {"owner_id": decoded.owner_id, "instance_id": decoded.instance_id,
|
||||
"probe_only": decoded.probe_only, "entries": decoded.entries.iter().map(|entry| json!({
|
||||
"bucket": entry.bucket, "bucket_incarnation": uuid::Uuid::from_slice(&entry.bucket_incarnation).ok().map(|value| value.to_string()),
|
||||
"generation": entry.generation })).collect::<Vec<_>>() }
|
||||
})),
|
||||
});
|
||||
packets.push(packet.clone());
|
||||
packet
|
||||
};
|
||||
let (reply, response) = oneshot::channel();
|
||||
let worker_packet = packet.clone();
|
||||
let worker_state = state.clone();
|
||||
// This owner survives an HTTP/2 stream reset from the original 5 s caller.
|
||||
// Every request is settled once and is discarded at the session deadline.
|
||||
let task = tokio::spawn(async move {
|
||||
let result = tokio::time::timeout_at(worker_state.deadline, async {
|
||||
let fault = loop {
|
||||
if let Some(fault) = *selected.borrow_and_update() {
|
||||
break fault;
|
||||
}
|
||||
selected
|
||||
.changed()
|
||||
.await
|
||||
.map_err(|_| io::Error::other("scoped ACK queue closed"))?;
|
||||
};
|
||||
if fault == ScopedAckFault::RequestLoss {
|
||||
worker_packet.receipt.lock().expect("scoped ACK receipt")["outcome"] = json!("request_lost");
|
||||
return Err(io::Error::new(io::ErrorKind::ConnectionAborted, "injected scoped ACK request loss"));
|
||||
}
|
||||
if worker_state.closed.load(Ordering::SeqCst) || Instant::now() >= worker_state.deadline {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::ConnectionAborted,
|
||||
"scoped ACK session ended or expired before forwarding",
|
||||
));
|
||||
}
|
||||
{
|
||||
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
|
||||
receipt["forwarded"] = json!(true);
|
||||
receipt["forwarded_at_ms"] = json!(millis(&worker_state));
|
||||
}
|
||||
let backend = forward_to_backend(request, client, target).await?;
|
||||
let (parts, body) = backend.into_parts();
|
||||
let collected = Limited::new(body, MAX_BYTES).collect().await.map_err(io::Error::other)?;
|
||||
let trailers = collected.trailers().cloned();
|
||||
let bytes = collected.to_bytes();
|
||||
let grpc_status = trailers
|
||||
.as_ref()
|
||||
.and_then(|values| values.get("grpc-status"))
|
||||
.or_else(|| parts.headers.get("grpc-status"))
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let response = unary_payload(&bytes)
|
||||
.ok()
|
||||
.and_then(|payload| ScannerScopedDirtyUsageAckResponse::decode(payload).ok())
|
||||
.map(|response| {
|
||||
json!({"owner_id": response.owner_id, "instance_id": response.instance_id,
|
||||
"supported": response.supported, "cleared": response.cleared,
|
||||
"response_proof_bytes": response.response_proof.len()})
|
||||
});
|
||||
{
|
||||
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
|
||||
receipt["backend_complete"] = json!(true);
|
||||
receipt["backend_completed_at_ms"] = json!(millis(&worker_state));
|
||||
receipt["response"] = json!({"status": parts.status.as_u16(), "grpc_status": grpc_status,
|
||||
"bytes": bytes.len(), "sha256": hash(&bytes), "frame_hex": hex(&bytes), "headers_sha256": header_hash(&parts.headers),
|
||||
"trailers": trailer_receipt(trailers.as_ref()),
|
||||
"trailers_sha256": trailers.as_ref().map(header_hash), "ack": response});
|
||||
receipt["outcome"] = json!(if fault == ScopedAckFault::ResponseLoss {
|
||||
"response_lost"
|
||||
} else {
|
||||
"forwarded"
|
||||
});
|
||||
}
|
||||
if fault == ScopedAckFault::ResponseLoss {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::ConnectionAborted,
|
||||
"injected scoped ACK response loss after backend trailers",
|
||||
));
|
||||
}
|
||||
Ok(Response::from_parts(parts, restored_body(bytes, trailers)))
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| Err(io::Error::new(io::ErrorKind::TimedOut, "scoped ACK retention expired")));
|
||||
{
|
||||
let mut receipt = worker_packet.receipt.lock().expect("scoped ACK receipt");
|
||||
if result.is_err() && receipt["outcome"] == "held" {
|
||||
receipt["outcome"] = json!("transport_failed_or_expired");
|
||||
}
|
||||
receipt["completed"] = json!(true);
|
||||
receipt["completed_at_ms"] = json!(millis(&worker_state));
|
||||
}
|
||||
let _ = reply.send(result);
|
||||
worker_state.changed.notify_one();
|
||||
});
|
||||
*packet.task.lock().expect("scoped ACK task") = Some(task.abort_handle());
|
||||
if state.closed.load(Ordering::SeqCst) {
|
||||
task.abort();
|
||||
}
|
||||
drop(task);
|
||||
state.changed.notify_one();
|
||||
response
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::ConnectionAborted, "scoped ACK queue ended"))?
|
||||
.map(Interception::Handled)
|
||||
}
|
||||
@@ -70,6 +70,9 @@ pub(crate) mod multipart_rpc_delay {
|
||||
#[cfg(test)]
|
||||
pub(crate) mod multipart_fence_acceptance {
|
||||
pub(crate) use super::RUSTFS_META_MULTIPART_BUCKET;
|
||||
|
||||
/// Targeted lock diagnostics for the metadata-delay quorum case.
|
||||
pub(crate) const LOCK_DIAGNOSTICS_FILTER: &str = "warn,rustfs_lock::distributed_lock=debug,rustfs_ecstore::object_lock_diag=trace,rustfs_ecstore::cluster::rpc::remote_locker=info";
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -80,3 +83,14 @@ pub(crate) mod multipart_source_rpc {
|
||||
PublicationIdentity, SourceGeneration, SourceGeometry, SourceLineage,
|
||||
};
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod scanner_peer_bucket {
|
||||
use rustfs_ecstore::api::disk as ecstore_disk;
|
||||
use rustfs_ecstore::api::erasure as ecstore_erasure;
|
||||
use rustfs_ecstore::api::rpc as ecstore_rpc;
|
||||
|
||||
pub(crate) use ecstore_disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
pub(crate) use ecstore_erasure::{ReedSolomonEncoder, calc_shard_size};
|
||||
pub(crate) use ecstore_rpc::PeerRestClient;
|
||||
}
|
||||
|
||||
@@ -1112,8 +1112,9 @@ async fn wait_for_upgrade_writes(
|
||||
match response {
|
||||
Ok(Ok(_)) => break,
|
||||
Ok(Err(error)) => {
|
||||
let code = error.as_service_error().and_then(ProvideErrorMetadata::code);
|
||||
if error.raw_response().map(|response| response.status().as_u16()) != Some(503)
|
||||
|| error.as_service_error().and_then(ProvideErrorMetadata::code) != Some("ServiceUnavailable")
|
||||
|| !matches!(code, Some("ServiceUnavailable" | "SlowDown" | "SlowDownRead"))
|
||||
{
|
||||
return Err(format!("{phase}: node {node} {probe} failed: {error:?}").into());
|
||||
}
|
||||
|
||||
@@ -225,9 +225,11 @@ metrics = { workspace = true }
|
||||
# The guard scripts/check_no_tokio_io_uring.sh allows an explicit
|
||||
# io-uring integration; only the tokio "io-uring" runtime feature is banned.
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
# SharedReadBudget and the bounded read APIs are available in the 0.3.0
|
||||
# registry release; keep the application on the published contract.
|
||||
rustfs-uring = "0.3.0"
|
||||
# Native completion owners keep physical I/O leases alive after cancellation.
|
||||
rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf" }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dev-dependencies]
|
||||
rustfs-uring = { git = "https://github.com/rustfs/uring.git", rev = "eb8fc160e71d804427de1e905f5c226d580f71bf", features = ["fault-injection"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
winapi-util.workspace = true
|
||||
|
||||
@@ -447,6 +447,9 @@ pub mod data_usage {
|
||||
}
|
||||
|
||||
pub mod disk {
|
||||
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
pub use crate::disk::local::{ReplacementExecutionLease, ScanGuard};
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -475,6 +478,10 @@ pub mod disk {
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
|
||||
validate_batch_read_version_item_count,
|
||||
};
|
||||
pub use crate::disk::{
|
||||
BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler,
|
||||
PhysicalResourcePlan, RemoteBackgroundIo, with_background_resource_unit, with_remote_background_io,
|
||||
};
|
||||
pub use bytes::Bytes;
|
||||
pub use endpoint::Endpoint;
|
||||
pub use error::DiskError;
|
||||
@@ -615,6 +622,7 @@ pub mod rio {
|
||||
}
|
||||
|
||||
pub mod rpc {
|
||||
pub use crate::cluster::rpc::http_auth::with_rpc_background_request;
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
|
||||
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
|
||||
@@ -624,8 +632,8 @@ pub mod rpc {
|
||||
TONIC_RPC_PREFIX, TonicInterceptor, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce,
|
||||
decode_heal_bucket_rpc_options, encode_heal_bucket_rpc_options, gen_signature_headers, gen_tonic_replay_scope_headers,
|
||||
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, rpc_background_admission,
|
||||
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
@@ -675,8 +683,9 @@ pub mod storage {
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
all_local_disk_path, find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
pub use crate::store::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
}
|
||||
|
||||
@@ -1503,7 +1503,11 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
|
||||
bucket: &str,
|
||||
parse: bool,
|
||||
) -> Result<(BucketMetadata, bool)> {
|
||||
let (mut bm, persisted) = match read_bucket_metadata(api.clone(), bucket).await {
|
||||
let (metadata_res, incarnation_res) = tokio::join!(
|
||||
Box::pin(read_bucket_metadata(api.clone(), bucket)),
|
||||
Box::pin(load_bucket_incarnation(api.clone(), bucket)),
|
||||
);
|
||||
let (mut bm, persisted) = match metadata_res {
|
||||
Ok(res) => (res, true),
|
||||
Err(err) => {
|
||||
if err != Error::ConfigNotFound {
|
||||
@@ -1514,7 +1518,7 @@ pub(crate) async fn load_bucket_metadata_parse_with_presence(
|
||||
}
|
||||
};
|
||||
|
||||
let incarnation = load_bucket_incarnation(api.clone(), bucket).await?;
|
||||
let incarnation = incarnation_res?;
|
||||
if persisted {
|
||||
if bm.name.is_empty() {
|
||||
bm.name = bucket.to_string();
|
||||
|
||||
@@ -1435,6 +1435,7 @@ pub(crate) async fn get_object_lock_config_state_in(
|
||||
|
||||
/// Re-read the Object Lock state and bucket incarnation from the same
|
||||
/// authoritative metadata blob while the caller holds the transaction lock.
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
@@ -1453,6 +1454,27 @@ pub(crate) async fn get_object_lock_config_and_incarnation_from_disk_with_guard_
|
||||
get_delete_config_from_disk_in(ctx, bucket, guard, None).await
|
||||
}
|
||||
|
||||
#[allow(clippy::type_complexity)]
|
||||
pub(crate) async fn get_object_lock_and_quota_config_from_disk_with_guard_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(
|
||||
ObjectLockConfigState,
|
||||
Uuid,
|
||||
OffsetDateTime,
|
||||
std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>,
|
||||
)> {
|
||||
let (state, incarnation_id, updated_at, quota_res) =
|
||||
get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, None, true).await?;
|
||||
Ok((
|
||||
state,
|
||||
incarnation_id,
|
||||
updated_at,
|
||||
quota_res.unwrap_or(Ok((None, incarnation_id, updated_at))),
|
||||
))
|
||||
}
|
||||
|
||||
/// Validate queued expiration against the same fresh metadata used for Object
|
||||
/// Lock admission. The caller holds the transaction read lock through commit,
|
||||
/// so a successful lifecycle update cannot be followed by an old-policy delete.
|
||||
@@ -1462,6 +1484,24 @@ pub(crate) async fn get_delete_config_from_disk_in(
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
|
||||
) -> Result<(ObjectLockConfigState, Uuid, OffsetDateTime)> {
|
||||
let (state, incarnation_id, updated_at, _) =
|
||||
get_delete_and_quota_config_from_disk_in(ctx, bucket, guard, expiry_policy, false).await?;
|
||||
Ok((state, incarnation_id, updated_at))
|
||||
}
|
||||
|
||||
#[allow(clippy::type_complexity)]
|
||||
async fn get_delete_and_quota_config_from_disk_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
expiry_policy: Option<&rustfs_lifecycle::ExpiryPolicySnapshot>,
|
||||
include_quota: bool,
|
||||
) -> Result<(
|
||||
ObjectLockConfigState,
|
||||
Uuid,
|
||||
OffsetDateTime,
|
||||
Option<std::result::Result<(Option<BucketQuota>, Uuid, OffsetDateTime), String>>,
|
||||
)> {
|
||||
let bucket_meta_sys_lock = bucket_metadata_sys_of(ctx)?;
|
||||
let bucket_meta_sys = bucket_meta_sys_lock.read().await.clone();
|
||||
|
||||
@@ -1477,7 +1517,7 @@ pub(crate) async fn get_delete_config_from_disk_in(
|
||||
.await?
|
||||
}
|
||||
};
|
||||
match authority {
|
||||
match &authority {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && !metadata.bucket_incarnation_id.is_nil() =>
|
||||
{
|
||||
@@ -1489,11 +1529,11 @@ pub(crate) async fn get_delete_config_from_disk_in(
|
||||
}
|
||||
}
|
||||
let object_lock_config_updated_at = metadata.object_lock_config_updated_at;
|
||||
Ok((
|
||||
object_lock_config_state_from_authoritative_metadata(&metadata)?,
|
||||
metadata.bucket_incarnation_id,
|
||||
object_lock_config_updated_at,
|
||||
))
|
||||
let state = object_lock_config_state_from_authoritative_metadata(metadata)?;
|
||||
let incarnation_id = metadata.bucket_incarnation_id;
|
||||
let quota_res = include_quota
|
||||
.then(|| quota_config_and_incarnation_from_authority(bucket, authority).map_err(|err| err.to_string()));
|
||||
Ok((state, incarnation_id, object_lock_config_updated_at, quota_res))
|
||||
}
|
||||
BucketMetadataAuthority::Authoritative(_) => {
|
||||
Err(Error::other(format!("bucket incarnation metadata is not authoritative: {bucket}")))
|
||||
@@ -1564,6 +1604,23 @@ pub(crate) async fn get_quota_config_and_incarnation_from_disk_for_options_in(
|
||||
return Err(Error::other("quota snapshot metadata transaction fence was released"));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
let has_injected_error = sys.object_lock_disk_read_errors.read().await.contains(bucket);
|
||||
#[cfg(not(test))]
|
||||
let has_injected_error = false;
|
||||
if !ctx.lock_manager().is_disabled()
|
||||
&& !has_injected_error
|
||||
&& let Some(fresh_quota) = snapshot.take_fresh_put_quota()
|
||||
{
|
||||
if snapshot_guard.is_lock_lost()
|
||||
|| snapshot_guard.is_released()
|
||||
|| !snapshot.is_valid_for_destructive_put(store.id, bucket, opts.expected_bucket_incarnation_id.unwrap_or_default())
|
||||
{
|
||||
return Err(Error::other("quota snapshot fence or bucket incarnation changed during the disk read"));
|
||||
}
|
||||
return fresh_quota.map_err(Error::other);
|
||||
}
|
||||
|
||||
// The snapshot carries the caller's lifecycle fence, excluding deletion.
|
||||
// Its transaction guard excludes config writes, so a same-name object
|
||||
// owner need not recursively acquire the bucket namespace here.
|
||||
@@ -1750,9 +1807,22 @@ struct MetadataPublishGuard {
|
||||
_guard: tokio::sync::OwnedMutexGuard<MetadataPublishLockState>,
|
||||
}
|
||||
|
||||
static LOCAL_META_MUTATION_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
|
||||
|
||||
pub(crate) fn bump_local_meta_mutation_gen() {
|
||||
LOCAL_META_MUTATION_GEN.fetch_add(1, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
|
||||
pub(crate) fn current_local_meta_mutation_gen() -> u64 {
|
||||
LOCAL_META_MUTATION_GEN.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
|
||||
type LocalAuthorityCache = Arc<StdMutex<HashMap<String, (u64, Arc<BucketMetadata>)>>>;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BucketMetadataSys {
|
||||
metadata_map: Arc<RwLock<HashMap<String, Arc<BucketMetadata>>>>,
|
||||
local_authority_cache: LocalAuthorityCache,
|
||||
/// Serializes metadata-map commits and their derived cache updates for one
|
||||
/// bucket. Namespace locks, when present, are acquired before this lock.
|
||||
metadata_publish_locks: Arc<MetadataPublishLockRegistry>,
|
||||
@@ -1796,6 +1866,7 @@ impl BucketMetadataSys {
|
||||
pub fn new(api: Arc<ECStore>) -> Self {
|
||||
Self {
|
||||
metadata_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
local_authority_cache: Arc::new(StdMutex::new(HashMap::new())),
|
||||
metadata_publish_locks: Arc::new(MetadataPublishLockRegistry {
|
||||
locks: StdMutex::new(HashMap::new()),
|
||||
}),
|
||||
@@ -1830,6 +1901,15 @@ impl BucketMetadataSys {
|
||||
self.api.upgrade()
|
||||
}
|
||||
|
||||
/// Whether the backing `ECStore` has been dropped.
|
||||
///
|
||||
/// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
|
||||
/// `ECStore::drop` so a shared bootstrap context does not retain a
|
||||
/// `BucketMetadataSys` whose weak handle is dangling.
|
||||
pub(crate) fn is_store_stale(&self) -> bool {
|
||||
self.api.upgrade().is_none()
|
||||
}
|
||||
|
||||
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
|
||||
Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
|
||||
}
|
||||
@@ -2160,6 +2240,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
pub async fn set(&self, bucket: String, bm: Arc<BucketMetadata>) {
|
||||
bump_local_meta_mutation_gen();
|
||||
if !is_meta_bucketname(&bucket) {
|
||||
let publish_lock = self.metadata_publish_lock(&bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
@@ -2178,6 +2259,7 @@ impl BucketMetadataSys {
|
||||
///
|
||||
/// Returns `true` if an entry was present. Reserved meta buckets are ignored.
|
||||
pub async fn remove(&self, bucket: &str) -> bool {
|
||||
bump_local_meta_mutation_gen();
|
||||
if is_meta_bucketname(bucket) {
|
||||
return false;
|
||||
}
|
||||
@@ -2198,6 +2280,7 @@ impl BucketMetadataSys {
|
||||
}
|
||||
|
||||
async fn _reset(&mut self) {
|
||||
bump_local_meta_mutation_gen();
|
||||
let mut map = self.metadata_map.write().await;
|
||||
map.clear();
|
||||
drop(map);
|
||||
@@ -3001,30 +3084,77 @@ impl BucketMetadataSys {
|
||||
return Err(Error::other(format!("injected Object Lock metadata disk read failure: {bucket}")));
|
||||
}
|
||||
|
||||
match await_bucket_namespace_operation(Some(guard), bucket, "bucket metadata snapshot existence check", async {
|
||||
self.object_store()
|
||||
.get_bucket_info_from_sets_at_read_quorum(bucket, &crate::storage_api_contracts::bucket::BucketOptions::default())
|
||||
.await
|
||||
})
|
||||
.await
|
||||
{
|
||||
let store = self.object_store();
|
||||
let can_use_local_authority_cache = !store.ctx.lock_manager().is_disabled() && store.all_disks_local_and_online();
|
||||
let gen_before = current_local_meta_mutation_gen();
|
||||
let cached_authority = can_use_local_authority_cache
|
||||
.then(|| {
|
||||
self.local_authority_cache
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(bucket)
|
||||
.filter(|(cached_gen, _)| *cached_gen == gen_before)
|
||||
.map(|(_, bm)| Arc::clone(bm))
|
||||
})
|
||||
.flatten();
|
||||
if let Some(cached) = cached_authority {
|
||||
await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
std::future::ready(Ok::<_, Error>(())),
|
||||
)
|
||||
.await?;
|
||||
if current_local_meta_mutation_gen() == gen_before && store.all_disks_local_and_online() {
|
||||
return Ok(BucketMetadataAuthority::Authoritative(cached));
|
||||
}
|
||||
}
|
||||
|
||||
let (bucket_info_res, metadata_res) = tokio::join!(
|
||||
Box::pin(await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata snapshot existence check",
|
||||
async {
|
||||
store
|
||||
.get_bucket_info_from_sets_at_read_quorum(
|
||||
bucket,
|
||||
&crate::storage_api_contracts::bucket::BucketOptions::default(),
|
||||
)
|
||||
.await
|
||||
},
|
||||
)),
|
||||
Box::pin(await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(Arc::clone(&store), bucket, true),
|
||||
)),
|
||||
);
|
||||
match bucket_info_res {
|
||||
Ok(_) => {}
|
||||
Err(Error::VolumeNotFound) => return Ok(BucketMetadataAuthority::MissingBucket),
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
let (metadata, persisted) = await_bucket_namespace_operation(
|
||||
Some(guard),
|
||||
bucket,
|
||||
"bucket metadata authoritative snapshot",
|
||||
load_bucket_metadata_parse_with_presence(self.object_store(), bucket, true),
|
||||
)
|
||||
.await?;
|
||||
let (metadata, persisted) = metadata_res?;
|
||||
if persisted && metadata.lock_enabled && !metadata.bucket_creation_committed {
|
||||
return Err(Error::ErasureWriteQuorum);
|
||||
}
|
||||
if persisted {
|
||||
Ok(BucketMetadataAuthority::Authoritative(Arc::new(metadata)))
|
||||
let metadata = Arc::new(metadata);
|
||||
if can_use_local_authority_cache
|
||||
&& metadata.bucket_incarnation_sidecar
|
||||
&& !metadata.bucket_incarnation_id.is_nil()
|
||||
&& current_local_meta_mutation_gen() == gen_before
|
||||
&& store.all_disks_local_and_online()
|
||||
{
|
||||
self.local_authority_cache
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(bucket.to_string(), (gen_before, Arc::clone(&metadata)));
|
||||
}
|
||||
Ok(BucketMetadataAuthority::Authoritative(metadata))
|
||||
} else {
|
||||
Ok(BucketMetadataAuthority::Fabricated)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,682 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use crate::disk::background_memory::BackgroundBuffer;
|
||||
use std::io::{self, IoSlice};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::task::{Context, Poll, Waker, ready};
|
||||
use tokio::io::{AsyncRead, AsyncWrite, DuplexStream, ReadBuf};
|
||||
|
||||
pub struct BackgroundPipeReader {
|
||||
inner: PipeEnd,
|
||||
}
|
||||
|
||||
pub struct BackgroundPipeWriter {
|
||||
inner: PipeEnd,
|
||||
}
|
||||
|
||||
impl BackgroundPipeWriter {
|
||||
/// Capture only the payload owner, without retaining node or physical units.
|
||||
pub fn memory_budget(&self) -> Option<Arc<crate::disk::BackgroundMemoryBudget>> {
|
||||
match &self.inner {
|
||||
PipeEnd::Foreground(_) => None,
|
||||
PipeEnd::Background(state) => state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.buffer
|
||||
.memory_budget(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enum PipeEnd {
|
||||
Foreground(DuplexStream),
|
||||
Background(Arc<Mutex<PipeState>>),
|
||||
}
|
||||
|
||||
// The pipe lock may reserve memory. The memory budget never locks a pipe.
|
||||
struct PipeState {
|
||||
buffer: BackgroundBuffer,
|
||||
read_offset: usize,
|
||||
readable: usize,
|
||||
capacity: usize,
|
||||
reader_closed: bool,
|
||||
writer_closed: bool,
|
||||
admission_failure: Option<io::ErrorKind>,
|
||||
read_waker: Option<Waker>,
|
||||
write_waker: Option<Waker>,
|
||||
}
|
||||
|
||||
/// Capture only memory ownership while the request is admitted. Payload storage
|
||||
/// is reserved and initialized once, before the first nonempty background write.
|
||||
pub fn background_pipe(capacity: usize) -> (BackgroundPipeReader, BackgroundPipeWriter) {
|
||||
let capacity = capacity.max(1);
|
||||
let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
|
||||
match memory {
|
||||
Some(memory) => {
|
||||
let state = Arc::new(Mutex::new(PipeState {
|
||||
buffer: BackgroundBuffer::new(Some(memory)),
|
||||
read_offset: 0,
|
||||
readable: 0,
|
||||
capacity,
|
||||
reader_closed: false,
|
||||
writer_closed: false,
|
||||
admission_failure: None,
|
||||
read_waker: None,
|
||||
write_waker: None,
|
||||
}));
|
||||
(
|
||||
BackgroundPipeReader {
|
||||
inner: PipeEnd::Background(state.clone()),
|
||||
},
|
||||
BackgroundPipeWriter {
|
||||
inner: PipeEnd::Background(state),
|
||||
},
|
||||
)
|
||||
}
|
||||
None => {
|
||||
let (reader, writer) = tokio::io::duplex(capacity);
|
||||
(
|
||||
BackgroundPipeReader {
|
||||
inner: PipeEnd::Foreground(reader),
|
||||
},
|
||||
BackgroundPipeWriter {
|
||||
inner: PipeEnd::Foreground(writer),
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_ring(offset: usize, count: usize, capacity: usize) -> usize {
|
||||
let to_end = capacity - offset;
|
||||
if count < to_end { offset + count } else { count - to_end }
|
||||
}
|
||||
|
||||
fn poll_background_read(state: &Mutex<PipeState>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
if output.remaining() == 0 {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.readable == 0 {
|
||||
if let Some(kind) = state.admission_failure.take() {
|
||||
drop(state);
|
||||
return Poll::Ready(Err(io::Error::new(kind, "background pipe storage admission failed")));
|
||||
}
|
||||
if state.writer_closed {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
let replaced = state.read_waker.replace(cx.waker().clone());
|
||||
drop(state);
|
||||
drop(replaced);
|
||||
return Poll::Pending;
|
||||
}
|
||||
let length = state.readable.min(output.remaining());
|
||||
let first = length.min(state.capacity - state.read_offset);
|
||||
output.put_slice(&state.buffer.as_slice()[state.read_offset..state.read_offset + first]);
|
||||
output.put_slice(&state.buffer.as_slice()[..length - first]);
|
||||
state.read_offset = advance_ring(state.read_offset, length, state.capacity);
|
||||
state.readable -= length;
|
||||
let wake = state.write_waker.take();
|
||||
drop(state);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_background_write(state: &Mutex<PipeState>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
|
||||
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.reader_closed || state.writer_closed {
|
||||
return Poll::Ready(Err(io::ErrorKind::BrokenPipe.into()));
|
||||
}
|
||||
if input.iter().all(|part| part.is_empty()) {
|
||||
return Poll::Ready(Ok(0));
|
||||
}
|
||||
let available = state.capacity - state.readable;
|
||||
if available == 0 {
|
||||
let replaced = state.write_waker.replace(cx.waker().clone());
|
||||
drop(state);
|
||||
drop(replaced);
|
||||
return Poll::Pending;
|
||||
}
|
||||
let capacity = state.capacity;
|
||||
if let Err(error) = state.buffer.ensure_len(capacity) {
|
||||
// Legacy walk completion may suppress ordinary producer errors. A new
|
||||
// pipe admission failure must also reach the reader instead of EOF.
|
||||
state.admission_failure = Some(error.kind());
|
||||
state.writer_closed = true;
|
||||
let wake = state.read_waker.take();
|
||||
drop(state);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
return Poll::Ready(Err(error));
|
||||
}
|
||||
let mut offset = advance_ring(state.read_offset, state.readable, capacity);
|
||||
let mut written = 0;
|
||||
for part in input {
|
||||
let length = part.len().min(available - written);
|
||||
let first = length.min(capacity - offset);
|
||||
state.buffer.as_mut_slice()[offset..offset + first].copy_from_slice(&part[..first]);
|
||||
state.buffer.as_mut_slice()[..length - first].copy_from_slice(&part[first..length]);
|
||||
offset = advance_ring(offset, length, capacity);
|
||||
written += length;
|
||||
if written == available {
|
||||
break;
|
||||
}
|
||||
}
|
||||
state.readable += written;
|
||||
let wake = state.read_waker.take();
|
||||
drop(state);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
Poll::Ready(Ok(written))
|
||||
}
|
||||
|
||||
impl AsyncRead for BackgroundPipeReader {
|
||||
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
match &mut self.get_mut().inner {
|
||||
PipeEnd::Foreground(reader) => Pin::new(reader).poll_read(cx, output),
|
||||
PipeEnd::Background(state) => {
|
||||
let coop = ready!(tokio::task::coop::poll_proceed(cx));
|
||||
let result = poll_background_read(state, cx, output);
|
||||
if result.is_ready() {
|
||||
coop.made_progress();
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for BackgroundPipeWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[u8]) -> Poll<io::Result<usize>> {
|
||||
match &mut self.get_mut().inner {
|
||||
PipeEnd::Foreground(writer) => Pin::new(writer).poll_write(cx, input),
|
||||
PipeEnd::Background(state) => {
|
||||
let coop = ready!(tokio::task::coop::poll_proceed(cx));
|
||||
let result = poll_background_write(state, cx, &[IoSlice::new(input)]);
|
||||
if result.is_ready() {
|
||||
coop.made_progress();
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_write_vectored(self: Pin<&mut Self>, cx: &mut Context<'_>, input: &[IoSlice<'_>]) -> Poll<io::Result<usize>> {
|
||||
match &mut self.get_mut().inner {
|
||||
PipeEnd::Foreground(writer) => Pin::new(writer).poll_write_vectored(cx, input),
|
||||
PipeEnd::Background(state) => {
|
||||
let coop = ready!(tokio::task::coop::poll_proceed(cx));
|
||||
let result = poll_background_write(state, cx, input);
|
||||
if result.is_ready() {
|
||||
coop.made_progress();
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn is_write_vectored(&self) -> bool {
|
||||
match &self.inner {
|
||||
PipeEnd::Foreground(writer) => writer.is_write_vectored(),
|
||||
PipeEnd::Background(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
match &mut self.get_mut().inner {
|
||||
PipeEnd::Foreground(writer) => Pin::new(writer).poll_flush(cx),
|
||||
PipeEnd::Background(_) => Poll::Ready(Ok(())),
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
match &mut self.get_mut().inner {
|
||||
PipeEnd::Foreground(writer) => Pin::new(writer).poll_shutdown(cx),
|
||||
PipeEnd::Background(state) => {
|
||||
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.writer_closed = true;
|
||||
let wake = state.read_waker.take();
|
||||
drop(state);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BackgroundPipeReader {
|
||||
fn drop(&mut self) {
|
||||
if let PipeEnd::Background(state) = &self.inner {
|
||||
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.reader_closed = true;
|
||||
let wake = state.write_waker.take();
|
||||
let cancelled = state.read_waker.take();
|
||||
drop(state);
|
||||
drop(cancelled);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for BackgroundPipeWriter {
|
||||
fn drop(&mut self) {
|
||||
if let PipeEnd::Background(state) = &self.inner {
|
||||
let mut state = state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.writer_closed = true;
|
||||
let wake = state.read_waker.take();
|
||||
let cancelled = state.write_waker.take();
|
||||
drop(state);
|
||||
drop(cancelled);
|
||||
if let Some(wake) = wake {
|
||||
wake.wake();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::disk::BackgroundMemoryBudget;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
pub(super) fn budget(capacity: usize) -> Arc<BackgroundMemoryBudget> {
|
||||
let budget = BackgroundMemoryBudget::new();
|
||||
budget.set_sampled_limit(u64::try_from(capacity).expect("test capacity fits u64"), Duration::from_secs(30));
|
||||
budget
|
||||
}
|
||||
|
||||
pub(super) fn local_pipe(
|
||||
memory: &Arc<BackgroundMemoryBudget>,
|
||||
capacity: usize,
|
||||
) -> (BackgroundPipeReader, BackgroundPipeWriter) {
|
||||
let state = Arc::new(Mutex::new(PipeState {
|
||||
buffer: BackgroundBuffer::new(Some(memory.clone())),
|
||||
read_offset: 0,
|
||||
readable: 0,
|
||||
capacity,
|
||||
reader_closed: false,
|
||||
writer_closed: false,
|
||||
admission_failure: None,
|
||||
read_waker: None,
|
||||
write_waker: None,
|
||||
}));
|
||||
(
|
||||
BackgroundPipeReader {
|
||||
inner: PipeEnd::Background(state.clone()),
|
||||
},
|
||||
BackgroundPipeWriter {
|
||||
inner: PipeEnd::Background(state),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
pub(super) fn write_now(writer: &mut (impl AsyncWrite + Unpin), bytes: &[u8]) -> Poll<io::Result<usize>> {
|
||||
Pin::new(writer).poll_write(&mut Context::from_waker(Waker::noop()), bytes)
|
||||
}
|
||||
|
||||
pub(super) fn read_now(reader: &mut (impl AsyncRead + Unpin), bytes: &mut [u8]) -> Poll<io::Result<usize>> {
|
||||
let mut output = ReadBuf::new(bytes);
|
||||
match Pin::new(reader).poll_read(&mut Context::from_waker(Waker::noop()), &mut output) {
|
||||
Poll::Ready(Ok(())) => Poll::Ready(Ok(output.filled().len())),
|
||||
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn fixed_ring_allocates_lazily_once_and_keeps_last_owner_charged() {
|
||||
for capacity in [1, 7, 64 * 1024, 1024 * 1024] {
|
||||
let memory = budget(capacity);
|
||||
let (mut reader, mut writer) = local_pipe(&memory, capacity);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
assert!(matches!(write_now(&mut writer, &[]), Poll::Ready(Ok(0))));
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
|
||||
assert_eq!(memory.snapshot().used, capacity as u64);
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
let mut output = [0];
|
||||
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
|
||||
assert_eq!(output, *b"x");
|
||||
assert!(matches!(write_now(&mut writer, b"y"), Poll::Ready(Ok(1))));
|
||||
drop(writer);
|
||||
assert_eq!(memory.snapshot().used, capacity as u64, "reader still owns the real ring");
|
||||
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(1))));
|
||||
assert_eq!(output, *b"y");
|
||||
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
|
||||
assert_eq!(memory.snapshot().used, capacity as u64, "draining does not destroy allocation ownership");
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn admission_error_reaches_writer_and_reader_once_without_payload() {
|
||||
for sample in [None, Some((6, Duration::from_secs(30))), Some((7, Duration::ZERO))] {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
if let Some((limit, validity)) = sample {
|
||||
memory.set_sampled_limit(limit, validity);
|
||||
}
|
||||
let (mut reader, mut writer) = local_pipe(&memory, 7);
|
||||
let error = match write_now(&mut writer, b"x") {
|
||||
Poll::Ready(Err(e)) => e,
|
||||
_ => panic!("reject before copying first payload"),
|
||||
};
|
||||
assert!(
|
||||
crate::disk::background_memory::is_memory_pressure(&error),
|
||||
"writer retains the original budget marker"
|
||||
);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
let error = match read_now(&mut reader, &mut [0]) {
|
||||
Poll::Ready(Err(e)) => e,
|
||||
_ => panic!("reader must see a pipe admission failure"),
|
||||
};
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))), "failure is delivered once");
|
||||
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
|
||||
drop((reader, writer));
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allocation_overflow_is_a_reader_failure_without_a_leaked_reservation() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(u64::MAX, Duration::from_secs(30));
|
||||
let (mut reader, mut writer) = local_pipe(&memory, usize::MAX);
|
||||
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
assert!(matches!(read_now(&mut reader,&mut[0]),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::OutOfMemory));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scalar_and_vectored_wrap_match_a_reference_queue() {
|
||||
use std::collections::VecDeque;
|
||||
for capacity in [1, 3, 7, 31] {
|
||||
let memory = budget(capacity);
|
||||
let (mut reader, mut writer) = local_pipe(&memory, capacity);
|
||||
let mut expected = VecDeque::new();
|
||||
let mut next = 0u8;
|
||||
for turn in 0..300 {
|
||||
let input: Vec<_> = (0..(turn % 11 + 1))
|
||||
.map(|_| {
|
||||
let byte = next;
|
||||
next = next.wrapping_add(1);
|
||||
byte
|
||||
})
|
||||
.collect();
|
||||
let available = capacity - expected.len();
|
||||
let written = if turn % 2 == 0 {
|
||||
write_now(&mut writer, &input)
|
||||
} else {
|
||||
let split = input.len() / 2;
|
||||
Pin::new(&mut writer).poll_write_vectored(
|
||||
&mut Context::from_waker(Waker::noop()),
|
||||
&[
|
||||
io::IoSlice::new(&[]),
|
||||
io::IoSlice::new(&input[..split]),
|
||||
io::IoSlice::new(&input[split..]),
|
||||
],
|
||||
)
|
||||
};
|
||||
if available == 0 {
|
||||
assert!(written.is_pending());
|
||||
} else {
|
||||
let accepted = input.len().min(available);
|
||||
assert!(matches!(written,Poll::Ready(Ok(n)) if n==accepted));
|
||||
expected.extend(&input[..accepted]);
|
||||
}
|
||||
let mut output = vec![0; turn % 5 + 1];
|
||||
let count = output.len().min(expected.len());
|
||||
assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
|
||||
for byte in &output[..count] {
|
||||
assert_eq!(Some(*byte), expected.pop_front());
|
||||
}
|
||||
assert_eq!(memory.snapshot().used, capacity as u64);
|
||||
}
|
||||
drop(writer);
|
||||
let mut output = vec![0; capacity];
|
||||
let count = expected.len();
|
||||
assert!(matches!(read_now(&mut reader,&mut output),Poll::Ready(Ok(n)) if n==count));
|
||||
assert_eq!(&output[..count], expected.into_iter().collect::<Vec<_>>().as_slice());
|
||||
assert!(matches!(read_now(&mut reader, &mut output), Poll::Ready(Ok(0))));
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reader_drop_wakes_blocked_writer_and_writer_drop_wakes_empty_reader() {
|
||||
let memory = budget(7);
|
||||
let (reader, mut writer) = local_pipe(&memory, 7);
|
||||
writer.write_all(b"1234567").await.expect("fill the ring");
|
||||
let task = tokio::spawn(async move { writer.write_all(b"blocked").await });
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!task.is_finished());
|
||||
drop(reader);
|
||||
let error = tokio::time::timeout(Duration::from_secs(1), task)
|
||||
.await
|
||||
.expect("reader drop wakes writer")
|
||||
.expect("writer task joins")
|
||||
.expect_err("closed reader is BrokenPipe");
|
||||
assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
let (mut reader, writer) = local_pipe(&memory, 7);
|
||||
let task = tokio::spawn(async move { reader.read(&mut [0]).await });
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!task.is_finished());
|
||||
drop(writer);
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(1), task)
|
||||
.await
|
||||
.expect("writer drop wakes reader")
|
||||
.expect("reader task joins")
|
||||
.expect("clean EOF"),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_wakes_a_full_writer_without_changing_byte_limit() {
|
||||
let memory = budget(7);
|
||||
let (mut reader, mut writer) = local_pipe(&memory, 7);
|
||||
writer.write_all(b"1234567").await.expect("fill initial byte limit");
|
||||
let task = tokio::spawn(async move {
|
||||
writer.write_all(b"x").await.expect("resume after one byte is read");
|
||||
writer
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!task.is_finished());
|
||||
let mut first = [0];
|
||||
reader.read_exact(&mut first).await.expect("read one byte");
|
||||
let writer = tokio::time::timeout(Duration::from_secs(1), task)
|
||||
.await
|
||||
.expect("read wakes writer")
|
||||
.expect("writer joins");
|
||||
assert_eq!(&first, b"1");
|
||||
drop(writer);
|
||||
let mut all = Vec::new();
|
||||
reader.read_to_end(&mut all).await.expect("drain wrapped bytes");
|
||||
assert_eq!(all, b"234567x");
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn shutdown_drains_then_eof_and_cancelled_writes_accept_nothing() {
|
||||
let memory = budget(7);
|
||||
let (mut reader, mut writer) = local_pipe(&memory, 7);
|
||||
writer
|
||||
.write_all(b"1234567")
|
||||
.await
|
||||
.expect("fill before cancelling a blocked write");
|
||||
{
|
||||
let blocked = writer.write(b"lost");
|
||||
tokio::pin!(blocked);
|
||||
assert!(futures::poll!(&mut blocked).is_pending());
|
||||
}
|
||||
writer.shutdown().await.expect("shutdown does not discard buffered bytes");
|
||||
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
|
||||
let mut out = Vec::new();
|
||||
reader.read_to_end(&mut out).await.expect("drain after shutdown");
|
||||
assert_eq!(out, b"1234567");
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 7, "writer still holds the allocation");
|
||||
drop(writer);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_keeps_native_duplex_semantics() {
|
||||
let (reader, mut writer) = background_pipe(7);
|
||||
assert!(matches!(write_now(&mut writer, b"12345678"), Poll::Ready(Ok(7))));
|
||||
assert!(write_now(&mut writer, b"x").is_pending());
|
||||
drop(reader);
|
||||
assert!(matches!(write_now(&mut writer,b"x"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
|
||||
let (mut reader, writer) = background_pipe(7);
|
||||
drop(writer);
|
||||
assert!(matches!(read_now(&mut reader, &mut [0]), Poll::Ready(Ok(0))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancelled_reader_leaves_the_real_ring_charged_until_writer_drop() {
|
||||
let memory = budget(7);
|
||||
let (reader, mut writer) = local_pipe(&memory, 7);
|
||||
assert!(matches!(write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 7, "the surviving writer still owns the ring allocation");
|
||||
assert!(matches!(write_now(&mut writer,b"y"),Poll::Ready(Err(e)) if e.kind()==io::ErrorKind::BrokenPipe));
|
||||
drop(writer);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_shutdown_releases_queued_writer_but_not_the_surviving_reader() {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("build isolated runtime");
|
||||
let memory = budget(7);
|
||||
let (reader, mut writer) = local_pipe(&memory, 7);
|
||||
runtime.block_on(async {
|
||||
writer.write_all(b"x").await.expect("allocate ring before runtime shutdown");
|
||||
tokio::spawn(async move {
|
||||
let _writer = writer;
|
||||
std::future::pending::<()>().await;
|
||||
});
|
||||
tokio::task::yield_now().await;
|
||||
});
|
||||
drop(runtime);
|
||||
assert_eq!(memory.snapshot().used, 7, "reader holds the actual allocation after runtime cancellation");
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ready_operations_keep_tokio_cooperative_yields() {
|
||||
let memory = budget(7);
|
||||
let (_reader, mut writer) = local_pipe(&memory, 7);
|
||||
let observed = Arc::new(AtomicUsize::new(0));
|
||||
let flag = observed.clone();
|
||||
let task = tokio::spawn(async move {
|
||||
flag.store(1, Ordering::SeqCst);
|
||||
});
|
||||
for _ in 0..1000 {
|
||||
assert_eq!(writer.write(&[]).await.expect("empty operation remains successful"), 0);
|
||||
}
|
||||
assert_eq!(
|
||||
observed.load(Ordering::SeqCst),
|
||||
1,
|
||||
"ready pipe operations must not starve other ready tasks"
|
||||
);
|
||||
task.await.expect("observer task joins");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod waker_tests {
|
||||
use super::*;
|
||||
use std::sync::{
|
||||
Weak,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::task::Wake;
|
||||
|
||||
struct LockCheck {
|
||||
state: Weak<Mutex<PipeState>>,
|
||||
wakes: Arc<AtomicUsize>,
|
||||
drops: Arc<AtomicUsize>,
|
||||
}
|
||||
impl LockCheck {
|
||||
fn unlocked(&self) {
|
||||
if let Some(state) = self.state.upgrade() {
|
||||
assert!(state.try_lock().is_ok(), "waker code must run outside the pipe lock");
|
||||
}
|
||||
}
|
||||
}
|
||||
impl Wake for LockCheck {
|
||||
fn wake(self: Arc<Self>) {
|
||||
self.unlocked();
|
||||
self.wakes.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
impl Drop for LockCheck {
|
||||
fn drop(&mut self) {
|
||||
self.unlocked();
|
||||
self.drops.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replaced_and_woken_reader_and_writer_wakers_run_outside_the_lock() {
|
||||
for waiting_reader in [true, false] {
|
||||
let memory = super::tests::budget(7);
|
||||
let (mut reader, mut writer) = super::tests::local_pipe(&memory, 7);
|
||||
let PipeEnd::Background(state) = &reader.inner else { panic!("background fixture") };
|
||||
let weak = Arc::downgrade(state);
|
||||
let wakes = Arc::new(AtomicUsize::new(0));
|
||||
let drops = Arc::new(AtomicUsize::new(0));
|
||||
if !waiting_reader {
|
||||
assert!(matches!(super::tests::write_now(&mut writer, b"1234567"), Poll::Ready(Ok(7))));
|
||||
}
|
||||
for _ in 0..2 {
|
||||
let waker = Waker::from(Arc::new(LockCheck {
|
||||
state: weak.clone(),
|
||||
wakes: wakes.clone(),
|
||||
drops: drops.clone(),
|
||||
}));
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
if waiting_reader {
|
||||
let mut bytes = [0];
|
||||
assert!(
|
||||
Pin::new(&mut reader)
|
||||
.poll_read(&mut cx, &mut ReadBuf::new(&mut bytes))
|
||||
.is_pending()
|
||||
);
|
||||
} else {
|
||||
assert!(Pin::new(&mut writer).poll_write(&mut cx, b"x").is_pending());
|
||||
}
|
||||
drop(waker);
|
||||
}
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 1, "the replaced waker has been dropped");
|
||||
if waiting_reader {
|
||||
assert!(matches!(super::tests::write_now(&mut writer, b"x"), Poll::Ready(Ok(1))));
|
||||
} else {
|
||||
assert!(matches!(super::tests::read_now(&mut reader, &mut [0]), Poll::Ready(Ok(1))));
|
||||
}
|
||||
assert_eq!(wakes.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,586 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use crate::disk::background_memory::{BackgroundMemoryBudget, Reservation};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures::Stream;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
struct ReservedReadBytes {
|
||||
// Free each payload view before releasing its shared allocation receipt.
|
||||
bytes: Bytes,
|
||||
_reservation: Arc<Reservation>,
|
||||
}
|
||||
|
||||
impl AsRef<[u8]> for ReservedReadBytes {
|
||||
fn as_ref(&self) -> &[u8] {
|
||||
&self.bytes
|
||||
}
|
||||
}
|
||||
|
||||
struct ReaderBuffer {
|
||||
// These initialized bytes are spare storage, not unconsumed source data.
|
||||
bytes: BytesMut,
|
||||
reservation: Option<Arc<Reservation>>,
|
||||
budget: Arc<BackgroundMemoryBudget>,
|
||||
capacity: usize,
|
||||
}
|
||||
|
||||
impl ReaderBuffer {
|
||||
fn new(budget: Arc<BackgroundMemoryBudget>, capacity: usize) -> Self {
|
||||
Self {
|
||||
bytes: BytesMut::new(),
|
||||
reservation: None,
|
||||
budget,
|
||||
capacity: capacity.max(1),
|
||||
}
|
||||
}
|
||||
|
||||
fn clear(&mut self) {
|
||||
self.bytes = BytesMut::new();
|
||||
self.reservation = None;
|
||||
}
|
||||
|
||||
fn ensure_readable(&mut self) -> io::Result<()> {
|
||||
if !self.bytes.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if self.reservation.is_some() {
|
||||
if self.bytes.try_reclaim(self.capacity) {
|
||||
self.bytes.resize(self.capacity, 0);
|
||||
return Ok(());
|
||||
}
|
||||
// A live output owns this allocation. Drop our empty view before
|
||||
// its receipt, then reserve independent replacement storage.
|
||||
self.clear();
|
||||
}
|
||||
let reservation = self.budget.reserve(self.capacity)?;
|
||||
let mut bytes = Vec::new();
|
||||
bytes
|
||||
.try_reserve_exact(self.capacity)
|
||||
.map_err(|_| io::ErrorKind::OutOfMemory)?;
|
||||
bytes.resize(self.capacity, 0);
|
||||
// Fresh ordinary Bytes is unique and spans the complete allocation.
|
||||
// Unlike from_owner output, this conversion cannot require a copy.
|
||||
let bytes = Bytes::from(bytes.into_boxed_slice())
|
||||
.try_into_mut()
|
||||
.map_err(|_| io::Error::other("fresh reader storage was not unique"))?;
|
||||
self.bytes = bytes;
|
||||
self.reservation = Some(Arc::new(reservation));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn take_bytes(&mut self, length: usize) -> io::Result<Bytes> {
|
||||
let reservation = self
|
||||
.reservation
|
||||
.as_ref()
|
||||
.cloned()
|
||||
.ok_or_else(|| io::Error::other("reader storage has no memory reservation"))?;
|
||||
Ok(Bytes::from_owner(ReservedReadBytes {
|
||||
bytes: self.bytes.split_to(length).freeze(),
|
||||
_reservation: reservation,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
/// RPC response chunks retain their background memory reservation until the
|
||||
/// last clone or slice is dropped. Foreground readers use the existing stream.
|
||||
pub struct BackgroundReaderStream<R> {
|
||||
inner: ReaderState<R>,
|
||||
}
|
||||
|
||||
enum ReaderState<R> {
|
||||
Foreground(ReaderStream<R>),
|
||||
Background { reader: Option<R>, buffer: ReaderBuffer },
|
||||
}
|
||||
|
||||
impl<R: AsyncRead> BackgroundReaderStream<R> {
|
||||
/// Capture memory ownership inside the admitted scope, before moving the
|
||||
/// response body out of it. This stream retains no node or device capacity.
|
||||
pub fn with_capacity(reader: R, capacity: usize) -> Self {
|
||||
let memory = crate::disk::current_resource_unit().and_then(|resources| resources.memory_budget());
|
||||
let inner = match memory {
|
||||
Some(memory) => ReaderState::Background {
|
||||
reader: Some(reader),
|
||||
buffer: ReaderBuffer::new(memory, capacity),
|
||||
},
|
||||
None => ReaderState::Foreground(ReaderStream::with_capacity(reader, capacity)),
|
||||
};
|
||||
Self { inner }
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: AsyncRead + Unpin> Stream for BackgroundReaderStream<R> {
|
||||
type Item = io::Result<Bytes>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let this = self.get_mut();
|
||||
match &mut this.inner {
|
||||
ReaderState::Foreground(reader) => Pin::new(reader).poll_next(cx),
|
||||
ReaderState::Background { reader, buffer } => {
|
||||
let Some(inner) = reader.as_mut() else {
|
||||
return Poll::Ready(None);
|
||||
};
|
||||
if let Err(error) = buffer.ensure_readable() {
|
||||
*reader = None;
|
||||
return Poll::Ready(Some(Err(error)));
|
||||
}
|
||||
let mut output = ReadBuf::new(&mut buffer.bytes);
|
||||
match std::task::ready!(Pin::new(inner).poll_read(cx, &mut output)) {
|
||||
Ok(()) => {
|
||||
let length = output.filled().len();
|
||||
if length == 0 {
|
||||
buffer.clear();
|
||||
*reader = None;
|
||||
Poll::Ready(None)
|
||||
} else {
|
||||
let bytes = buffer.take_bytes(length);
|
||||
if bytes.is_err() {
|
||||
buffer.clear();
|
||||
*reader = None;
|
||||
}
|
||||
Poll::Ready(Some(bytes))
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
buffer.clear();
|
||||
*reader = None;
|
||||
Poll::Ready(Some(Err(error)))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
use futures::StreamExt;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
struct ObservedReader {
|
||||
polls: Arc<AtomicUsize>,
|
||||
fail: bool,
|
||||
}
|
||||
|
||||
impl AsyncRead for ObservedReader {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, output: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
self.polls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.fail {
|
||||
Poll::Ready(Err(io::Error::from(io::ErrorKind::PermissionDenied)))
|
||||
} else {
|
||||
output.put_slice(b"data");
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_frames_keep_capacity_until_the_last_slice() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(16, Duration::from_secs(30));
|
||||
let (resources, gates) = test_resource_units(2);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(io::Cursor::new(b"data"), 16)
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
gates.iter().all(|gate| gate.snapshot().active == 0),
|
||||
"the stream captures only memory ownership"
|
||||
);
|
||||
assert_eq!(memory.snapshot().used, 0, "construction does not allocate payload storage");
|
||||
let bytes = stream.next().await.expect("first frame").expect("read frame");
|
||||
assert_eq!(bytes.as_ref(), b"data");
|
||||
assert_eq!(memory.snapshot().used, 16, "charge full capacity rather than visible length");
|
||||
let slice = bytes.slice(1..3);
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
drop(bytes);
|
||||
drop(stream);
|
||||
assert_eq!(slice.as_ref(), b"at");
|
||||
assert_eq!(memory.snapshot().used, 16);
|
||||
drop(slice);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_denial_happens_before_reading_and_never_falls_back() {
|
||||
for sample in [None, Some((7, Duration::from_secs(30))), Some((8, Duration::ZERO))] {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
if let Some((limit, validity)) = sample {
|
||||
memory.set_sampled_limit(limit, validity);
|
||||
}
|
||||
let polls = Arc::new(AtomicUsize::new(0));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(
|
||||
ObservedReader {
|
||||
polls: polls.clone(),
|
||||
fail: false,
|
||||
},
|
||||
8,
|
||||
)
|
||||
})
|
||||
.await;
|
||||
let error = stream
|
||||
.next()
|
||||
.await
|
||||
.expect("allocation failure")
|
||||
.expect_err("background allocation denied");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(polls.load(Ordering::SeqCst), 0, "denial cannot consume source bytes");
|
||||
assert!(stream.next().await.is_none());
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_pending_read_keeps_storage_through_shrink_or_cancellation() {
|
||||
for cancelled in [false, true] {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(16, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let (mut sender, reader) = tokio::io::duplex(4);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(reader, 16)
|
||||
})
|
||||
.await;
|
||||
assert!(futures::poll!(stream.next()).is_pending());
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
assert_eq!(memory.snapshot().used, 16);
|
||||
let bytes = if cancelled {
|
||||
None
|
||||
} else {
|
||||
sender.write_all(b"data").await.expect("release pending read");
|
||||
Some(
|
||||
stream
|
||||
.next()
|
||||
.await
|
||||
.expect("pending frame")
|
||||
.expect("existing allocation survives shrink"),
|
||||
)
|
||||
};
|
||||
drop(stream);
|
||||
if let Some(bytes) = bytes {
|
||||
assert_eq!(bytes.as_ref(), b"data");
|
||||
assert_eq!(memory.snapshot().used, 16);
|
||||
drop(bytes);
|
||||
}
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_read_error_and_eof_release_storage() {
|
||||
for fail in [false, true] {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(8, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
let reader: Box<dyn AsyncRead + Unpin> = if fail {
|
||||
Box::new(ObservedReader {
|
||||
polls: Arc::new(AtomicUsize::new(0)),
|
||||
fail: true,
|
||||
})
|
||||
} else {
|
||||
Box::new(tokio::io::empty())
|
||||
};
|
||||
BackgroundReaderStream::with_capacity(reader, 8)
|
||||
})
|
||||
.await;
|
||||
if fail {
|
||||
let error = stream.next().await.expect("read failure").expect_err("preserve reader error");
|
||||
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
|
||||
}
|
||||
assert!(stream.next().await.is_none());
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn retained_frame_blocks_new_storage_until_released() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(8, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
|
||||
})
|
||||
.await;
|
||||
let first = stream.next().await.expect("first frame").expect("fits exact budget");
|
||||
assert_eq!(first.as_ref(), b"firstone");
|
||||
let error = stream
|
||||
.next()
|
||||
.await
|
||||
.expect("next allocation")
|
||||
.expect_err("held frame exhausts budget");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(memory.snapshot().used, 8);
|
||||
drop(stream);
|
||||
drop(first);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn consumed_frames_keep_reusable_capacity_until_eof() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(8, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(io::Cursor::new(b"firstonefollowup"), 8)
|
||||
})
|
||||
.await;
|
||||
for expected in [b"firstone", b"followup"] {
|
||||
let bytes = stream.next().await.expect("next frame").expect("released previous capacity");
|
||||
assert_eq!(bytes.as_ref(), expected);
|
||||
assert_eq!(memory.snapshot().used, 8);
|
||||
drop(bytes);
|
||||
assert_eq!(memory.snapshot().used, 8, "the reusable allocation remains owned by the stream");
|
||||
}
|
||||
assert!(stream.next().await.is_none());
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn foreground_stream_and_unpolled_cancellation_preserve_existing_behavior() {
|
||||
let mut stream = BackgroundReaderStream::with_capacity(io::Cursor::new(b"foreground"), 4);
|
||||
assert!(matches!(stream.inner, ReaderState::Foreground(_)));
|
||||
let mut bytes = Vec::new();
|
||||
while let Some(chunk) = stream.next().await {
|
||||
bytes.extend_from_slice(&chunk.expect("foreground chunk"));
|
||||
}
|
||||
assert_eq!(bytes, b"foreground");
|
||||
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let stream = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(tokio::io::empty(), 8)
|
||||
})
|
||||
.await;
|
||||
drop(stream);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod reuse_tests {
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use futures::StreamExt;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use std::{
|
||||
io,
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
static DATA: [u8; 1024] = [0x37; 1024];
|
||||
struct ShortReader {
|
||||
offset: usize,
|
||||
length: usize,
|
||||
chunk: usize,
|
||||
fail_at: Option<usize>,
|
||||
}
|
||||
impl AsyncRead for ShortReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _: &mut Context<'_>, out: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
if self.fail_at == Some(self.offset) {
|
||||
return Poll::Ready(Err(io::ErrorKind::PermissionDenied.into()));
|
||||
}
|
||||
let n = (self.length - self.offset).min(self.chunk).min(out.remaining());
|
||||
out.put_slice(&DATA[self.offset..self.offset + n]);
|
||||
self.offset += n;
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
fn budget(limit: u64) -> Arc<crate::disk::BackgroundMemoryBudget> {
|
||||
let b = crate::disk::BackgroundMemoryBudget::new();
|
||||
b.set_sampled_limit(limit, Duration::from_secs(30));
|
||||
b
|
||||
}
|
||||
async fn stream<R: AsyncRead>(
|
||||
budget: &Arc<crate::disk::BackgroundMemoryBudget>,
|
||||
reader: R,
|
||||
capacity: usize,
|
||||
) -> BackgroundReaderStream<R> {
|
||||
let (resources, _) = crate::disk::test_resource_units(1);
|
||||
crate::disk::with_optional_resource_unit(Some(resources.with_memory_budget(budget.clone())), async {
|
||||
BackgroundReaderStream::with_capacity(reader, capacity)
|
||||
})
|
||||
.await
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn complete_consumed_frames_reclaim_the_same_storage_after_expiry() {
|
||||
let b = budget(64);
|
||||
let mut s = stream(&b, io::Cursor::new(&DATA[..]), 64).await;
|
||||
let mut pointer = None;
|
||||
for _ in 0..16 {
|
||||
let frame = s.next().await.expect("next full frame").expect("reclaimed backing");
|
||||
assert_eq!(frame.len(), 64);
|
||||
assert_eq!(frame.as_ref(), &DATA[..64]);
|
||||
if let Some(p) = pointer {
|
||||
assert_eq!(frame.as_ptr(), p);
|
||||
} else {
|
||||
pointer = Some(frame.as_ptr());
|
||||
}
|
||||
b.set_sampled_limit(0, Duration::ZERO);
|
||||
drop(frame);
|
||||
assert_eq!(b.snapshot().used, 64);
|
||||
}
|
||||
assert!(s.next().await.is_none());
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn short_live_frames_use_disjoint_tail_then_reserve_separate_storage() {
|
||||
let b = budget(32);
|
||||
let mut s = stream(
|
||||
&b,
|
||||
ShortReader {
|
||||
offset: 0,
|
||||
length: 32,
|
||||
chunk: 4,
|
||||
fail_at: None,
|
||||
},
|
||||
16,
|
||||
)
|
||||
.await;
|
||||
let mut frames: Vec<Bytes> = Vec::new();
|
||||
for i in 0..4 {
|
||||
let frame = s.next().await.expect("short frame").expect("remaining initialized tail");
|
||||
if i != 0 {
|
||||
assert_eq!(frame.as_ptr(), frames[0].as_ptr().wrapping_add(i * 4));
|
||||
}
|
||||
assert_eq!(frame.as_ref(), &DATA[..4]);
|
||||
frames.push(frame);
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
}
|
||||
let next = s.next().await.expect("independent frame").expect("second backing fits");
|
||||
assert_ne!(next.as_ptr(), frames[0].as_ptr());
|
||||
assert_eq!(b.snapshot().used, 32);
|
||||
drop(frames);
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
drop(s);
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
drop(next);
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn final_slice_prevents_reclaim_and_keeps_old_plus_new_charged() {
|
||||
let b = budget(32);
|
||||
let mut s = stream(&b, io::Cursor::new(&DATA[..32]), 16).await;
|
||||
let first = s.next().await.expect("old frame").expect("old backing");
|
||||
let last = first.slice(15..);
|
||||
drop(first);
|
||||
let next = s.next().await.expect("new frame").expect("second backing fits");
|
||||
assert_eq!(last.as_ref(), &DATA[..1]);
|
||||
assert_ne!(next.as_ptr(), last.as_ptr());
|
||||
assert_eq!(b.snapshot().used, 32);
|
||||
drop(s);
|
||||
assert_eq!(b.snapshot().used, 32);
|
||||
drop(next);
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
drop(last);
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn expiry_uses_tail_but_rejects_new_storage_before_consuming_source() {
|
||||
let b = budget(16);
|
||||
let mut s = stream(
|
||||
&b,
|
||||
ShortReader {
|
||||
offset: 0,
|
||||
length: 32,
|
||||
chunk: 4,
|
||||
fail_at: None,
|
||||
},
|
||||
16,
|
||||
)
|
||||
.await;
|
||||
let first = s.next().await.expect("first frame").expect("initial backing");
|
||||
b.set_sampled_limit(0, Duration::ZERO);
|
||||
for _ in 0..3 {
|
||||
drop(s.next().await.expect("tail frame").expect("reuse charged storage"));
|
||||
}
|
||||
let error = s
|
||||
.next()
|
||||
.await
|
||||
.expect("new backing denied")
|
||||
.expect_err("live frame prevents reclaim");
|
||||
assert!(crate::disk::background_memory::is_memory_pressure(&error));
|
||||
assert!(s.next().await.is_none());
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
drop(first);
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn eof_and_error_release_spare_but_keep_live_output_charged() {
|
||||
for fail_at in [None, Some(4)] {
|
||||
let b = budget(16);
|
||||
let mut s = stream(
|
||||
&b,
|
||||
ShortReader {
|
||||
offset: 0,
|
||||
length: 4,
|
||||
chunk: 4,
|
||||
fail_at,
|
||||
},
|
||||
16,
|
||||
)
|
||||
.await;
|
||||
let frame = s.next().await.expect("first short frame").expect("payload");
|
||||
match fail_at {
|
||||
None => assert!(s.next().await.is_none()),
|
||||
Some(_) => assert_eq!(
|
||||
s.next()
|
||||
.await
|
||||
.expect("read error")
|
||||
.expect_err("preserve source failure")
|
||||
.kind(),
|
||||
io::ErrorKind::PermissionDenied
|
||||
),
|
||||
}
|
||||
assert!(s.next().await.is_none());
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
assert_eq!(frame.as_ref(), &DATA[..4]);
|
||||
drop(frame);
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn runtime_shutdown_drops_a_pending_stream_without_refunding_early() {
|
||||
for abort in [false, true] {
|
||||
let b = budget(16);
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("runtime");
|
||||
let (_tx, reader) = tokio::io::duplex(4);
|
||||
let (began, wait) = tokio::sync::oneshot::channel();
|
||||
let memory = b.clone();
|
||||
let handle = rt.spawn(async move {
|
||||
let mut s = stream(&memory, reader, 16).await;
|
||||
assert!(futures::poll!(s.next()).is_pending());
|
||||
assert_eq!(memory.snapshot().used, 16);
|
||||
began.send(()).expect("notify pending allocation");
|
||||
let _ = s.next().await;
|
||||
});
|
||||
rt.block_on(wait).expect("stream pending");
|
||||
assert_eq!(b.snapshot().used, 16);
|
||||
if abort {
|
||||
handle.abort();
|
||||
let _ = rt.block_on(handle);
|
||||
}
|
||||
drop(rt);
|
||||
assert_eq!(b.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,10 +17,10 @@ use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
|
||||
use crate::cluster::rpc::http_auth::{
|
||||
AuthenticatedPeerReplayCapabilities, RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER,
|
||||
RPC_CONTENT_SHA256_HEADER, RPC_REPLAY_CACHE_CAPABILITY_HEADER, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
|
||||
RollingMutationBodyDigest, TIMESTAMP_HEADER, internode_rpc_body_digest_strict,
|
||||
verify_tonic_peer_replay_capabilities_response,
|
||||
RollingMutationBodyDigest, TIMESTAMP_HEADER, gen_tonic_replay_scope_headers_with_admission, internode_rpc_body_digest_strict,
|
||||
rpc_background_admission, verify_tonic_peer_replay_capabilities_response,
|
||||
};
|
||||
use crate::cluster::rpc::{gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
||||
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
|
||||
#[cfg(test)]
|
||||
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
|
||||
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
|
||||
@@ -415,7 +415,16 @@ where
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
) {
|
||||
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
|
||||
match rpc_background_admission(request.headers()).and_then(|background| {
|
||||
gen_tonic_replay_scope_headers_with_admission(
|
||||
audience,
|
||||
request.uri().path(),
|
||||
timestamp,
|
||||
content_sha256,
|
||||
boot_epoch,
|
||||
background,
|
||||
)
|
||||
}) {
|
||||
Ok(headers) => request.headers_mut().extend(headers),
|
||||
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
|
||||
}
|
||||
|
||||
@@ -61,6 +61,9 @@ use uuid::Uuid;
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
|
||||
pub const RPC_BACKGROUND_ADMISSION_HEADER: &str = "x-rustfs-rpc-background-admission";
|
||||
const RPC_BACKGROUND_ADMISSION_V1: &str = "v1";
|
||||
const RPC_BACKGROUND_ADMISSION_DOMAIN: &[u8] = b"rustfs-rpc-background-admission-v1\0";
|
||||
pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
|
||||
pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
|
||||
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
|
||||
@@ -77,6 +80,46 @@ pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_HEADER: &str = "x-rustfs-rpc-replay
|
||||
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-replay-cache-capability-proof";
|
||||
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||
|
||||
tokio::task_local! {
|
||||
static RPC_BACKGROUND_REQUEST: bool;
|
||||
}
|
||||
|
||||
pub(crate) fn current_rpc_background_request() -> bool {
|
||||
RPC_BACKGROUND_REQUEST
|
||||
.try_with(|background| *background)
|
||||
.unwrap_or_else(|_| crate::disk::has_resource_io_scope())
|
||||
}
|
||||
|
||||
/// Preserve the authenticated wire class while polling locally admitted work.
|
||||
/// This scope carries only the class, never node, disk, or memory ownership.
|
||||
pub async fn with_rpc_background_request<F: std::future::Future>(background: bool, operation: F) -> F::Output {
|
||||
RPC_BACKGROUND_REQUEST.scope(background, operation).await
|
||||
}
|
||||
|
||||
/// Parse the request class only; callers must verify the signature before using it.
|
||||
pub fn rpc_background_admission(headers: &HeaderMap) -> std::io::Result<bool> {
|
||||
let mut values = headers.get_all(RPC_BACKGROUND_ADMISSION_HEADER).iter();
|
||||
match (values.next(), values.next()) {
|
||||
(None, None) => Ok(false),
|
||||
(Some(value), None) if value == RPC_BACKGROUND_ADMISSION_V1 => Ok(true),
|
||||
_ => Err(std::io::Error::other("Invalid RPC background admission version")),
|
||||
}
|
||||
}
|
||||
|
||||
fn bind_background_admission(mac: &mut HmacSha256, background: bool) {
|
||||
if background {
|
||||
// Every auth lane binds the class, so stripping the marker or falling
|
||||
// back to an older signature cannot turn admitted work into foreground I/O.
|
||||
mac.update(RPC_BACKGROUND_ADMISSION_DOMAIN);
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_background_admission(headers: &mut HeaderMap, background: bool) {
|
||||
if background {
|
||||
headers.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static(RPC_BACKGROUND_ADMISSION_V1));
|
||||
}
|
||||
}
|
||||
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||
const RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-replay-cache-capability-proof-v1\0";
|
||||
@@ -519,21 +562,23 @@ fn redacted_rpc_path(url: &str) -> String {
|
||||
}
|
||||
|
||||
/// Generate HMAC-SHA256 signature for the given data
|
||||
fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64) -> String {
|
||||
fn generate_signature(secret: &str, url: &str, method: &Method, timestamp: i64, background: bool) -> String {
|
||||
let data = signature_payload(url, method, timestamp);
|
||||
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
|
||||
bind_background_admission(&mut mac, background);
|
||||
mac.update(data.as_bytes());
|
||||
let result = mac.finalize();
|
||||
base64_simd::STANDARD.encode_to_string(result.into_bytes())
|
||||
}
|
||||
|
||||
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str) -> bool {
|
||||
fn verify_signature(secret: &str, url: &str, method: &Method, timestamp: i64, signature: &str, background: bool) -> bool {
|
||||
let Ok(signature) = base64_simd::STANDARD.decode_to_vec(signature) else {
|
||||
return false;
|
||||
};
|
||||
|
||||
let data = signature_payload(url, method, timestamp);
|
||||
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).expect("HMAC can take key of any size");
|
||||
bind_background_admission(&mut mac, background);
|
||||
mac.update(data.as_bytes());
|
||||
mac.verify_slice(&signature).is_ok()
|
||||
}
|
||||
@@ -771,6 +816,7 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct SignatureV2Scope<'a> {
|
||||
background: bool,
|
||||
audience: &'a str,
|
||||
service: &'a str,
|
||||
rpc_method: &'a str,
|
||||
@@ -780,6 +826,7 @@ struct SignatureV2Scope<'a> {
|
||||
}
|
||||
|
||||
fn update_signature_v2(mac: &mut HmacSha256, scope: SignatureV2Scope<'_>) {
|
||||
bind_background_admission(mac, scope.background);
|
||||
for part in [
|
||||
b"rustfs-rpc-auth-v2|".as_slice(),
|
||||
scope.audience.as_bytes(),
|
||||
@@ -818,6 +865,7 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ReplayScope<'a> {
|
||||
background: bool,
|
||||
audience: &'a str,
|
||||
path: &'a str,
|
||||
timestamp: &'a str,
|
||||
@@ -827,6 +875,7 @@ struct ReplayScope<'a> {
|
||||
}
|
||||
|
||||
fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
|
||||
bind_background_admission(mac, scope.background);
|
||||
mac.update(RPC_REPLAY_SCOPE_DOMAIN);
|
||||
for part in [
|
||||
scope.audience.as_bytes(),
|
||||
@@ -970,6 +1019,24 @@ pub fn gen_tonic_replay_scope_headers(
|
||||
timestamp: &str,
|
||||
content_sha256: &str,
|
||||
boot_epoch: Uuid,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
gen_tonic_replay_scope_headers_with_admission(
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
content_sha256,
|
||||
boot_epoch,
|
||||
current_rpc_background_request(),
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn gen_tonic_replay_scope_headers_with_admission(
|
||||
audience: &str,
|
||||
path: &str,
|
||||
timestamp: &str,
|
||||
content_sha256: &str,
|
||||
boot_epoch: Uuid,
|
||||
background: bool,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
|
||||
@@ -983,6 +1050,7 @@ pub fn gen_tonic_replay_scope_headers(
|
||||
let signature = generate_replay_scope_signature(
|
||||
&get_shared_secret()?,
|
||||
ReplayScope {
|
||||
background,
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
@@ -992,6 +1060,7 @@ pub fn gen_tonic_replay_scope_headers(
|
||||
},
|
||||
)?;
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_background_admission(&mut headers, background);
|
||||
headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
|
||||
headers.insert(
|
||||
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
|
||||
@@ -1226,10 +1295,12 @@ pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) -
|
||||
pub fn gen_signature_headers(url: &str, method: &Method) -> std::io::Result<HeaderMap> {
|
||||
let secret = get_shared_secret()?;
|
||||
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
|
||||
let background = current_rpc_background_request();
|
||||
|
||||
let signature = generate_signature(&secret, url, method, timestamp);
|
||||
let signature = generate_signature(&secret, url, method, timestamp, background);
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_background_admission(&mut headers, background);
|
||||
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).expect("Invalid header value"));
|
||||
headers.insert(
|
||||
TIMESTAMP_HEADER,
|
||||
@@ -1264,10 +1335,12 @@ pub fn gen_tonic_signature_headers(
|
||||
let timestamp_header = timestamp.to_string();
|
||||
let body_nonce = (content_sha256 != UNSIGNED_PAYLOAD).then(|| Uuid::new_v4().to_string());
|
||||
let nonce = body_nonce.as_deref().unwrap_or(UNSIGNED_PAYLOAD_NONCE);
|
||||
let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp);
|
||||
let background = current_rpc_background_request();
|
||||
let legacy_signature = generate_signature(&secret, TONIC_RPC_PREFIX, &Method::GET, timestamp, background);
|
||||
let signature_v2 = generate_signature_v2(
|
||||
&secret,
|
||||
SignatureV2Scope {
|
||||
background,
|
||||
audience,
|
||||
service,
|
||||
rpc_method,
|
||||
@@ -1278,6 +1351,7 @@ pub fn gen_tonic_signature_headers(
|
||||
)?;
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_background_admission(&mut headers, background);
|
||||
headers.insert(SIGNATURE_HEADER, header_value(&legacy_signature, SIGNATURE_HEADER)?);
|
||||
headers.insert(TIMESTAMP_HEADER, header_value(×tamp_header, TIMESTAMP_HEADER)?);
|
||||
headers.insert(RPC_AUTH_VERSION_HEADER, HeaderValue::from_static(RPC_AUTH_VERSION_V2));
|
||||
@@ -1501,6 +1575,7 @@ fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &Hea
|
||||
if !verify_replay_scope_signature(
|
||||
&secret,
|
||||
ReplayScope {
|
||||
background: rpc_background_admission(headers)?,
|
||||
audience,
|
||||
path,
|
||||
timestamp,
|
||||
@@ -1625,6 +1700,9 @@ fn verify_tonic_rpc_signature_with_strictness(
|
||||
strict: bool,
|
||||
) -> std::io::Result<()> {
|
||||
if !has_v2_auth_headers(headers) {
|
||||
if rpc_background_admission(headers)? {
|
||||
return Err(std::io::Error::other("RPC background admission requires v2 authentication"));
|
||||
}
|
||||
// RUSTFS_COMPAT_TODO(heal-rpc-auth-v2): accept old peers during rolling upgrades. Remove after the minimum
|
||||
// supported RustFS peer version sends v2 authentication on every internode gRPC request.
|
||||
if strict {
|
||||
@@ -1696,6 +1774,7 @@ fn verify_tonic_rpc_signature_with_strictness(
|
||||
if !verify_signature_v2(
|
||||
&secret,
|
||||
SignatureV2Scope {
|
||||
background: rpc_background_admission(headers)?,
|
||||
audience,
|
||||
service,
|
||||
rpc_method,
|
||||
@@ -1736,7 +1815,7 @@ pub fn verify_rpc_signature(url: &str, method: &Method, headers: &HeaderMap) ->
|
||||
// Verify signature with constant-time HMAC comparison.
|
||||
let secret = get_shared_secret()?;
|
||||
|
||||
if !verify_signature(&secret, url, method, timestamp, signature) {
|
||||
if !verify_signature(&secret, url, method, timestamp, signature, rpc_background_admission(headers)?) {
|
||||
let rpc_path = redacted_rpc_path(url);
|
||||
error!(
|
||||
rpc_path = %rpc_path,
|
||||
@@ -1828,6 +1907,98 @@ mod tests {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_admission_is_bound_in_http_and_both_tonic_signature_lanes() {
|
||||
ensure_test_rpc_secret();
|
||||
let secret = get_shared_secret().expect("RPC secret");
|
||||
let path = "/node_service.NodeService/ReadAll";
|
||||
for background in [false, true] {
|
||||
let (http, tonic) = with_rpc_background_request(background, async {
|
||||
(
|
||||
gen_signature_headers("/rustfs/rpc/read?disk=a", &Method::GET).expect("HTTP signature"),
|
||||
gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "ReadAll", None)
|
||||
.expect("gRPC signature"),
|
||||
)
|
||||
})
|
||||
.await;
|
||||
assert_eq!(rpc_background_admission(&http).expect("HTTP class"), background);
|
||||
assert_eq!(rpc_background_admission(&tonic).expect("gRPC class"), background);
|
||||
verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &http).expect("HTTP authentication");
|
||||
verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &tonic, true).expect("v2 authentication");
|
||||
|
||||
let timestamp = tonic[TIMESTAMP_HEADER].to_str().expect("timestamp");
|
||||
let content = tonic[RPC_CONTENT_SHA256_HEADER].to_str().expect("content digest");
|
||||
assert_eq!(
|
||||
verify_signature_v2(
|
||||
&secret,
|
||||
SignatureV2Scope {
|
||||
background: false,
|
||||
audience: "node-a:9000",
|
||||
service: "node_service.NodeService",
|
||||
rpc_method: "ReadAll",
|
||||
timestamp,
|
||||
nonce: tonic[RPC_NONCE_HEADER].to_str().expect("nonce"),
|
||||
content_sha256: content,
|
||||
},
|
||||
tonic[RPC_SIGNATURE_V2_HEADER].to_str().expect("signature"),
|
||||
),
|
||||
!background,
|
||||
"an old peer cannot authenticate a background request as foreground"
|
||||
);
|
||||
|
||||
let mut replay = tonic.clone();
|
||||
// The channel can run outside the caller's task-local scope.
|
||||
replay.extend(
|
||||
gen_tonic_replay_scope_headers_with_admission(
|
||||
"node-a:9000",
|
||||
path,
|
||||
timestamp,
|
||||
content,
|
||||
tonic_rpc_boot_epoch(),
|
||||
background,
|
||||
)
|
||||
.expect("v3 signature"),
|
||||
);
|
||||
for mutation in ["strip-or-add", "unknown", "duplicate"] {
|
||||
let alter = |headers: &HeaderMap| {
|
||||
let mut changed = headers.clone();
|
||||
match mutation {
|
||||
"strip-or-add" if background => {
|
||||
changed.remove(RPC_BACKGROUND_ADMISSION_HEADER);
|
||||
}
|
||||
"strip-or-add" => insert_background_admission(&mut changed, true),
|
||||
"unknown" => {
|
||||
changed.insert(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v2"));
|
||||
}
|
||||
_ => {
|
||||
insert_background_admission(&mut changed, true);
|
||||
changed.append(RPC_BACKGROUND_ADMISSION_HEADER, HeaderValue::from_static("v1"));
|
||||
}
|
||||
}
|
||||
changed
|
||||
};
|
||||
assert!(verify_rpc_signature("/rustfs/rpc/read?disk=a", &Method::GET, &alter(&http)).is_err());
|
||||
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &alter(&tonic), true).is_err());
|
||||
assert!(verify_tonic_replay_scope_signature("node-a:9000", path, &alter(&replay)).is_err());
|
||||
}
|
||||
verify_tonic_replay_scope_signature("node-a:9000", path, &replay).expect("v3 authentication");
|
||||
assert!(
|
||||
verify_tonic_replay_scope_signature("node-a:9000", path, &replay).is_err(),
|
||||
"replay rejected"
|
||||
);
|
||||
|
||||
if background {
|
||||
let mut downgraded = HeaderMap::new();
|
||||
for name in [SIGNATURE_HEADER, TIMESTAMP_HEADER, RPC_BACKGROUND_ADMISSION_HEADER] {
|
||||
downgraded.insert(name, tonic[name].clone());
|
||||
}
|
||||
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
|
||||
downgraded.remove(RPC_BACKGROUND_ADMISSION_HEADER);
|
||||
assert!(verify_tonic_rpc_signature_with_strictness("node-a:9000", path, &downgraded, false).is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_capability_proof_binds_challenge_and_server_epoch() {
|
||||
let secret = "test-scanner-capability-secret";
|
||||
@@ -1965,8 +2136,8 @@ mod tests {
|
||||
let method = Method::GET;
|
||||
let timestamp = 1640995200; // Fixed timestamp
|
||||
|
||||
let signature1 = generate_signature(secret, url, &method, timestamp);
|
||||
let signature2 = generate_signature(secret, url, &method, timestamp);
|
||||
let signature1 = generate_signature(secret, url, &method, timestamp, false);
|
||||
let signature2 = generate_signature(secret, url, &method, timestamp, false);
|
||||
|
||||
assert_eq!(signature1, signature2, "Same inputs should produce same signature");
|
||||
assert!(!signature1.is_empty(), "Signature should not be empty");
|
||||
@@ -1979,10 +2150,10 @@ mod tests {
|
||||
let method = Method::GET;
|
||||
let timestamp = 1640995200;
|
||||
|
||||
let signature1 = generate_signature(secret, url, &method, timestamp);
|
||||
let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp);
|
||||
let signature3 = generate_signature(secret, url, &Method::POST, timestamp);
|
||||
let signature4 = generate_signature(secret, url, &method, timestamp + 1);
|
||||
let signature1 = generate_signature(secret, url, &method, timestamp, false);
|
||||
let signature2 = generate_signature(secret, "http://different.com/api/test2", &method, timestamp, false);
|
||||
let signature3 = generate_signature(secret, url, &Method::POST, timestamp, false);
|
||||
let signature4 = generate_signature(secret, url, &method, timestamp + 1, false);
|
||||
|
||||
assert_ne!(signature1, signature2, "Different URLs should produce different signatures");
|
||||
assert_ne!(signature1, signature3, "Different methods should produce different signatures");
|
||||
@@ -2088,14 +2259,14 @@ mod tests {
|
||||
let url = "http://example.com/api/test";
|
||||
let method = Method::GET;
|
||||
let timestamp = 1640995200;
|
||||
let signature = generate_signature(secret, url, &method, timestamp);
|
||||
let signature = generate_signature(secret, url, &method, timestamp, false);
|
||||
let mut tampered = base64_simd::STANDARD.decode_to_vec(&signature).unwrap();
|
||||
tampered[0] ^= 1;
|
||||
let tampered_signature = base64_simd::STANDARD.encode_to_string(tampered);
|
||||
|
||||
assert!(verify_signature(secret, url, &method, timestamp, &signature));
|
||||
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature));
|
||||
assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature"));
|
||||
assert!(verify_signature(secret, url, &method, timestamp, &signature, false));
|
||||
assert!(!verify_signature(secret, url, &method, timestamp, &tampered_signature, false));
|
||||
assert!(!verify_signature(secret, url, &method, timestamp, "invalid-signature", false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2112,11 +2283,11 @@ mod tests {
|
||||
);
|
||||
let method = Method::GET;
|
||||
let timestamp = 1_640_995_200;
|
||||
let signature = generate_signature(secret, signed_url, &method, timestamp);
|
||||
let signature = generate_signature(secret, signed_url, &method, timestamp, false);
|
||||
|
||||
assert!(verify_signature(secret, signed_url, &method, timestamp, &signature));
|
||||
assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature));
|
||||
assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature));
|
||||
assert!(verify_signature(secret, signed_url, &method, timestamp, &signature, false));
|
||||
assert!(!verify_signature(secret, downgraded_url, &method, timestamp, &signature, false));
|
||||
assert!(!verify_signature(secret, tampered_body_digest, &method, timestamp, &signature, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2126,7 +2297,7 @@ mod tests {
|
||||
let method = Method::GET;
|
||||
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
|
||||
let secret = get_shared_secret().expect("test RPC secret should resolve");
|
||||
let expected_signature = generate_signature(&secret, url, &method, timestamp);
|
||||
let expected_signature = generate_signature(&secret, url, &method, timestamp, false);
|
||||
let invalid_signature = "invalid-signature";
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
@@ -2164,7 +2335,7 @@ mod tests {
|
||||
// Set expired timestamp (older than SIGNATURE_VALID_DURATION)
|
||||
let expired_timestamp = OffsetDateTime::now_utc().unix_timestamp() - SIGNATURE_VALID_DURATION - 10;
|
||||
let secret = get_shared_secret().expect("test RPC secret should resolve");
|
||||
let signature = generate_signature(&secret, url, &method, expired_timestamp);
|
||||
let signature = generate_signature(&secret, url, &method, expired_timestamp, false);
|
||||
|
||||
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
|
||||
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&expired_timestamp.to_string()).unwrap());
|
||||
@@ -2186,7 +2357,7 @@ mod tests {
|
||||
|
||||
let future_timestamp = OffsetDateTime::now_utc().unix_timestamp() + SIGNATURE_VALID_DURATION + 10;
|
||||
let secret = get_shared_secret().expect("test RPC secret should resolve");
|
||||
let signature = generate_signature(&secret, url, &method, future_timestamp);
|
||||
let signature = generate_signature(&secret, url, &method, future_timestamp, false);
|
||||
|
||||
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
|
||||
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&future_timestamp.to_string()).unwrap());
|
||||
@@ -2300,7 +2471,7 @@ mod tests {
|
||||
// Test timestamp just within valid duration
|
||||
let valid_timestamp = current_time - SIGNATURE_VALID_DURATION + 1;
|
||||
|
||||
let signature = generate_signature(&secret, url, &method, valid_timestamp);
|
||||
let signature = generate_signature(&secret, url, &method, valid_timestamp, false);
|
||||
|
||||
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
|
||||
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&valid_timestamp.to_string()).unwrap());
|
||||
@@ -2311,7 +2482,7 @@ mod tests {
|
||||
// Test timestamp just outside valid duration
|
||||
let mut headers = HeaderMap::new();
|
||||
let invalid_timestamp = current_time - SIGNATURE_VALID_DURATION - 15;
|
||||
let signature = generate_signature(&secret, url, &method, invalid_timestamp);
|
||||
let signature = generate_signature(&secret, url, &method, invalid_timestamp, false);
|
||||
|
||||
headers.insert(SIGNATURE_HEADER, HeaderValue::from_str(&signature).unwrap());
|
||||
headers.insert(TIMESTAMP_HEADER, HeaderValue::from_str(&invalid_timestamp.to_string()).unwrap());
|
||||
|
||||
@@ -33,7 +33,9 @@ use rustfs_config::{
|
||||
DEFAULT_INTERNODE_DATA_TRANSPORT, ENV_RUSTFS_INTERNODE_DATA_TRANSPORT, INTERNODE_DATA_TRANSPORT_TCP,
|
||||
KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS,
|
||||
};
|
||||
use rustfs_rio::{ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter};
|
||||
use rustfs_rio::{
|
||||
ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter, HttpWriterMemoryReservation, HttpWriterMemoryReserver,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
@@ -66,6 +68,19 @@ const PUT_FILE_CAPABILITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const CONTENT_TYPE_JSON: &str = "application/json";
|
||||
const CONTENT_TYPE_MSGPACK: &str = "application/msgpack";
|
||||
|
||||
fn http_writer_memory_reserver() -> Option<HttpWriterMemoryReserver> {
|
||||
crate::disk::current_resource_unit()
|
||||
.and_then(|resources| resources.memory_budget())
|
||||
.map(|budget| {
|
||||
// Payloads can outlive the operation; never retain its node or disk units.
|
||||
Arc::new(move |capacity| {
|
||||
budget
|
||||
.reserve(capacity)
|
||||
.map(|reservation| Box::new(reservation) as HttpWriterMemoryReservation)
|
||||
}) as HttpWriterMemoryReserver
|
||||
})
|
||||
}
|
||||
|
||||
fn unsupported_transport_message(transport: &str) -> String {
|
||||
format!(
|
||||
"invalid {ENV_RUSTFS_INTERNODE_DATA_TRANSPORT}={transport:?}; supported values: {}",
|
||||
@@ -346,13 +361,17 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
}
|
||||
|
||||
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
|
||||
let memory_reserver = http_writer_memory_reserver();
|
||||
let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
|
||||
let auth_scope = server_epoch.map(|server_epoch| (Uuid::new_v4(), server_epoch));
|
||||
let url = build_put_file_stream_url(&request, auth_scope);
|
||||
let endpoint = request.endpoint;
|
||||
let mut headers = json_headers();
|
||||
build_auth_headers(&url, &Method::PUT, &mut headers)?;
|
||||
let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
|
||||
let writer = match memory_reserver {
|
||||
Some(reserve) => HttpWriter::new_with_memory_reserver(url.clone(), Method::PUT, headers, reserve).await?,
|
||||
None => HttpWriter::new(url.clone(), Method::PUT, headers).await?,
|
||||
};
|
||||
match auth_scope {
|
||||
Some((nonce, server_epoch)) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce, endpoint, server_epoch))),
|
||||
None => Ok(Box::new(writer)),
|
||||
@@ -948,6 +967,193 @@ mod tests {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::sync::{Barrier, Notify};
|
||||
|
||||
fn memory_test_write_request(authenticated: bool) -> WriteStreamRequest {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
let endpoint = format!("http://writer-memory-{}.invalid", Uuid::new_v4());
|
||||
put_file_capability_cache_entry(&endpoint).write().cached = Some(if authenticated {
|
||||
PutFileCapabilityState::V1 {
|
||||
server_epoch: Uuid::new_v4(),
|
||||
revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
|
||||
}
|
||||
} else {
|
||||
PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL)
|
||||
});
|
||||
WriteStreamRequest {
|
||||
endpoint,
|
||||
disk: "disk-a".to_owned(),
|
||||
volume: "bucket".to_owned(),
|
||||
path: "part.1".to_owned(),
|
||||
append: false,
|
||||
size: 4,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_write_memory_reserver_outlives_only_the_memory_scope() {
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
|
||||
assert!(http_writer_memory_reserver().is_none(), "foreground must use the existing writer");
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(2 * 1024 * 1024, Duration::from_secs(30));
|
||||
let (resources, gates) = test_resource_units(2);
|
||||
let request = memory_test_write_request(true);
|
||||
let endpoint = request.endpoint.clone();
|
||||
let writer = with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
TcpHttpInternodeDataTransport.open_write(request),
|
||||
)
|
||||
.await;
|
||||
PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
|
||||
let mut writer = writer.expect("admitted transport writer");
|
||||
assert!(
|
||||
gates.iter().all(|gate| gate.snapshot().active == 0),
|
||||
"the callback must not retain disk units"
|
||||
);
|
||||
assert_eq!(memory.snapshot().used, 1024 * 1024);
|
||||
writer.write_all(b"data").await.expect("buffered bytes");
|
||||
writer.flush().await.expect("transfer charged allocation to queue");
|
||||
memory.set_sampled_limit(2 * 1024 * 1024, Duration::ZERO);
|
||||
writer
|
||||
.write_all(&vec![0x5a; 1024 * 1024 - 4])
|
||||
.await
|
||||
.expect("reuse the already charged tail after observation expiry");
|
||||
assert_eq!(memory.snapshot().used, 1024 * 1024, "reuse must not allocate separate storage");
|
||||
let error = writer
|
||||
.write(b"rejected")
|
||||
.await
|
||||
.expect_err("expired observation must reject new storage");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert!(
|
||||
crate::disk::background_memory::is_memory_pressure(&error),
|
||||
"preserve the typed memory-pressure marker"
|
||||
);
|
||||
assert_eq!(memory.snapshot().used, 1024 * 1024, "queued storage remains charged after expiry");
|
||||
drop(writer);
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_write_memory_denial_preserves_the_budget_error() {
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
|
||||
for (authenticated, sample) in [false, true].into_iter().flat_map(|authenticated| {
|
||||
[
|
||||
None,
|
||||
Some((1024 * 1024 - 1, Duration::from_secs(30))),
|
||||
Some((1024 * 1024, Duration::ZERO)),
|
||||
]
|
||||
.into_iter()
|
||||
.map(move |sample| (authenticated, sample))
|
||||
}) {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
if let Some((limit, valid_for)) = sample {
|
||||
memory.set_sampled_limit(limit, valid_for);
|
||||
}
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let request = memory_test_write_request(authenticated);
|
||||
let endpoint = request.endpoint.clone();
|
||||
let result = with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
TcpHttpInternodeDataTransport.open_write(request),
|
||||
)
|
||||
.await;
|
||||
PUT_FILE_CAPABILITY_CACHE.write().remove(&endpoint);
|
||||
let error = match result {
|
||||
Err(Error::Io(error)) => error,
|
||||
Err(error) => panic!("unexpected transport error: {error}"),
|
||||
Ok(_) => panic!("unavailable memory must reject construction"),
|
||||
};
|
||||
assert!(crate::disk::background_memory::is_memory_pressure(&error));
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admitted_remote_reads_bind_receiver_admission_to_the_http_signature() {
|
||||
use crate::disk::with_background_resource_unit;
|
||||
use rustfs_concurrency::adaptive::AdaptiveAdmission;
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("remote admission listener");
|
||||
let address = listener.local_addr().expect("remote admission address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.expect("remote read connection");
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let mut chunk = [0; 1024];
|
||||
let read = socket.read(&mut chunk).await.expect("remote read headers");
|
||||
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete request headers");
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
let request = std::str::from_utf8(&bytes).expect("HTTP request");
|
||||
let mut lines = request.split("\r\n");
|
||||
let target = lines
|
||||
.next()
|
||||
.expect("request line")
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.expect("request target")
|
||||
.to_owned();
|
||||
let mut headers = HeaderMap::new();
|
||||
for line in lines.take_while(|line| !line.is_empty()) {
|
||||
let (name, value) = line.split_once(':').expect("header field");
|
||||
headers.append(
|
||||
http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
|
||||
HeaderValue::from_str(value.trim()).expect("header value"),
|
||||
);
|
||||
}
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 5\r\nconnection: close\r\n\r\nhello")
|
||||
.await
|
||||
.expect("remote read response");
|
||||
(target, headers)
|
||||
});
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let resource = gate.try_acquire().expect("sender device admission");
|
||||
let mut reader = tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
with_background_resource_unit(
|
||||
resource,
|
||||
TcpHttpInternodeDataTransport.open_read(ReadStreamRequest {
|
||||
endpoint: format!("http://{address}"),
|
||||
disk: "disk-a".to_owned(),
|
||||
volume: "bucket".to_owned(),
|
||||
path: "part.1".to_owned(),
|
||||
offset: 0,
|
||||
length: 5,
|
||||
stall_timeout: Some(Duration::from_secs(5)),
|
||||
}),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("bounded remote request")
|
||||
.expect("remote reader");
|
||||
let mut body = Vec::new();
|
||||
reader.read_to_end(&mut body).await.expect("complete remote body");
|
||||
assert_eq!(body, b"hello");
|
||||
let (target, headers) = server.await.expect("remote request capture");
|
||||
assert_eq!(gate.snapshot().active, 0, "remote body does not retain the sender's device");
|
||||
assert_eq!(
|
||||
headers
|
||||
.get("x-rustfs-rpc-background-admission")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("v1"),
|
||||
"resource-controlled remote work must request receiver-side admission"
|
||||
);
|
||||
crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &headers).expect("bound request signature");
|
||||
let mut stripped = headers;
|
||||
stripped.remove("x-rustfs-rpc-background-admission");
|
||||
assert!(
|
||||
crate::cluster::rpc::verify_rpc_signature(&target, &Method::GET, &stripped).is_err(),
|
||||
"removing background admission must invalidate the signature"
|
||||
);
|
||||
}
|
||||
|
||||
async fn wait_for_capability_flight_waiters(entry: &PutFileCapabilityCacheEntry, waiters: usize) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
|
||||
@@ -13,6 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod background_monitor;
|
||||
pub(crate) mod background_pipe;
|
||||
pub(crate) mod background_reader_stream;
|
||||
pub(crate) mod client;
|
||||
pub(crate) mod context_propagation;
|
||||
pub(crate) mod http_auth;
|
||||
@@ -33,16 +35,17 @@ pub use client::{
|
||||
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
|
||||
#[allow(unused_imports)]
|
||||
pub use http_auth::{
|
||||
TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer, check_and_record_signed_rpc_nonce, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
|
||||
set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest,
|
||||
sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
|
||||
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
|
||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
RPC_BACKGROUND_ADMISSION_HEADER, TONIC_RPC_PREFIX, build_auth_headers, build_put_file_auth_trailer,
|
||||
check_and_record_signed_rpc_nonce, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
normalize_tonic_rpc_audience, rpc_background_admission, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest,
|
||||
set_tonic_rolling_canonical_body_digest, set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability,
|
||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -17,6 +17,7 @@ use crate::cluster::rpc::client::{
|
||||
node_service_delete_recovery_client, node_service_time_out_client, node_service_time_out_client_for_class,
|
||||
node_service_time_out_client_no_auth,
|
||||
};
|
||||
use crate::cluster::rpc::http_auth::{current_rpc_background_request, with_rpc_background_request};
|
||||
use crate::cluster::rpc::internode_data_transport::{
|
||||
InternodeDataTransport, NsScannerCapabilityRequest, NsScannerStreamRequest, ReadStreamRequest, WalkDirStreamRequest,
|
||||
WriteStreamRequest,
|
||||
@@ -570,6 +571,7 @@ struct RetryingRemoteReader {
|
||||
reader: Option<FileReader>,
|
||||
transport: Arc<dyn InternodeDataTransport>,
|
||||
request: ReadStreamRequest,
|
||||
background: bool,
|
||||
emitted: usize,
|
||||
retried: bool,
|
||||
initial_read_timeout: Option<Duration>,
|
||||
@@ -590,6 +592,7 @@ impl RetryingRemoteReader {
|
||||
reader: Some(reader),
|
||||
transport,
|
||||
request,
|
||||
background: current_rpc_background_request(),
|
||||
emitted: 0,
|
||||
retried: false,
|
||||
initial_read_timeout,
|
||||
@@ -607,9 +610,10 @@ impl RetryingRemoteReader {
|
||||
let request = resumed_read_request(&self.request, self.emitted)?;
|
||||
let recovery_timeout = self.recovery_timeout;
|
||||
let transport = Arc::clone(&self.transport);
|
||||
self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
|
||||
with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await
|
||||
})));
|
||||
self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
|
||||
self.background,
|
||||
async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_fresh(request)).await },
|
||||
))));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -717,6 +721,7 @@ struct RetryingRemoteChunkReader {
|
||||
reader: Option<rustfs_rio::ChunkReaderBox>,
|
||||
transport: Arc<dyn InternodeDataTransport>,
|
||||
request: ReadStreamRequest,
|
||||
background: bool,
|
||||
emitted: usize,
|
||||
retried: bool,
|
||||
initial_read_timeout: Option<Duration>,
|
||||
@@ -737,6 +742,7 @@ impl RetryingRemoteChunkReader {
|
||||
reader: Some(reader),
|
||||
transport,
|
||||
request,
|
||||
background: current_rpc_background_request(),
|
||||
emitted: 0,
|
||||
retried: false,
|
||||
initial_read_timeout,
|
||||
@@ -754,9 +760,10 @@ impl RetryingRemoteChunkReader {
|
||||
let request = resumed_read_request(&self.request, self.emitted)?;
|
||||
let recovery_timeout = self.recovery_timeout;
|
||||
let transport = Arc::clone(&self.transport);
|
||||
self.resume = Some(AbortOnDropTask::new(tokio::spawn(async move {
|
||||
with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await
|
||||
})));
|
||||
self.resume = Some(AbortOnDropTask::new(tokio::spawn(with_rpc_background_request(
|
||||
self.background,
|
||||
async move { with_remote_read_recovery_timeout(recovery_timeout, transport.open_read_chunks_fresh(request)).await },
|
||||
))));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -7769,6 +7776,98 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_read_retries_preserve_signed_admission_after_sender_scope_ends() {
|
||||
use crate::cluster::rpc::{rpc_background_admission, verify_rpc_signature};
|
||||
use crate::disk::with_background_resource_unit;
|
||||
use rustfs_concurrency::adaptive::AdaptiveAdmission;
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
for background in [false, true] {
|
||||
for chunked in [false, true] {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.expect("retry listener");
|
||||
let address = listener.local_addr().expect("retry address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut socket, _) = listener.accept().await.expect("fresh connection");
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let mut chunk = [0; 1024];
|
||||
let read = socket.read(&mut chunk).await.expect("retry request");
|
||||
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete headers");
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
let request = std::str::from_utf8(&bytes).expect("HTTP request");
|
||||
let mut lines = request.split("\r\n");
|
||||
let target = lines
|
||||
.next()
|
||||
.expect("request line")
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.expect("target")
|
||||
.to_owned();
|
||||
let mut headers = http::HeaderMap::new();
|
||||
for line in lines.take_while(|line| !line.is_empty()) {
|
||||
let (name, value) = line.split_once(':').expect("header");
|
||||
headers.append(
|
||||
http::header::HeaderName::from_bytes(name.as_bytes()).expect("header name"),
|
||||
http::HeaderValue::from_str(value.trim()).expect("header value"),
|
||||
);
|
||||
}
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 200 OK\r\ncontent-length: 6\r\nconnection: close\r\n\r\n456789")
|
||||
.await
|
||||
.expect("retry response");
|
||||
(target, headers)
|
||||
});
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let request = ReadStreamRequest {
|
||||
endpoint: format!("http://{address}"),
|
||||
..resume_request(10)
|
||||
};
|
||||
let create = async {
|
||||
let transport = Arc::new(TcpHttpInternodeDataTransport);
|
||||
if chunked {
|
||||
Box::new(RetryingRemoteChunkReader::new_with_timeouts(
|
||||
resume_step_chunk_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
|
||||
transport,
|
||||
request,
|
||||
None,
|
||||
None,
|
||||
)) as FileReader
|
||||
} else {
|
||||
Box::new(RetryingRemoteReader::new_with_timeouts(
|
||||
resume_step_reader(ResumeReadStep::PartialThenReset(b"0123".to_vec())),
|
||||
transport,
|
||||
request,
|
||||
None,
|
||||
None,
|
||||
)) as FileReader
|
||||
}
|
||||
};
|
||||
let mut reader = if background {
|
||||
with_background_resource_unit(gate.try_acquire().expect("sender admission"), create).await
|
||||
} else {
|
||||
create.await
|
||||
};
|
||||
assert_eq!(gate.snapshot().active, 0, "retry reader cannot retain sender disk capacity");
|
||||
let mut output = Vec::new();
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
with_rpc_background_request(!background, reader.read_to_end(&mut output)),
|
||||
)
|
||||
.await
|
||||
.expect("bounded retry")
|
||||
.expect("resumed body");
|
||||
assert_eq!(output, b"0123456789", "chunked={chunked}, background={background}");
|
||||
let (target, headers) = server.await.expect("captured retry");
|
||||
assert!(target.contains("offset=11") && target.contains("length=6"), "{target}");
|
||||
assert_eq!(rpc_background_admission(&headers).expect("admission marker"), background);
|
||||
verify_rpc_signature(&target, &http::Method::GET, &headers).expect("signed retry class");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_reader_resumes_from_emitted_bytes_without_duplicates() {
|
||||
let transport = Arc::new(ResumeTransport::with_read_steps(vec![ResumeReadStep::Data(b"456789".to_vec())]));
|
||||
|
||||
@@ -391,7 +391,7 @@ pub(super) async fn fixture_inline_pool_meta_replicas(payload: &[u8], modified:
|
||||
writer.shutdown().await.expect("fixture bitrot writer should finish");
|
||||
let mut info = base.clone();
|
||||
info.erasure.index = *coding_index;
|
||||
info.data = Some(Bytes::from(writer.into_inline_data().expect("fixture writer should retain inline bytes")));
|
||||
info.data = Some(writer.into_inline_data().expect("fixture writer should retain inline bytes"));
|
||||
let mut meta = FileMeta::new();
|
||||
meta.add_version(info).expect("test inline FileInfo should serialize");
|
||||
raw.push(Bytes::from(meta.marshal_msg().expect("test xl.meta should serialize")));
|
||||
|
||||
@@ -84,7 +84,7 @@ pub(crate) enum CrashPoint {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use armed::{arm, disarm, should_crash_at};
|
||||
pub(crate) use armed::{arm, disarm, is_armed, should_crash_at};
|
||||
|
||||
/// Production no-op: the compiler inlines this to `false`, so armed crash points
|
||||
/// exist only in `#[cfg(test)]` builds and never add a branch to real commits.
|
||||
@@ -94,6 +94,12 @@ pub(crate) fn should_crash_at(_point: CrashPoint, _key: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
#[inline(always)]
|
||||
pub(crate) fn is_armed(_point: CrashPoint, _key: &str) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod armed {
|
||||
use super::CrashPoint;
|
||||
@@ -132,6 +138,10 @@ mod armed {
|
||||
guard().retain(|(p, k)| !(*p == point && k == key));
|
||||
}
|
||||
|
||||
pub(crate) fn is_armed(point: CrashPoint, key: &str) -> bool {
|
||||
guard().iter().any(|(p, k)| *p == point && k == key)
|
||||
}
|
||||
|
||||
/// Returns `true` (consuming the arm) iff a crash is armed for exactly this
|
||||
/// `point` and `key`.
|
||||
pub(crate) fn should_crash_at(point: CrashPoint, key: &str) -> bool {
|
||||
|
||||
@@ -624,58 +624,58 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleanup_dispatch_retains_resource_units_without_inheriting_disk_policy() {
|
||||
use crate::disk::io_admission::{current_disk_io_context, with_background_disk_io, with_background_resource_unit};
|
||||
use rustfs_concurrency::adaptive::AdaptiveAdmission;
|
||||
use std::num::NonZeroU16;
|
||||
use crate::disk::io_admission::{current_disk_io_context, test_resource_units, with_background_disk_io};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
for mode in 0..4 {
|
||||
let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"));
|
||||
let worker_runtime = runtime.clone();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let work = async move {
|
||||
started_tx
|
||||
.send((current_resource_unit().is_some(), current_disk_io_context().is_none()))
|
||||
.expect("report worker context");
|
||||
release_rx.await.expect("release owned worker");
|
||||
Ok::<_, super::super::error::DiskError>(())
|
||||
};
|
||||
let task = tokio::spawn(with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
with_background_resource_unit(gate.acquire().await, async move {
|
||||
match mode {
|
||||
0 => worker_runtime
|
||||
.disk_operation(work, DiskAdmission::Request)
|
||||
.await
|
||||
.expect("disk operation"),
|
||||
1 => worker_runtime.gc_operation(move |_| work).await.expect("GC operation"),
|
||||
2 => {
|
||||
let reservation = worker_runtime.reserve().expect("cleanup reservation");
|
||||
run(Some(reservation), async { work.await.expect("cleanup operation") }).await;
|
||||
for physical_count in [1, 2] {
|
||||
for mode in 0..4 {
|
||||
let runtime = Arc::new(CleanupRuntime::new(config(1, 1, 1)).expect("cleanup runtime"));
|
||||
let worker_runtime = runtime.clone();
|
||||
let (unit, gates) = test_resource_units(physical_count);
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let work = async move {
|
||||
started_tx
|
||||
.send((current_resource_unit().is_some(), current_disk_io_context().is_none()))
|
||||
.expect("report worker context");
|
||||
release_rx.await.expect("release owned worker");
|
||||
Ok::<_, super::super::error::DiskError>(())
|
||||
};
|
||||
let task = tokio::spawn(with_background_disk_io(
|
||||
CancellationToken::new(),
|
||||
with_optional_resource_unit(Some(unit), async move {
|
||||
match mode {
|
||||
0 => worker_runtime
|
||||
.disk_operation(work, DiskAdmission::Request)
|
||||
.await
|
||||
.expect("disk operation"),
|
||||
1 => worker_runtime.gc_operation(move |_| work).await.expect("GC operation"),
|
||||
2 => {
|
||||
let reservation = worker_runtime.reserve().expect("cleanup reservation");
|
||||
run(Some(reservation), async { work.await.expect("cleanup operation") }).await;
|
||||
}
|
||||
_ => spawn(None, work)
|
||||
.await
|
||||
.expect("unisolated worker")
|
||||
.expect("unisolated operation"),
|
||||
}
|
||||
_ => spawn(None, work)
|
||||
.await
|
||||
.expect("unisolated worker")
|
||||
.expect("unisolated operation"),
|
||||
}),
|
||||
));
|
||||
let (inherited, original_policy) = deadline(started_rx).await.expect("owned work starts");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("waiter cancelled").is_cancelled());
|
||||
let held: Vec<_> = gates.iter().map(|gate| gate.snapshot().active).collect();
|
||||
release_tx.send(()).expect("owned work survives cancellation");
|
||||
deadline(async {
|
||||
while gates.iter().any(|gate| gate.snapshot().active != 0) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}),
|
||||
));
|
||||
let (inherited, original_policy) = deadline(started_rx).await.expect("owned work starts");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("waiter cancelled").is_cancelled());
|
||||
let held = gate.snapshot().active;
|
||||
release_tx.send(()).expect("owned work survives cancellation");
|
||||
deadline(async {
|
||||
while gate.snapshot().active != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert!(inherited, "mode {mode} lost resource ownership");
|
||||
assert!(original_policy, "mode {mode} changed the detached worker's disk policy");
|
||||
assert_eq!(held, 1, "mode {mode} released a live worker's unit");
|
||||
})
|
||||
.await;
|
||||
assert!(inherited, "mode {mode} lost resource ownership");
|
||||
assert!(original_policy, "mode {mode} changed the detached worker's disk policy");
|
||||
assert_eq!(held, vec![1; physical_count], "mode {mode} released a live worker's unit");
|
||||
}
|
||||
}
|
||||
}
|
||||
fn config(pending: usize, workers: usize, blocking_threads: usize) -> Config {
|
||||
|
||||
@@ -40,6 +40,7 @@ use std::sync::OnceLock;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
io,
|
||||
path::PathBuf,
|
||||
pin::Pin,
|
||||
sync::{
|
||||
@@ -99,6 +100,16 @@ async fn observe_disk_operation<F: std::future::Future>(
|
||||
}
|
||||
}
|
||||
|
||||
// The health, admission and timeout layers above a disk operation each embed its
|
||||
// future, and unoptimized builds copy it into every awaiting frame. Box it where
|
||||
// it is created so those layers carry only a pointer: one allocation per operation.
|
||||
fn boxed_disk_operation<F, Fut>(operation: F, io_permit: Option<crate::disk::io_admission::DiskIoPermit>) -> Pin<Box<Fut>>
|
||||
where
|
||||
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
|
||||
{
|
||||
Box::pin(operation(io_permit))
|
||||
}
|
||||
|
||||
struct SlowDiskWriter {
|
||||
inner: crate::disk::FileWriter,
|
||||
endpoint: Endpoint,
|
||||
@@ -1574,6 +1585,8 @@ pub struct LocalDiskWrapper {
|
||||
metrics: Arc<DiskHealthMetricEpoch>,
|
||||
/// Per-drive foreground/background admission shared across wrapper reconnects.
|
||||
io_admission: Arc<DiskIoAdmission>,
|
||||
/// Physical leaves are refreshed by the node sampler, never by object I/O.
|
||||
physical_admission: Arc<crate::disk::io_admission::PhysicalDiskBinding>,
|
||||
/// Whether health checking is enabled
|
||||
health_check: bool,
|
||||
/// Cancellation token for monitoring tasks
|
||||
@@ -1728,6 +1741,7 @@ impl LocalDiskWrapper {
|
||||
health,
|
||||
metrics,
|
||||
io_admission,
|
||||
physical_admission: Arc::new(crate::disk::io_admission::PhysicalDiskBinding::default()),
|
||||
health_check: health_check && env_health_check,
|
||||
cancel_token: CancellationToken::new(),
|
||||
disk_id: Arc::new(RwLock::new(None)),
|
||||
@@ -1745,6 +1759,10 @@ impl LocalDiskWrapper {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn physical_admission(&self) -> Arc<crate::disk::io_admission::PhysicalDiskBinding> {
|
||||
self.physical_admission.clone()
|
||||
}
|
||||
|
||||
pub fn get_disk(&self) -> Arc<LocalDisk> {
|
||||
self.disk.clone()
|
||||
}
|
||||
@@ -1786,6 +1804,33 @@ impl LocalDiskWrapper {
|
||||
context
|
||||
}
|
||||
|
||||
// Projection cannot wait: callers may already hold namespace or cache locks.
|
||||
// Enter before identity probes, disk permits, or owned native jobs are created.
|
||||
async fn with_native_resources<F: std::future::Future>(&self, operation: F) -> Result<F::Output> {
|
||||
let Some(resources) = current_resource_unit() else {
|
||||
if super::remote_io_admission::active() {
|
||||
return Err(DiskError::Io(io::Error::from(io::ErrorKind::WouldBlock)));
|
||||
}
|
||||
return Ok(operation.await);
|
||||
};
|
||||
let resources = if let Some(permits) = resources.as_set() {
|
||||
let unavailable = || DiskError::Io(io::Error::from(io::ErrorKind::WouldBlock));
|
||||
let plan = self.physical_admission.plan().ok_or_else(unavailable)?;
|
||||
let native = plan
|
||||
.capture_or_acquire_native(permits)
|
||||
.map_err(|error| DiskError::Io(io::Error::new(io::ErrorKind::WouldBlock, error)))?;
|
||||
if !plan.is_current() {
|
||||
return Err(unavailable());
|
||||
}
|
||||
resources.with_permits(native)
|
||||
} else {
|
||||
resources
|
||||
};
|
||||
#[cfg(target_os = "linux")]
|
||||
let resources = resources.with_mount_lease(self.disk.native_mount_lease());
|
||||
Ok(with_optional_resource_unit(Some(resources), operation).await)
|
||||
}
|
||||
|
||||
async fn acquire_disk_io_permit(&self) -> Result<Option<crate::disk::io_admission::DiskIoPermit>> {
|
||||
if let Some(context) = current_disk_io_context() {
|
||||
if let Some(permit) = context.permit
|
||||
@@ -2201,6 +2246,19 @@ impl LocalDiskWrapper {
|
||||
versions: Vec<FileInfoVersions>,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Vec<Option<Error>> {
|
||||
let count = versions.len();
|
||||
self.with_native_resources(self.delete_versions_admitted(volume, versions, opts, namespace_owner))
|
||||
.await
|
||||
.unwrap_or_else(|error| vec![Some(error); count])
|
||||
}
|
||||
|
||||
async fn delete_versions_admitted(
|
||||
&self,
|
||||
volume: &str,
|
||||
versions: Vec<FileInfoVersions>,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Vec<Option<Error>> {
|
||||
let version_count = versions.len();
|
||||
self.metrics.record_operation_call("delete_versions");
|
||||
@@ -2253,6 +2311,19 @@ impl LocalDiskWrapper {
|
||||
versions: Vec<GuardedFileInfoVersions>,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Vec<Option<Error>> {
|
||||
let count = versions.len();
|
||||
self.with_native_resources(self.delete_versions_guarded_admitted(volume, versions, opts, namespace_owner))
|
||||
.await
|
||||
.unwrap_or_else(|error| vec![Some(error); count])
|
||||
}
|
||||
|
||||
async fn delete_versions_guarded_admitted(
|
||||
&self,
|
||||
volume: &str,
|
||||
versions: Vec<GuardedFileInfoVersions>,
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Vec<Option<Error>> {
|
||||
let version_count = versions.len();
|
||||
self.metrics.record_operation_call("delete_versions");
|
||||
@@ -2327,6 +2398,51 @@ impl LocalDiskWrapper {
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<Option<(crate::disk::DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
DiskMetricMutation::Delete,
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
.await
|
||||
})
|
||||
},
|
||||
crate::disk::get_delete_recovery_timeout()?,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: uuid::Uuid,
|
||||
journal: &crate::disk::DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<bool> {
|
||||
self.track_disk_health_mutation(
|
||||
"settle_delete_transaction",
|
||||
DiskMetricMutation::Delete,
|
||||
|| {
|
||||
Box::pin(async {
|
||||
self.disk
|
||||
.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.await
|
||||
})
|
||||
},
|
||||
crate::disk::get_delete_recovery_timeout()?,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Track disk health for an operation.
|
||||
/// This method should wrap disk operations to ensure health checking.
|
||||
pub async fn track_disk_health<T, F, Fut>(&self, operation: F, timeout_duration: Duration) -> Result<T>
|
||||
@@ -2451,6 +2567,28 @@ impl LocalDiskWrapper {
|
||||
timeout_health_action: TimeoutHealthAction,
|
||||
mutation: DiskMetricMutation,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
self.with_native_resources(self.track_admitted_disk_operation(
|
||||
op,
|
||||
operation,
|
||||
timeout_duration,
|
||||
timeout_health_action,
|
||||
mutation,
|
||||
))
|
||||
.await?
|
||||
}
|
||||
|
||||
async fn track_admitted_disk_operation<T, F, Fut>(
|
||||
&self,
|
||||
op: &'static str,
|
||||
operation: F,
|
||||
timeout_duration: Duration,
|
||||
timeout_health_action: TimeoutHealthAction,
|
||||
mutation: DiskMetricMutation,
|
||||
) -> Result<T>
|
||||
where
|
||||
F: FnOnce(Option<crate::disk::io_admission::DiskIoPermit>) -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
@@ -2483,13 +2621,14 @@ impl LocalDiskWrapper {
|
||||
let _waiting_guard = self.health.waiting_guard();
|
||||
let _metric_waiting_guard = self.metrics.waiting_guard();
|
||||
let started = Instant::now();
|
||||
let operation = observe_disk_operation(|| self.endpoint(), op, operation(io_permit));
|
||||
let operation = observe_disk_operation(|| self.endpoint(), op, boxed_disk_operation(operation, io_permit));
|
||||
|
||||
if timeout_duration == Duration::ZERO {
|
||||
let result = operation.await;
|
||||
self.metrics.record_operation_latency(op, started.elapsed());
|
||||
self.record_result_error_metrics(&result);
|
||||
if result.is_ok() {
|
||||
super::remote_io_admission::record_progress();
|
||||
self.health.record_operation_success(&self.endpoint(), "operation_success");
|
||||
self.metrics.record_mutation_success(mutation);
|
||||
}
|
||||
@@ -2504,6 +2643,7 @@ impl LocalDiskWrapper {
|
||||
self.record_result_error_metrics(&operation_result);
|
||||
// Log success; the waiting guard balances every exit path.
|
||||
if operation_result.is_ok() {
|
||||
super::remote_io_admission::record_progress();
|
||||
self.health.record_operation_success(&self.endpoint(), "operation_success");
|
||||
self.metrics.record_mutation_success(mutation);
|
||||
}
|
||||
@@ -2675,7 +2815,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
if self.health.is_faulty() {
|
||||
return false;
|
||||
}
|
||||
let Ok(Some(disk_id)) = self.disk.get_disk_id().await else {
|
||||
let Ok(Some(disk_id)) = self.get_disk_id().await else {
|
||||
return false;
|
||||
};
|
||||
|
||||
@@ -2708,7 +2848,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn get_disk_id(&self) -> Result<Option<Uuid>> {
|
||||
self.disk.get_disk_id().await
|
||||
self.with_native_resources(self.disk.get_disk_id()).await?
|
||||
}
|
||||
|
||||
async fn set_disk_id(&self, id: Option<Uuid>) -> Result<()> {
|
||||
@@ -3140,25 +3280,29 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn read_file(&self, volume: &str, path: &str) -> Result<crate::disk::FileReader> {
|
||||
let reader = self
|
||||
.track_disk_health_with_op(
|
||||
"read_file",
|
||||
|| async { self.disk.read_file(volume, path).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await?;
|
||||
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context())))
|
||||
self.track_disk_health_with_op(
|
||||
"read_file",
|
||||
|| async {
|
||||
let reader = self.disk.read_file(volume, path).await?;
|
||||
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context()))
|
||||
as crate::disk::FileReader)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_file_stream(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<crate::disk::FileReader> {
|
||||
let reader = self
|
||||
.track_disk_health_with_op(
|
||||
"read_file_stream",
|
||||
|| async { self.disk.read_file_stream(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await?;
|
||||
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context())))
|
||||
self.track_disk_health_with_op(
|
||||
"read_file_stream",
|
||||
|| async {
|
||||
let reader = self.disk.read_file_stream(volume, path, offset, length).await?;
|
||||
Ok(Box::new(AdmissionReader::new(reader, self.io_admission.clone(), self.disk_io_context()))
|
||||
as crate::disk::FileReader)
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_file_stream_chunks(
|
||||
@@ -3168,20 +3312,21 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
offset: usize,
|
||||
length: usize,
|
||||
) -> Result<Option<rustfs_rio::ChunkReaderBox>> {
|
||||
let reader = self
|
||||
.track_disk_health_with_op(
|
||||
"read_file_stream_chunks",
|
||||
|| async { self.disk.read_file_stream_chunks(volume, path, offset, length).await },
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await?;
|
||||
Ok(reader.map(|reader| {
|
||||
Box::new(AdmissionReader::new(
|
||||
BoxedChunkReader(reader),
|
||||
self.io_admission.clone(),
|
||||
self.disk_io_context(),
|
||||
)) as rustfs_rio::ChunkReaderBox
|
||||
}))
|
||||
self.track_disk_health_with_op(
|
||||
"read_file_stream_chunks",
|
||||
|| async {
|
||||
let reader = self.disk.read_file_stream_chunks(volume, path, offset, length).await?;
|
||||
Ok(reader.map(|reader| {
|
||||
Box::new(AdmissionReader::new(
|
||||
BoxedChunkReader(reader),
|
||||
self.io_admission.clone(),
|
||||
self.disk_io_context(),
|
||||
)) as rustfs_rio::ChunkReaderBox
|
||||
}))
|
||||
},
|
||||
get_max_timeout_duration(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_file_mmap_copy(&self, volume: &str, path: &str, offset: usize, length: usize) -> Result<bytes::Bytes> {
|
||||
@@ -4146,6 +4291,573 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_disk_resource_identity_probes_keep_only_their_native_device_after_cancel() {
|
||||
assert_native_identity_probe_cancellation(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_background_native_identity_cancellation_keeps_receiver_device() {
|
||||
assert_native_identity_probe_cancellation(true).await;
|
||||
}
|
||||
|
||||
async fn assert_native_identity_probe_cancellation(remote: bool) {
|
||||
use rustfs_concurrency::adaptive::{
|
||||
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
|
||||
permit_set::AdaptivePermitSetOwner,
|
||||
};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
for read in [false, true] {
|
||||
for online in [false, true] {
|
||||
let dir = tempfile::tempdir().expect("format probe fixture");
|
||||
let mut endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
let meta = dir.path().join(RUSTFS_META_BUCKET);
|
||||
tokio::fs::create_dir_all(&meta).await.expect("metadata directory");
|
||||
let mut format = FormatV3::new(1, 1);
|
||||
format.erasure.this = format.erasure.sets[0][0];
|
||||
tokio::fs::write(meta.join(FORMAT_CONFIG_FILE), format.to_json().expect("format bytes"))
|
||||
.await
|
||||
.expect("write format");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("formatted disk"));
|
||||
{
|
||||
let mut cache = disk.format_info.write().await;
|
||||
cache.last_check = None;
|
||||
if read {
|
||||
cache.file_info = None;
|
||||
}
|
||||
}
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
disk.install_format_probe_hook(read, move || {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.blocking_recv();
|
||||
});
|
||||
let wrapper = Arc::new(LocalDiskWrapper::new(disk, false));
|
||||
let a = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let b = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![a.clone(), b.clone()]).expect("catalog");
|
||||
wrapper
|
||||
.physical_admission
|
||||
.set_test_shared_capacity(shared.clone(), vec![a.clone()])
|
||||
.expect("native device A");
|
||||
let node = NodeAdmission::new(NonZeroU16::MIN);
|
||||
node.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
let parent = (!remote).then(|| shared.try_acquire_with_node(&node).expect("planned devices"));
|
||||
let task = tokio::spawn({
|
||||
let wrapper = wrapper.clone();
|
||||
let node = node.clone();
|
||||
async move {
|
||||
let operation = async {
|
||||
if online {
|
||||
assert!(wrapper.is_online().await);
|
||||
} else {
|
||||
wrapper.get_disk_id().await.expect("disk identity");
|
||||
}
|
||||
};
|
||||
if remote {
|
||||
crate::disk::with_remote_background_io(Some(crate::disk::RemoteBackgroundIo::new(node)), async {
|
||||
crate::disk::remote_io_admission::admit(&wrapper.physical_admission).expect("RPC disk admission");
|
||||
operation.await;
|
||||
})
|
||||
.await;
|
||||
} else {
|
||||
let parent = parent.expect("local operation admission");
|
||||
crate::disk::with_background_resource_set(parent.resources().clone(), operation).await;
|
||||
parent.complete();
|
||||
}
|
||||
}
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), started_rx)
|
||||
.await
|
||||
.expect("native probe starts")
|
||||
.expect("native signal");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancel waiter").is_cancelled());
|
||||
let held = [node.snapshot().active, a.snapshot().active, b.snapshot().active];
|
||||
let healthy_available = b.try_acquire().is_some();
|
||||
release_tx.send(()).expect("release native probe before assertions");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while a.snapshot().active != 0 || b.snapshot().active != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("native probe releases its device");
|
||||
assert_eq!(held, [0, 1, 0], "read={read}, online={online}");
|
||||
assert!(healthy_available);
|
||||
assert_eq!([node.completed(), a.completed(), b.completed()], [Some(0), Some(0), Some(0)]);
|
||||
assert_eq!(wrapper.get_disk_id().await.expect("format remains readable"), Some(format.erasure.this));
|
||||
assert!(wrapper.is_online().await);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_disk_resource_projection_releases_unrelated_devices_after_native_cancellation() {
|
||||
use crate::disk::io_admission::spawn_blocking_with_disk_io_permit;
|
||||
use rustfs_concurrency::adaptive::{
|
||||
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
|
||||
permit_set::AdaptivePermitSetOwner,
|
||||
};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
for auxiliary in [false, true] {
|
||||
let dir = tempfile::tempdir().expect("local disk fixture");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
let first = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let second = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![first.clone(), second.clone()])
|
||||
.expect("shared operation resources");
|
||||
wrapper
|
||||
.physical_admission
|
||||
.set_test_shared_capacity(shared.clone(), vec![if auxiliary { second.clone() } else { first.clone() }])
|
||||
.expect("only the first device backs this wrapper");
|
||||
let node = NodeAdmission::new(NonZeroU16::MIN);
|
||||
node.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
let parent = if auxiliary {
|
||||
shared
|
||||
.select(std::slice::from_ref(&first))
|
||||
.expect("initial object device")
|
||||
.try_acquire_with_node(&node)
|
||||
} else {
|
||||
shared.try_acquire_with_node(&node)
|
||||
}
|
||||
.expect("initial object admission");
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
crate::disk::with_background_resource_set(parent.resources().clone(), async {
|
||||
wrapper
|
||||
.track_disk_health_with_op_timeout_action_and_mutation_with_io_permit(
|
||||
"read_all",
|
||||
move |permit| async move {
|
||||
spawn_blocking_with_disk_io_permit(permit, move || {
|
||||
started_tx.send(()).expect("signal native start");
|
||||
release_rx.blocking_recv().expect("release native I/O");
|
||||
})
|
||||
.await
|
||||
.expect("native I/O joined");
|
||||
Ok(())
|
||||
},
|
||||
Duration::ZERO,
|
||||
TimeoutHealthAction::MarkFailure,
|
||||
DiskMetricMutation::None,
|
||||
)
|
||||
.await
|
||||
.expect("wrapped I/O");
|
||||
})
|
||||
.await;
|
||||
parent.complete();
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), started_rx)
|
||||
.await
|
||||
.expect("native work starts")
|
||||
.expect("start notification");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancel waiter").is_cancelled());
|
||||
let held = [node.snapshot().active, first.snapshot().active, second.snapshot().active];
|
||||
let healthy_available = if auxiliary {
|
||||
first.try_acquire().is_some()
|
||||
} else {
|
||||
second.try_acquire().is_some()
|
||||
};
|
||||
release_tx.send(()).expect("finish native work before asserting");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while first.snapshot().active != 0 || second.snapshot().active != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("native resource return");
|
||||
assert_eq!(
|
||||
held,
|
||||
if auxiliary { [0, 0, 1] } else { [0, 1, 0] },
|
||||
"only the actual native device remains charged"
|
||||
);
|
||||
assert!(healthy_available, "healthy work must resume before native I/O finishes");
|
||||
assert_eq!(node.completed(), Some(0), "cancelled work is not progress");
|
||||
assert_eq!([first.completed(), second.completed()], [Some(0), Some(0)]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_disk_resource_projection_escaped_readers_keep_only_their_device() {
|
||||
use rustfs_concurrency::adaptive::{AdaptiveAdmission, permit_set::AdaptivePermitSetOwner};
|
||||
use std::num::NonZeroU16;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let dir = tempfile::tempdir().expect("reader fixture");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
tokio::fs::create_dir_all(dir.path().join("bucket"))
|
||||
.await
|
||||
.expect("bucket directory");
|
||||
let bytes = b"physical disk reader payload";
|
||||
tokio::fs::write(dir.path().join("bucket/object"), bytes)
|
||||
.await
|
||||
.expect("reader bytes");
|
||||
let first = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let second = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![first.clone(), second.clone()]).expect("parent resources");
|
||||
wrapper
|
||||
.physical_admission
|
||||
.set_test_shared_capacity(shared.clone(), vec![first.clone()])
|
||||
.expect("reader device");
|
||||
|
||||
for stream in [false, true] {
|
||||
let a = first.try_acquire().expect("first device");
|
||||
let b = second.try_acquire().expect("second device");
|
||||
let parent = shared.capture(&[&a, &b]).expect("admitted parent");
|
||||
drop((a, b));
|
||||
let mut reader = crate::disk::with_background_resource_set(parent, async {
|
||||
if stream {
|
||||
wrapper.read_file_stream("bucket", "object", 0, bytes.len()).await
|
||||
} else {
|
||||
wrapper.read_file("bucket", "object").await
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("reader escapes the resource scope");
|
||||
assert_eq!([first.snapshot().active, second.snapshot().active], [1, 0]);
|
||||
assert!(second.try_acquire().is_some(), "unrelated work is available while the reader lives");
|
||||
let mut actual = Vec::new();
|
||||
reader.read_to_end(&mut actual).await.expect("consume escaped reader");
|
||||
assert_eq!(actual, bytes);
|
||||
drop(reader);
|
||||
assert_eq!(first.snapshot().active, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn local_disk_resource_projection_cancelled_stream_read_retains_native_device() {
|
||||
use rustfs_concurrency::adaptive::{AdaptiveAdmission, permit_set::AdaptivePermitSetOwner};
|
||||
use std::num::NonZeroU16;
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.max_blocking_threads(1)
|
||||
.build()
|
||||
.expect("one-thread native read pool");
|
||||
runtime.block_on(async {
|
||||
let dir = tempfile::tempdir().expect("native reader fixture");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
std::fs::create_dir(dir.path().join("bucket")).expect("bucket directory");
|
||||
let bytes = b"physical disk native reader payload";
|
||||
std::fs::write(dir.path().join("bucket/object"), bytes).expect("reader bytes");
|
||||
let first = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let second = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let shared =
|
||||
AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![first.clone(), second.clone()]).expect("parent resources");
|
||||
wrapper
|
||||
.physical_admission
|
||||
.set_test_shared_capacity(shared.clone(), vec![first.clone()])
|
||||
.expect("reader device");
|
||||
let mut observations = Vec::new();
|
||||
for stream in [false, true] {
|
||||
let a = first.try_acquire().expect("first device");
|
||||
let b = second.try_acquire().expect("second device");
|
||||
let parent = shared.capture(&[&a, &b]).expect("admitted parent");
|
||||
drop((a, b));
|
||||
let mut reader = crate::disk::with_background_resource_set(
|
||||
parent,
|
||||
crate::disk::with_background_disk_io(CancellationToken::new(), async {
|
||||
if stream {
|
||||
wrapper.read_file_stream("bucket", "object", 3, bytes.len() - 3).await
|
||||
} else {
|
||||
wrapper.read_file("bucket", "object").await
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("background reader escapes its parent");
|
||||
assert_eq!([first.snapshot().active, second.snapshot().active], [1, 0]);
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel::<()>();
|
||||
let blocker = tokio::task::spawn_blocking(move || {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
started_rx.await.expect("native pool occupied after open");
|
||||
let (pending_tx, pending_rx) = tokio::sync::oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut data = [0; 1];
|
||||
let mut pending_tx = Some(pending_tx);
|
||||
std::future::poll_fn(|cx| {
|
||||
let mut buffer = ReadBuf::new(&mut data);
|
||||
let result = Pin::new(&mut reader).poll_read(cx, &mut buffer);
|
||||
if result.is_pending()
|
||||
&& let Some(signal) = pending_tx.take()
|
||||
{
|
||||
let _ = signal.send(());
|
||||
}
|
||||
result
|
||||
})
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(5), pending_rx)
|
||||
.await
|
||||
.expect("read queues behind native worker")
|
||||
.expect("pending read signal");
|
||||
let admitted = wrapper.background_io_active_for_tests();
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancel pending read").is_cancelled());
|
||||
let held = [first.snapshot().active, second.snapshot().active];
|
||||
release_tx.send(()).expect("release native pool before asserting");
|
||||
blocker.await.expect("blocking worker released");
|
||||
tokio::task::spawn_blocking(|| ())
|
||||
.await
|
||||
.expect("drain the single native queue");
|
||||
assert_eq!([first.snapshot().active, second.snapshot().active], [0, 0]);
|
||||
observations.push((stream, admitted, held));
|
||||
}
|
||||
for (stream, admitted, held) in observations {
|
||||
assert!(admitted, "stream={stream}: cancellation follows disk admission, not its wait");
|
||||
assert_eq!(held, [1, 0], "stream={stream}: only the pending native read's device stays charged");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn projected_stream_memory_survives_scope_exit_and_hot_shrink() {
|
||||
use crate::disk::io_admission::{BackgroundMemoryBudget, ResourceIoUnit, with_optional_resource_unit};
|
||||
use rustfs_concurrency::adaptive::{AdaptiveAdmission, permit_set::AdaptivePermitSetOwner};
|
||||
use std::num::NonZeroU16;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let dir = tempfile::tempdir().expect("memory admission fixture");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
std::fs::create_dir(dir.path().join("bucket")).expect("bucket directory");
|
||||
let bytes = b"0123456789abcdefghijklmnopqrstuv";
|
||||
std::fs::write(dir.path().join("bucket/object"), bytes).expect("reader bytes");
|
||||
let first = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let second = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![first.clone(), second.clone()]).expect("parent resources");
|
||||
wrapper
|
||||
.physical_admission
|
||||
.set_test_shared_capacity(shared.clone(), vec![first.clone()])
|
||||
.expect("selected device");
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
|
||||
for stream in [false, true] {
|
||||
memory.set_sampled_limit(4, Duration::from_secs(30));
|
||||
let a = first.try_acquire().expect("first device");
|
||||
let b = second.try_acquire().expect("second device");
|
||||
let parent =
|
||||
ResourceIoUnit::from(shared.capture(&[&a, &b]).expect("admitted parent")).with_memory_budget(memory.clone());
|
||||
drop((a, b));
|
||||
let mut reader = with_optional_resource_unit(Some(parent), async {
|
||||
if stream {
|
||||
wrapper.read_file_stream("bucket", "object", 0, bytes.len()).await
|
||||
} else {
|
||||
wrapper.read_file("bucket", "object").await
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("reader escapes its resource scope");
|
||||
assert_eq!(memory.snapshot().used, 0, "opening does not allocate a read buffer");
|
||||
let mut prefix = [0; 4];
|
||||
reader.read_exact(&mut prefix).await.expect("initial read");
|
||||
assert_eq!(&prefix, &bytes[..4]);
|
||||
assert_eq!(memory.snapshot().used, 4, "projection retains the same memory owner");
|
||||
assert_eq!([first.snapshot().active, second.snapshot().active], [1, 0]);
|
||||
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
let mut reused = [0; 2];
|
||||
reader
|
||||
.read_exact(&mut reused)
|
||||
.await
|
||||
.expect("already allocated storage remains usable");
|
||||
assert_eq!(&reused, &bytes[4..6]);
|
||||
let mut next = [99; 8];
|
||||
let error = reader.read(&mut next).await.expect_err("new storage is blocked after shrink");
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::WouldBlock);
|
||||
assert_eq!(next, [99; 8], "denied allocation exposes no stale bytes");
|
||||
assert_eq!(memory.snapshot().used, 4);
|
||||
|
||||
memory.set_sampled_limit(12, Duration::from_secs(30));
|
||||
reader
|
||||
.read_exact(&mut next)
|
||||
.await
|
||||
.expect("growth includes old and replacement storage");
|
||||
assert_eq!(&next, &bytes[6..14], "denial did not advance the file");
|
||||
assert_eq!(memory.snapshot().used, 8);
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
assert_eq!([first.snapshot().active, second.snapshot().active], [0, 0]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_disk_resource_projection_auxiliary_metadata_retries_without_parking_or_false_progress() {
|
||||
use rustfs_concurrency::adaptive::{
|
||||
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
|
||||
permit_set::AdaptivePermitSetOwner,
|
||||
};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
let dir = tempfile::tempdir().expect("metadata fixture");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
tokio::fs::create_dir_all(dir.path().join("bucket"))
|
||||
.await
|
||||
.expect("metadata bucket");
|
||||
tokio::fs::write(dir.path().join("bucket/metadata"), b"auxiliary metadata")
|
||||
.await
|
||||
.expect("metadata bytes");
|
||||
let a = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let b = AdaptiveAdmission::new(NonZeroU16::MIN, 0);
|
||||
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![a.clone(), b.clone()]).expect("topology");
|
||||
wrapper
|
||||
.physical_admission
|
||||
.set_test_shared_capacity(shared.clone(), vec![b.clone()])
|
||||
.expect("metadata on B");
|
||||
let node = NodeAdmission::new(NonZeroU16::MIN);
|
||||
node.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
let parent = shared
|
||||
.select(std::slice::from_ref(&a))
|
||||
.expect("object on A")
|
||||
.try_acquire_with_node(&node)
|
||||
.expect("object admission");
|
||||
crate::disk::with_background_resource_set(parent.resources().clone(), async {
|
||||
let mut read = Box::pin(wrapper.read_all("bucket", "metadata"));
|
||||
let rejected = std::future::poll_fn(|cx| Poll::Ready(read.as_mut().poll(cx))).await;
|
||||
assert!(
|
||||
matches!(rejected, Poll::Ready(Err(DiskError::Io(error))) if error.kind() == io::ErrorKind::WouldBlock),
|
||||
"auxiliary admission must return immediately under storage locks"
|
||||
);
|
||||
drop(read);
|
||||
assert!(
|
||||
matches!(wrapper.get_disk_id().await, Err(DiskError::Io(error)) if error.kind() == io::ErrorKind::WouldBlock)
|
||||
);
|
||||
assert!(!wrapper.is_online().await, "identity checks cannot bypass closed physical capacity");
|
||||
assert_eq!([a.snapshot().active, b.snapshot().active, node.snapshot().active], [1, 0, 1]);
|
||||
assert_eq!([a.completed(), b.completed(), node.completed()], [Some(0), Some(0), Some(0)]);
|
||||
b.set_limit(1);
|
||||
for _ in 0..2 {
|
||||
assert_eq!(
|
||||
wrapper.read_all("bucket", "metadata").await.expect("retry metadata read"),
|
||||
Bytes::from_static(b"auxiliary metadata")
|
||||
);
|
||||
assert_eq!(b.snapshot().active, 1, "repeated I/O reuses the parent's physical unit");
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert_eq!(wrapper.metrics_snapshot().total_errors_availability, 0);
|
||||
assert_eq!([a.completed(), b.completed(), node.completed()], [Some(0), Some(0), Some(0)]);
|
||||
parent.complete();
|
||||
assert_eq!([a.completed(), b.completed(), node.completed()], [Some(1), Some(1), Some(1)]);
|
||||
assert_eq!([a.snapshot().active, b.snapshot().active, node.snapshot().active], [0, 0, 0]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_disk_resource_projection_defers_missing_or_foreign_owners_before_io() {
|
||||
use rustfs_concurrency::adaptive::{AdaptiveAdmission, permit_set::AdaptivePermitSetOwner};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
let dir = tempfile::tempdir().expect("unavailable resource fixture");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("fixture path")).expect("local endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("local disk"));
|
||||
let wrapper = LocalDiskWrapper::new(disk, false);
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![gate.clone()]).expect("parent");
|
||||
let permit = gate.try_acquire().expect("parent device");
|
||||
let parent = shared.capture(&[&permit]).expect("parent ownership");
|
||||
drop(permit);
|
||||
for foreign in [false, true] {
|
||||
if foreign {
|
||||
wrapper
|
||||
.physical_admission
|
||||
.set_test_capacity(vec![AdaptiveAdmission::new(NonZeroU16::MIN, 1)])
|
||||
.expect("replaced device");
|
||||
}
|
||||
crate::disk::with_background_resource_set(parent.clone(), async {
|
||||
let polled = std::cell::Cell::new(false);
|
||||
let result = wrapper
|
||||
.track_disk_health_with_op(
|
||||
"read_all",
|
||||
|| async {
|
||||
polled.set(true);
|
||||
Ok(())
|
||||
},
|
||||
Duration::ZERO,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(DiskError::Io(error)) if error.kind() == io::ErrorKind::WouldBlock));
|
||||
assert!(!polled.get(), "unavailable ownership cannot start native work");
|
||||
let group = FileInfoVersions::default();
|
||||
let plain = wrapper
|
||||
.delete_versions("bucket", vec![group.clone(); 2], DeleteOptions::default())
|
||||
.await;
|
||||
let guarded = wrapper
|
||||
.delete_versions_guarded_with_namespace_owner(
|
||||
"bucket",
|
||||
vec![
|
||||
GuardedFileInfoVersions {
|
||||
group,
|
||||
expected_current: crate::disk::CurrentDeleteGeneration::Absent
|
||||
};
|
||||
2
|
||||
],
|
||||
DeleteOptions::default(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
for errors in [plain, guarded] {
|
||||
assert_eq!(errors.len(), 2);
|
||||
assert!(errors.iter().all(|error| matches!(error,
|
||||
Some(DiskError::Io(error)) if error.kind() == io::ErrorKind::WouldBlock)));
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert_eq!(wrapper.health.waiting_count(), 0);
|
||||
assert_eq!(
|
||||
wrapper.metrics_snapshot().total_errors_availability,
|
||||
0,
|
||||
"resource deferral is not a failed drive"
|
||||
);
|
||||
assert_eq!(gate.snapshot().active, 1, "rejection does not refund parent ownership");
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_multipart_recovery_waiter_preserves_physical_owner(wait_timeout: Duration) {
|
||||
use crate::disk::multipart_source::{
|
||||
MULTIPART_SOURCE_PROTOCOL_VERSION, MultipartRecoveryRound, MultipartSourceOperation, MultipartSourcePhase,
|
||||
|
||||
+132
-62
@@ -16,40 +16,15 @@ use std::{
|
||||
collections::HashMap,
|
||||
fs::Metadata,
|
||||
path::{Path, PathBuf},
|
||||
sync::{Arc, Mutex, OnceLock},
|
||||
sync::Mutex,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tokio::{
|
||||
fs::{self, File},
|
||||
io,
|
||||
};
|
||||
use tokio::{fs::File, io};
|
||||
|
||||
static READONLY_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
|
||||
static WRITEONLY_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
|
||||
static READWRITE_OPTIONS: OnceLock<Arc<fs::OpenOptions>> = OnceLock::new();
|
||||
|
||||
fn get_readonly_options() -> &'static Arc<fs::OpenOptions> {
|
||||
READONLY_OPTIONS.get_or_init(|| {
|
||||
let mut opts = fs::OpenOptions::new();
|
||||
opts.read(true);
|
||||
Arc::new(opts)
|
||||
})
|
||||
}
|
||||
|
||||
fn get_writeonly_options() -> &'static Arc<fs::OpenOptions> {
|
||||
WRITEONLY_OPTIONS.get_or_init(|| {
|
||||
let mut opts = fs::OpenOptions::new();
|
||||
opts.write(true);
|
||||
Arc::new(opts)
|
||||
})
|
||||
}
|
||||
|
||||
fn get_readwrite_options() -> &'static Arc<fs::OpenOptions> {
|
||||
READWRITE_OPTIONS.get_or_init(|| {
|
||||
let mut opts = fs::OpenOptions::new();
|
||||
opts.read(true).write(true);
|
||||
Arc::new(opts)
|
||||
})
|
||||
async fn native_io<T: Send + 'static>(operation: impl FnOnce() -> io::Result<T> + Send + 'static) -> io::Result<T> {
|
||||
super::io_admission::spawn_blocking_with_resource_ownership(operation)
|
||||
.await
|
||||
.map_err(|_| io::Error::other("background task failed"))?
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
@@ -117,33 +92,32 @@ pub const O_APPEND: FileMode = 0x00400;
|
||||
// create_new: bool,
|
||||
|
||||
pub async fn open_file(path: impl AsRef<Path>, mode: FileMode) -> io::Result<File> {
|
||||
let base_opts = match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
|
||||
O_RDONLY => get_readonly_options(),
|
||||
O_WRONLY => get_writeonly_options(),
|
||||
O_RDWR => get_readwrite_options(),
|
||||
_ => get_readonly_options(),
|
||||
};
|
||||
|
||||
if (mode & (O_CREATE | O_APPEND | O_TRUNC)) != 0 {
|
||||
let mut opts = (**base_opts).clone();
|
||||
if mode & O_CREATE != 0 {
|
||||
opts.create(true);
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || {
|
||||
let mut options = std::fs::OpenOptions::new();
|
||||
match mode & (O_RDONLY | O_WRONLY | O_RDWR) {
|
||||
O_WRONLY => options.write(true),
|
||||
O_RDWR => options.read(true).write(true),
|
||||
_ => options.read(true),
|
||||
}
|
||||
if mode & O_APPEND != 0 {
|
||||
opts.append(true);
|
||||
.create(mode & O_CREATE != 0)
|
||||
.append(mode & O_APPEND != 0)
|
||||
.truncate(mode & O_TRUNC != 0);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
if mode == O_RDONLY {
|
||||
super::os::prepared_publication_test_hooks::run(
|
||||
super::os::prepared_publication_test_hooks::Stage::NativeReadOpen,
|
||||
&path,
|
||||
);
|
||||
}
|
||||
if mode & O_TRUNC != 0 {
|
||||
opts.truncate(true);
|
||||
}
|
||||
opts.open(path.as_ref()).await
|
||||
} else {
|
||||
base_opts.open(path.as_ref()).await
|
||||
}
|
||||
options.open(path).map(File::from_std)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::metadata(path).await?;
|
||||
Ok(())
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::metadata(path).map(|_| ())).await
|
||||
}
|
||||
|
||||
pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
@@ -152,7 +126,8 @@ pub fn access_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
|
||||
pub async fn lstat(path: impl AsRef<Path>) -> io::Result<Metadata> {
|
||||
fs::symlink_metadata(path).await
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::symlink_metadata(path)).await
|
||||
}
|
||||
|
||||
pub fn lstat_std(path: impl AsRef<Path>) -> io::Result<Metadata> {
|
||||
@@ -170,7 +145,7 @@ pub fn create_dir_all_std(path: &Path) -> io::Result<()> {
|
||||
|
||||
pub async fn make_dir_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
tokio::task::spawn_blocking(move || create_dir_all_std(&path)).await?
|
||||
native_io(move || create_dir_all_std(&path)).await
|
||||
}
|
||||
|
||||
fn is_dir_error(e: &io::Error) -> bool {
|
||||
@@ -183,9 +158,13 @@ fn is_dir_error(e: &io::Error) -> bool {
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir if it's a directory
|
||||
match fs::remove_file(path.as_ref()).await {
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => fs::remove_dir(path.as_ref()).await,
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir(path)).await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
@@ -193,9 +172,13 @@ pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
#[allow(dead_code, reason = "asserted by this file's tests (backlog#1823)")]
|
||||
pub async fn remove_all(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
// Try remove_file first; fall back to remove_dir_all if it's a directory
|
||||
match fs::remove_file(path.as_ref()).await {
|
||||
let file_path = path.as_ref().to_path_buf();
|
||||
match native_io(move || std::fs::remove_file(file_path)).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if is_dir_error(&e) => fs::remove_dir_all(path.as_ref()).await,
|
||||
Err(e) if is_dir_error(&e) => {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::remove_dir_all(path)).await
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
@@ -222,11 +205,14 @@ pub fn remove_all_std(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
|
||||
#[cfg(test)]
|
||||
pub async fn mkdir(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::create_dir(path.as_ref()).await
|
||||
let path = path.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::create_dir(path)).await
|
||||
}
|
||||
|
||||
pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
|
||||
fs::rename(from, to).await
|
||||
let from = from.as_ref().to_path_buf();
|
||||
let to = to.as_ref().to_path_buf();
|
||||
native_io(move || std::fs::rename(from, to)).await
|
||||
}
|
||||
|
||||
#[cfg(any(not(windows), test))]
|
||||
@@ -289,7 +275,7 @@ pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
|
||||
let path_buf = path.to_path_buf();
|
||||
let result = fs::metadata(&path_buf).await;
|
||||
let result = access(&path_buf).await;
|
||||
|
||||
match &result {
|
||||
Ok(_) => BUCKET_EXISTENCE_CACHE.record(path_buf, true),
|
||||
@@ -304,6 +290,7 @@ pub async fn cached_access(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
}
|
||||
|
||||
pub fn invalidate_bucket_cache(path: impl AsRef<Path>) {
|
||||
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
|
||||
BUCKET_EXISTENCE_CACHE.invalidate(path.as_ref());
|
||||
}
|
||||
|
||||
@@ -313,6 +300,89 @@ mod tests {
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
#[test]
|
||||
fn cancelled_filesystem_jobs_retain_physical_resources() {
|
||||
use crate::disk::io_admission::{test_resource_units, with_optional_resource_unit};
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.max_blocking_threads(1)
|
||||
.build()
|
||||
.expect("single native worker");
|
||||
runtime.block_on(async {
|
||||
let mut observations = Vec::new();
|
||||
for count in 1..=2 {
|
||||
for operation in [
|
||||
"access",
|
||||
"cached_access",
|
||||
"lstat",
|
||||
"mkdir",
|
||||
"make_dir_all",
|
||||
"remove",
|
||||
"remove_all",
|
||||
"rename",
|
||||
"open",
|
||||
] {
|
||||
let dir = TempDir::new().expect("filesystem ownership fixture");
|
||||
let source = dir.path().join("source");
|
||||
let target = dir.path().join("target");
|
||||
match operation {
|
||||
"mkdir" | "make_dir_all" | "open" => {}
|
||||
"remove_all" => {
|
||||
std::fs::create_dir(&source).expect("recursive removal directory");
|
||||
std::fs::write(source.join("child"), b"payload").expect("recursive removal child");
|
||||
}
|
||||
_ => std::fs::write(&source, b"payload").expect("existing file"),
|
||||
}
|
||||
let (resources, gates) = test_resource_units(count);
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = std::sync::mpsc::channel();
|
||||
let blocker = tokio::task::spawn_blocking(move || {
|
||||
let _ = started_tx.send(());
|
||||
let _ = release_rx.recv();
|
||||
});
|
||||
started_rx.await.expect("native worker occupied");
|
||||
let mut future = Box::pin(with_optional_resource_unit(Some(resources), async {
|
||||
match operation {
|
||||
"access" => access(&source).await,
|
||||
"cached_access" => cached_access(&source).await,
|
||||
"lstat" => lstat(&source).await.map(|_| ()),
|
||||
"mkdir" => mkdir(&source).await,
|
||||
"make_dir_all" => make_dir_all(source.join("child")).await,
|
||||
"remove" => remove(&source).await,
|
||||
"remove_all" => remove_all(&source).await,
|
||||
"rename" => rename(&source, &target).await,
|
||||
"open" => open_file(&source, O_CREATE | O_WRONLY).await.map(|_| ()),
|
||||
_ => unreachable!("fixed operation list"),
|
||||
}
|
||||
}));
|
||||
assert!(futures::poll!(&mut future).is_pending(), "{operation}: native job queued");
|
||||
drop(future);
|
||||
let held: Vec<_> = gates.iter().map(|gate| gate.snapshot().active).collect();
|
||||
release_tx.send(()).expect("release native worker before assertions");
|
||||
blocker.await.expect("native blocker completed");
|
||||
tokio::task::spawn_blocking(|| ()).await.expect("native queue drained");
|
||||
assert!(gates.iter().all(|gate| gate.snapshot().active == 0));
|
||||
match operation {
|
||||
"mkdir" => assert!(source.is_dir()),
|
||||
"make_dir_all" => assert!(source.join("child").is_dir()),
|
||||
"open" => assert!(source.is_file()),
|
||||
"remove" => assert!(!source.exists()),
|
||||
"rename" => {
|
||||
assert!(!source.exists());
|
||||
assert_eq!(std::fs::read(&target).expect("renamed file"), b"payload");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
observations.push((count, operation, held));
|
||||
}
|
||||
}
|
||||
for (count, operation, held) in observations {
|
||||
assert_eq!(held, vec![1; count], "{operation}: queued native job keeps its physical resources");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_file_mode_constants() {
|
||||
assert_eq!(O_RDONLY, 0x00000);
|
||||
|
||||
@@ -14,6 +14,18 @@
|
||||
|
||||
//! Per-local-disk foreground-priority admission for scanner I/O.
|
||||
|
||||
pub(crate) mod memory;
|
||||
mod physical_identity;
|
||||
mod physical_owners;
|
||||
mod physical_sampler;
|
||||
|
||||
pub use memory::BackgroundMemoryBudget;
|
||||
|
||||
pub(crate) use physical_sampler::PhysicalHealBinding;
|
||||
pub use physical_sampler::{
|
||||
PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler, PhysicalResourcePlan,
|
||||
};
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
@@ -25,18 +37,118 @@ use std::time::Duration;
|
||||
|
||||
use metrics::{counter, gauge, histogram};
|
||||
use rustfs_concurrency::adaptive::AdaptivePermit;
|
||||
use rustfs_concurrency::adaptive::permit_set::AdaptivePermitSet;
|
||||
use rustfs_rio::ChunkReader;
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio::sync::Notify;
|
||||
use tokio::time::Instant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
pub(crate) mod remote;
|
||||
pub use remote::{RemoteBackgroundIo, with_remote_background_io};
|
||||
|
||||
const BACKGROUND_FOREGROUND_BURST: u32 = 8;
|
||||
const BACKGROUND_MAX_WAIT: Duration = Duration::from_millis(100);
|
||||
|
||||
tokio::task_local! {
|
||||
static DISK_IO_CONTEXT: DiskIoContext;
|
||||
static RESOURCE_IO_UNIT: AdaptivePermit;
|
||||
static RESOURCE_IO_UNIT: ResourceIoUnit;
|
||||
}
|
||||
|
||||
/// Native I/O retains the physical subset selected before storage work. A
|
||||
/// layered disk may touch several leaves; unrelated devices stay outside it.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct ResourceIoUnit {
|
||||
permits: ResourceIoPermits,
|
||||
memory_budget: Option<Arc<BackgroundMemoryBudget>>,
|
||||
#[cfg(target_os = "linux")]
|
||||
_mount_lease: Option<Arc<std::fs::File>>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum ResourceIoPermits {
|
||||
Single(AdaptivePermit),
|
||||
Set(AdaptivePermitSet),
|
||||
}
|
||||
|
||||
impl Clone for ResourceIoPermits {
|
||||
fn clone(&self) -> Self {
|
||||
match self {
|
||||
Self::Single(permit) => Self::Single(permit.clone()),
|
||||
Self::Set(permits) => Self::Set(permits.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AdaptivePermit> for ResourceIoUnit {
|
||||
fn from(permit: AdaptivePermit) -> Self {
|
||||
Self {
|
||||
permits: ResourceIoPermits::Single(permit),
|
||||
memory_budget: BackgroundMemoryBudget::current(),
|
||||
#[cfg(target_os = "linux")]
|
||||
_mount_lease: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<AdaptivePermitSet> for ResourceIoUnit {
|
||||
fn from(permits: AdaptivePermitSet) -> Self {
|
||||
Self {
|
||||
permits: ResourceIoPermits::Set(permits),
|
||||
memory_budget: BackgroundMemoryBudget::current(),
|
||||
#[cfg(target_os = "linux")]
|
||||
_mount_lease: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ResourceIoUnit {
|
||||
pub(crate) fn memory_budget(&self) -> Option<Arc<BackgroundMemoryBudget>> {
|
||||
self.memory_budget.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn with_permits(mut self, permits: AdaptivePermitSet) -> Self {
|
||||
self.permits = ResourceIoPermits::Set(permits);
|
||||
self
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn with_memory_budget(mut self, budget: Arc<BackgroundMemoryBudget>) -> Self {
|
||||
self.memory_budget = Some(budget);
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn as_set(&self) -> Option<&AdaptivePermitSet> {
|
||||
match &self.permits {
|
||||
ResourceIoPermits::Single(_) => None,
|
||||
ResourceIoPermits::Set(permits) => Some(permits),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(crate) fn with_mount_lease(mut self, mount_lease: Arc<std::fs::File>) -> Self {
|
||||
// The selected disk's proc-fd paths must survive native jobs after its
|
||||
// async wrapper retires. Clones carry the same pinned mount generation.
|
||||
self._mount_lease = Some(mount_lease);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_resource_units(count: usize) -> (ResourceIoUnit, Vec<Arc<rustfs_concurrency::adaptive::AdaptiveAdmission>>) {
|
||||
use rustfs_concurrency::adaptive::{AdaptiveAdmission, permit_set::AdaptivePermitSetOwner};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
assert!((1..=2).contains(&count), "test single and layered disks");
|
||||
let gates: Vec<_> = (0..count).map(|_| AdaptiveAdmission::new(NonZeroU16::MIN, 1)).collect();
|
||||
let mut admitted: Vec<_> = gates.iter().map(|gate| gate.try_acquire().expect("test resource")).collect();
|
||||
let unit = if count == 1 {
|
||||
ResourceIoUnit::from(admitted.pop().expect("single resource"))
|
||||
} else {
|
||||
let owner = AdaptivePermitSetOwner::new(NonZeroU16::MIN, gates.clone()).expect("layered resource");
|
||||
ResourceIoUnit::from(owner.capture(&admitted.iter().collect::<Vec<_>>()).expect("admitted subset"))
|
||||
};
|
||||
(unit, gates)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
@@ -51,7 +163,7 @@ pub(crate) struct DiskIoContext {
|
||||
pub(crate) cancellation: CancellationToken,
|
||||
pub(crate) admission: Option<Arc<DiskIoAdmission>>,
|
||||
pub(crate) permit: Option<DiskIoPermit>,
|
||||
pub(crate) resource_unit: Option<AdaptivePermit>,
|
||||
pub(crate) resource_unit: Option<ResourceIoUnit>,
|
||||
}
|
||||
|
||||
/// Mark disk I/O issued by `future` as background work for local-disk admission.
|
||||
@@ -78,24 +190,46 @@ pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken,
|
||||
/// of a stalled native operation. Escaping buffers need their own memory leases.
|
||||
#[doc(hidden)]
|
||||
pub async fn with_background_resource_unit<F: Future>(unit: AdaptivePermit, future: F) -> F::Output {
|
||||
RESOURCE_IO_UNIT.scope(unit, future).await
|
||||
RESOURCE_IO_UNIT.scope(ResourceIoUnit::from(unit), future).await
|
||||
}
|
||||
|
||||
pub(crate) fn current_resource_unit() -> Option<AdaptivePermit> {
|
||||
RESOURCE_IO_UNIT.try_with(Clone::clone).ok().or_else(|| {
|
||||
DISK_IO_CONTEXT
|
||||
.try_with(|context| context.resource_unit.clone())
|
||||
.ok()
|
||||
.flatten()
|
||||
})
|
||||
/// Retain only the previously admitted physical leaves touched by this I/O.
|
||||
/// Projection of owned units never waits and may run under storage locks.
|
||||
/// Acquire the planned resources before locking; auxiliary leaves discovered
|
||||
/// later must use nonblocking admission. Do not pass unrelated parent resources
|
||||
/// to a native job that touches only some of its disks.
|
||||
#[doc(hidden)]
|
||||
pub async fn with_background_resource_set<F: Future>(units: AdaptivePermitSet, future: F) -> F::Output {
|
||||
RESOURCE_IO_UNIT.scope(ResourceIoUnit::from(units), future).await
|
||||
}
|
||||
|
||||
pub(crate) fn current_resource_unit() -> Option<ResourceIoUnit> {
|
||||
RESOURCE_IO_UNIT
|
||||
.try_with(Clone::clone)
|
||||
.ok()
|
||||
.or_else(|| {
|
||||
DISK_IO_CONTEXT
|
||||
.try_with(|context| context.resource_unit.clone())
|
||||
.ok()
|
||||
.flatten()
|
||||
})
|
||||
.or_else(remote::resources)
|
||||
}
|
||||
|
||||
pub(crate) fn has_resource_io_scope() -> bool {
|
||||
RESOURCE_IO_UNIT.try_with(|_| ()).is_ok()
|
||||
|| DISK_IO_CONTEXT
|
||||
.try_with(|context| context.resource_unit.is_some())
|
||||
.unwrap_or(false)
|
||||
|| remote::active()
|
||||
}
|
||||
|
||||
/// Transfer only resource ownership into an existing spawned task. Disk
|
||||
/// classification/permits are not inherited: changing those can introduce a
|
||||
/// wait behind a disk permit still held by the parent.
|
||||
pub(crate) async fn with_optional_resource_unit<F: Future>(unit: Option<AdaptivePermit>, future: F) -> F::Output {
|
||||
pub(crate) async fn with_optional_resource_unit<F: Future>(unit: Option<ResourceIoUnit>, future: F) -> F::Output {
|
||||
match unit {
|
||||
Some(unit) => with_background_resource_unit(unit, future).await,
|
||||
Some(unit) => RESOURCE_IO_UNIT.scope(unit, future).await,
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
@@ -233,7 +367,7 @@ impl DiskIoAdmission {
|
||||
self.foreground_with_resource(current_resource_unit())
|
||||
}
|
||||
|
||||
fn foreground_with_resource(self: &Arc<Self>, resource_unit: Option<AdaptivePermit>) -> DiskIoPermit {
|
||||
fn foreground_with_resource(self: &Arc<Self>, resource_unit: Option<ResourceIoUnit>) -> DiskIoPermit {
|
||||
self.foreground_active.fetch_add(1, Ordering::AcqRel);
|
||||
if self.background_waiters.load(Ordering::Acquire) > 0 {
|
||||
self.record_state();
|
||||
@@ -254,7 +388,7 @@ impl DiskIoAdmission {
|
||||
async fn background_with_resource(
|
||||
self: &Arc<Self>,
|
||||
cancellation: &CancellationToken,
|
||||
resource_unit: Option<AdaptivePermit>,
|
||||
resource_unit: Option<ResourceIoUnit>,
|
||||
) -> std::io::Result<DiskIoPermit> {
|
||||
let started_at = Instant::now();
|
||||
let ticket = {
|
||||
@@ -429,7 +563,7 @@ impl DiskIoPermit {
|
||||
struct DiskIoPermitLease {
|
||||
admission: Arc<DiskIoAdmission>,
|
||||
class: DiskIoClass,
|
||||
resource_unit: Option<AdaptivePermit>,
|
||||
resource_unit: Option<ResourceIoUnit>,
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_blocking_with_disk_io_permit<T, F>(
|
||||
@@ -450,6 +584,18 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
/// Run native filesystem work with the current resource ownership. Cancelling
|
||||
/// the async waiter cannot refund a unit while the native operation still runs.
|
||||
/// This does not acquire capacity or change foreground/background policy.
|
||||
#[doc(hidden)]
|
||||
pub fn spawn_blocking_with_resource_ownership<T, F>(operation: F) -> tokio::task::JoinHandle<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: FnOnce() -> T + Send + 'static,
|
||||
{
|
||||
spawn_blocking_with_disk_io_permit(None, operation)
|
||||
}
|
||||
|
||||
type AdmissionWait = Pin<Box<dyn Future<Output = io::Result<DiskIoPermit>> + Send>>;
|
||||
|
||||
impl Drop for DiskIoPermitLease {
|
||||
@@ -595,6 +741,63 @@ mod tests {
|
||||
use std::num::NonZeroU16;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
#[tokio::test]
|
||||
async fn physical_subset_survives_native_cancellation_without_retaining_unrelated_disks() {
|
||||
use rustfs_concurrency::adaptive::permit_set::AdaptivePermitSetOwner;
|
||||
|
||||
for disk_permit in [false, true] {
|
||||
let maximum = NonZeroU16::new(1).expect("one physical unit");
|
||||
let first = AdaptiveAdmission::new(maximum, 1);
|
||||
let second = AdaptiveAdmission::new(maximum, 1);
|
||||
let unrelated = AdaptiveAdmission::new(maximum, 1);
|
||||
let owner = AdaptivePermitSetOwner::new(maximum, vec![first.clone(), second.clone()]).expect("layered disk");
|
||||
let a = first.try_acquire().expect("first backing device");
|
||||
let b = second.try_acquire().expect("second backing device");
|
||||
let healthy = unrelated.try_acquire().expect("another disk in the parent operation");
|
||||
let subset = owner.capture(&[&a, &b]).expect("capture before storage work");
|
||||
drop((a, b, owner));
|
||||
let disk = DiskIoAdmission::new("physical-subset-native");
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
let _healthy = healthy;
|
||||
with_background_resource_set(subset, async {
|
||||
let permit = if disk_permit {
|
||||
Some(disk.background(&CancellationToken::new()).await.expect("logical disk permit"))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
spawn_blocking_with_disk_io_permit(permit, move || {
|
||||
started_tx.send(()).expect("native I/O started");
|
||||
release_rx.blocking_recv().expect("release native I/O");
|
||||
})
|
||||
.await
|
||||
.expect("native I/O completed");
|
||||
})
|
||||
.await;
|
||||
});
|
||||
started_rx.await.expect("native worker retains subset");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancel async waiter").is_cancelled());
|
||||
let active = [first.snapshot().active, second.snapshot().active, unrelated.snapshot().active];
|
||||
let healthy_available = unrelated.try_acquire().is_some();
|
||||
first.set_limit(0);
|
||||
second.set_limit(0);
|
||||
release_tx.send(()).expect("finish native work");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while first.snapshot().active != 0 || second.snapshot().active != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("both backing devices are returned after native completion");
|
||||
assert_eq!(active, [1, 1, 0], "only the native I/O's physical subset remains charged");
|
||||
assert!(healthy_available, "cancellation frees the independent disk immediately");
|
||||
assert!(first.try_acquire().is_none());
|
||||
assert!(second.try_acquire().is_none(), "old completions cannot reopen shrunken owners");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn resource_unit_outlives_cancelled_scope_until_native_io_finishes() {
|
||||
for permit_mode in 0..4 {
|
||||
|
||||
@@ -0,0 +1,547 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use bytes::Bytes;
|
||||
use std::io;
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
use std::time::Duration;
|
||||
use tokio::time::Instant;
|
||||
|
||||
static BACKGROUND_MEMORY: OnceLock<Arc<BackgroundMemoryBudget>> = OnceLock::new();
|
||||
|
||||
/// Process-wide payload storage retained by admitted background I/O. The
|
||||
/// existing resource sampler publishes limits; this owner starts no workers.
|
||||
#[derive(Debug)]
|
||||
pub struct BackgroundMemoryBudget {
|
||||
// This short critical section only changes counters and a deadline. It
|
||||
// never allocates, calls storage, or acquires another lock while held.
|
||||
state: Mutex<State>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct State {
|
||||
limit: u64,
|
||||
used: u64,
|
||||
valid_until: Instant,
|
||||
}
|
||||
|
||||
/// A coherent view of the leased cap and retained payload storage.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct BackgroundMemorySnapshot {
|
||||
/// Zero when the latest resource observation has expired.
|
||||
pub limit: u64,
|
||||
/// Existing storage remains charged even after the cap shrinks below it.
|
||||
pub used: u64,
|
||||
}
|
||||
|
||||
impl BackgroundMemoryBudget {
|
||||
/// Start closed until the existing resource sampler publishes a limit.
|
||||
pub fn new() -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
state: Mutex::new(State {
|
||||
limit: 0,
|
||||
used: 0,
|
||||
valid_until: Instant::now(),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// Install the sole process owner before starting adaptive background work.
|
||||
/// Returns false if an owner was already installed.
|
||||
pub fn install(self: &Arc<Self>) -> bool {
|
||||
BACKGROUND_MEMORY.set(self.clone()).is_ok()
|
||||
}
|
||||
|
||||
/// Return only the installed process memory policy; no node or disk ownership.
|
||||
/// No policy is installed in the default, non-adaptive startup path.
|
||||
pub fn current() -> Option<Arc<Self>> {
|
||||
BACKGROUND_MEMORY.get().cloned()
|
||||
}
|
||||
|
||||
/// Update admission without revoking storage retained by native tasks or
|
||||
/// readers. Missing observations must publish zero capacity.
|
||||
pub fn set_sampled_limit(&self, limit: u64, valid_for: Duration) {
|
||||
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
state.limit = limit;
|
||||
let now = Instant::now();
|
||||
state.valid_until = now.checked_add(valid_for).unwrap_or(now);
|
||||
}
|
||||
|
||||
/// Read the cap and retained byte count under the same short lock.
|
||||
pub fn snapshot(&self) -> BackgroundMemorySnapshot {
|
||||
let state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
BackgroundMemorySnapshot {
|
||||
limit: if Instant::now() < state.valid_until { state.limit } else { 0 },
|
||||
used: state.used,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reserve(self: &Arc<Self>, bytes: usize) -> io::Result<Reservation> {
|
||||
let bytes = u64::try_from(bytes).map_err(|_| io::Error::from(io::ErrorKind::OutOfMemory))?;
|
||||
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let used = state
|
||||
.used
|
||||
.checked_add(bytes)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::WouldBlock, BackgroundMemoryExhausted))?;
|
||||
if bytes != 0 && (Instant::now() >= state.valid_until || used > state.limit) {
|
||||
return Err(io::Error::new(io::ErrorKind::WouldBlock, BackgroundMemoryExhausted));
|
||||
}
|
||||
state.used = used;
|
||||
Ok(Reservation {
|
||||
owner: self.clone(),
|
||||
bytes,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct BackgroundMemoryExhausted;
|
||||
|
||||
impl std::fmt::Display for BackgroundMemoryExhausted {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str("background buffer memory budget exhausted")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for BackgroundMemoryExhausted {}
|
||||
|
||||
pub(crate) fn is_memory_pressure(error: &io::Error) -> bool {
|
||||
error.kind() == io::ErrorKind::OutOfMemory || error.get_ref().is_some_and(|inner| inner.is::<BackgroundMemoryExhausted>())
|
||||
}
|
||||
|
||||
pub(crate) struct Reservation {
|
||||
owner: Arc<BackgroundMemoryBudget>,
|
||||
bytes: u64,
|
||||
}
|
||||
|
||||
impl Reservation {
|
||||
pub(crate) fn belongs_to(&self, budget: &Arc<BackgroundMemoryBudget>) -> bool {
|
||||
Arc::ptr_eq(&self.owner, budget)
|
||||
}
|
||||
|
||||
pub(crate) fn wrap(bytes: Bytes, reservation: Option<Self>) -> Bytes {
|
||||
if bytes.is_empty() {
|
||||
drop(bytes);
|
||||
drop(reservation);
|
||||
return Bytes::new();
|
||||
}
|
||||
match reservation {
|
||||
Some(reservation) => Bytes::from_owner(BackgroundBytes {
|
||||
bytes,
|
||||
_reservation: reservation,
|
||||
}),
|
||||
None => bytes,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct BackgroundBytes {
|
||||
// Free the payload before refunding it, including after the last slice.
|
||||
bytes: Bytes,
|
||||
_reservation: Reservation,
|
||||
}
|
||||
|
||||
impl AsRef<[u8]> for BackgroundBytes {
|
||||
fn as_ref(&self) -> &[u8] {
|
||||
&self.bytes
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Reservation {
|
||||
fn drop(&mut self) {
|
||||
self.owner
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.used -= self.bytes;
|
||||
}
|
||||
}
|
||||
|
||||
/// Reusable initialized payload storage. The reservation is embedded in the
|
||||
/// existing native job state, with no per-read owner allocation or payload copy.
|
||||
pub(crate) struct BackgroundBuffer {
|
||||
// Field order releases the allocation before refunding its reservation.
|
||||
bytes: Box<[u8]>,
|
||||
reservation: Option<Reservation>,
|
||||
budget: Option<Arc<BackgroundMemoryBudget>>,
|
||||
}
|
||||
|
||||
impl BackgroundBuffer {
|
||||
pub(crate) fn new(budget: Option<Arc<BackgroundMemoryBudget>>) -> Self {
|
||||
Self {
|
||||
bytes: Box::default(),
|
||||
reservation: None,
|
||||
budget,
|
||||
}
|
||||
}
|
||||
|
||||
/// Call only after the previous contents have been consumed. Growth charges
|
||||
/// both allocations until the old buffer is freed, including after shrink.
|
||||
pub(crate) fn ensure_len(&mut self, length: usize) -> io::Result<()> {
|
||||
self.grow(length, 0)
|
||||
}
|
||||
|
||||
/// Preserve an initialized prefix when replacing storage. Callers keep
|
||||
/// `preserve` within both the existing allocation and the requested length.
|
||||
pub(crate) fn grow(&mut self, length: usize, preserve: usize) -> io::Result<()> {
|
||||
if self.bytes.len() >= length {
|
||||
return Ok(());
|
||||
}
|
||||
let reservation = self.budget.as_ref().map(|budget| budget.reserve(length)).transpose()?;
|
||||
let mut bytes = Vec::new();
|
||||
bytes
|
||||
.try_reserve_exact(length)
|
||||
.map_err(|_| io::Error::from(io::ErrorKind::OutOfMemory))?;
|
||||
bytes.resize(length, 0);
|
||||
bytes[..preserve].copy_from_slice(&self.bytes[..preserve]);
|
||||
self.bytes = bytes.into_boxed_slice();
|
||||
self.reservation = reservation;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Metadata is only a size hint: concurrent growth still reads through EOF.
|
||||
/// Every replacement is charged before copying the retained prefix.
|
||||
pub(crate) fn read_to_end(mut self, reader: &mut impl io::Read, hint: usize) -> io::Result<Bytes> {
|
||||
self.ensure_len(hint)?;
|
||||
let mut filled = 0;
|
||||
loop {
|
||||
if filled == self.bytes.len() {
|
||||
// Probe EOF without forcing a larger allocation for an exact fit.
|
||||
let mut probe = [0];
|
||||
match reader.read(&mut probe) {
|
||||
Ok(0) => break,
|
||||
Ok(_) => {
|
||||
let length = self.bytes.len().checked_mul(2).ok_or(io::ErrorKind::OutOfMemory)?.max(32);
|
||||
self.grow(length, filled)?;
|
||||
self.bytes[filled] = probe[0];
|
||||
filled += 1;
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
match reader.read(&mut self.bytes[filled..]) {
|
||||
Ok(0) => break,
|
||||
Ok(length) => filled += length,
|
||||
Err(error) if error.kind() == io::ErrorKind::Interrupted => continue,
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
Ok(self.take_bytes(filled))
|
||||
}
|
||||
|
||||
pub(crate) fn take_bytes(&mut self, length: usize) -> Bytes {
|
||||
let bytes = std::mem::take(&mut self.bytes);
|
||||
let reservation = self.reservation.take();
|
||||
let mut bytes = Bytes::from(bytes.into_vec());
|
||||
bytes.truncate(length);
|
||||
Reservation::wrap(bytes, reservation)
|
||||
}
|
||||
|
||||
pub(crate) fn memory_budget(&self) -> Option<Arc<BackgroundMemoryBudget>> {
|
||||
self.budget.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn as_mut_slice(&mut self) -> &mut [u8] {
|
||||
&mut self.bytes
|
||||
}
|
||||
|
||||
pub(crate) fn as_slice(&self) -> &[u8] {
|
||||
&self.bytes
|
||||
}
|
||||
}
|
||||
|
||||
/// Bounded encoded-frame storage for serializer output and raw read staging.
|
||||
/// Both paths reserve the actual allocation before initializing its bytes.
|
||||
pub struct BackgroundEncodeBuffer {
|
||||
buffer: BackgroundBuffer,
|
||||
filled: usize,
|
||||
maximum: usize,
|
||||
}
|
||||
|
||||
impl BackgroundEncodeBuffer {
|
||||
pub fn new(memory: Arc<BackgroundMemoryBudget>, maximum: usize) -> Self {
|
||||
Self {
|
||||
buffer: BackgroundBuffer::new(Some(memory)),
|
||||
filled: 0,
|
||||
maximum,
|
||||
}
|
||||
}
|
||||
|
||||
/// Borrow initialized raw-frame staging; decode only after a successful read.
|
||||
/// Staging is not encoded output: `into_bytes` remains empty for this path.
|
||||
pub fn prepare_read(&mut self, length: usize) -> io::Result<&mut [u8]> {
|
||||
if length > self.maximum {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"background encoded frame exceeds its size limit",
|
||||
));
|
||||
}
|
||||
self.buffer.ensure_len(length)?;
|
||||
self.filled = 0;
|
||||
Ok(&mut self.buffer.as_mut_slice()[..length])
|
||||
}
|
||||
|
||||
pub fn into_bytes(mut self) -> Bytes {
|
||||
self.buffer.take_bytes(self.filled)
|
||||
}
|
||||
}
|
||||
|
||||
impl io::Write for BackgroundEncodeBuffer {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
let required = self.filled.checked_add(bytes.len()).ok_or(io::ErrorKind::OutOfMemory)?;
|
||||
if required > self.maximum {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"background encoded frame exceeds its size limit",
|
||||
));
|
||||
}
|
||||
if required > self.buffer.as_slice().len() {
|
||||
let capacity = self
|
||||
.buffer
|
||||
.as_slice()
|
||||
.len()
|
||||
.saturating_mul(2)
|
||||
.max(32)
|
||||
.max(required)
|
||||
.min(self.maximum);
|
||||
self.buffer.grow(capacity, self.filled)?;
|
||||
}
|
||||
self.buffer.as_mut_slice()[self.filled..required].copy_from_slice(bytes);
|
||||
self.filled = required;
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn encoded_buffer_growth_retains_both_allocations_and_the_last_slice() {
|
||||
use std::io::Write;
|
||||
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(95, Duration::from_secs(30));
|
||||
let mut buffer = BackgroundEncodeBuffer::new(owner.clone(), 64);
|
||||
buffer.write_all(&[7; 32]).expect("initial allocation fits");
|
||||
assert_eq!(owner.snapshot().used, 32);
|
||||
|
||||
let error = buffer.write_all(&[8]).expect_err("old 32 plus replacement 64 exceed 95");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert!(is_memory_pressure(&error));
|
||||
assert_eq!(buffer.filled, 32);
|
||||
assert_eq!(buffer.buffer.as_slice(), &[7; 32]);
|
||||
assert_eq!(owner.snapshot().used, 32);
|
||||
|
||||
owner.set_sampled_limit(96, Duration::from_secs(30));
|
||||
buffer.write_all(&[8]).expect("old 32 plus replacement 64 fit 96");
|
||||
assert_eq!(owner.snapshot().used, 64);
|
||||
let bytes = buffer.into_bytes();
|
||||
assert_eq!(&bytes[..32], &[7; 32]);
|
||||
assert_eq!(bytes[32], 8);
|
||||
assert_eq!(bytes.len(), 33);
|
||||
let slice = bytes.slice(32..);
|
||||
drop(bytes);
|
||||
assert_eq!(slice.as_ref(), &[8]);
|
||||
assert_eq!(owner.snapshot().used, 64);
|
||||
drop(slice);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoded_buffer_checks_its_limit_before_reserving_storage() {
|
||||
use std::io::Write;
|
||||
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
for maximum in [0, 3] {
|
||||
let mut buffer = BackgroundEncodeBuffer::new(owner.clone(), maximum);
|
||||
buffer
|
||||
.write_all(&[])
|
||||
.expect("empty input needs no storage, including at limit zero");
|
||||
let error = buffer
|
||||
.write_all(&[7; 4])
|
||||
.expect_err("oversized input must fail before memory admission");
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(buffer.buffer.as_slice().is_empty());
|
||||
assert!(buffer.into_bytes().is_empty());
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn returned_bytes_keep_the_allocation_charged_through_slices_and_shrink() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(8, Duration::from_secs(30));
|
||||
let bytes = BackgroundBuffer::new(Some(owner.clone()))
|
||||
.read_to_end(&mut &b"abcdefgh"[..], 8)
|
||||
.expect("exact allocation fits without an EOF growth allocation");
|
||||
assert_eq!(bytes.as_ref(), b"abcdefgh");
|
||||
assert_eq!(owner.snapshot().used, 8);
|
||||
let slice = bytes.slice(2..6);
|
||||
owner.set_sampled_limit(0, Duration::ZERO);
|
||||
drop(bytes);
|
||||
assert_eq!(slice.as_ref(), b"cdef");
|
||||
assert_eq!(owner.snapshot().used, 8, "the final slice still owns the complete allocation");
|
||||
assert!(owner.reserve(1).is_err());
|
||||
drop(slice);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whole_file_size_hint_never_truncates_growth_or_exposes_unused_bytes() {
|
||||
for (hint, retained) in [(0, 32), (4, 32), (8, 8), (16, 16)] {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(36, Duration::from_secs(30));
|
||||
let bytes = BackgroundBuffer::new(Some(owner.clone()))
|
||||
.read_to_end(&mut &b"abcdefgh"[..], hint)
|
||||
.expect("read through EOF independently of the metadata hint");
|
||||
assert_eq!(bytes.as_ref(), b"abcdefgh", "hint={hint}");
|
||||
assert_eq!(owner.snapshot().used, retained, "charge allocated capacity, not visible length");
|
||||
drop(bytes);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(16, Duration::from_secs(30));
|
||||
let bytes = BackgroundBuffer::new(Some(owner.clone()))
|
||||
.read_to_end(&mut std::io::empty(), 16)
|
||||
.expect("file shrank to empty");
|
||||
assert!(bytes.is_empty());
|
||||
assert_eq!(owner.snapshot().used, 0, "empty results free unused storage before refund");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserving_growth_keeps_existing_payload_on_denial_and_reuse() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(11, Duration::from_secs(30));
|
||||
let mut buffer = BackgroundBuffer::new(Some(owner.clone()));
|
||||
buffer.ensure_len(4).expect("allocate initial payload");
|
||||
buffer.as_mut_slice().copy_from_slice(b"data");
|
||||
|
||||
let error = buffer
|
||||
.grow(8, 4)
|
||||
.expect_err("old and replacement allocations need twelve bytes");
|
||||
assert!(is_memory_pressure(&error));
|
||||
assert_eq!(buffer.as_slice(), b"data");
|
||||
assert_eq!(owner.snapshot().used, 4);
|
||||
|
||||
owner.set_sampled_limit(12, Duration::from_secs(30));
|
||||
buffer.grow(8, 4).expect("preserve the payload after admission");
|
||||
assert_eq!(buffer.as_slice(), b"data\0\0\0\0");
|
||||
assert_eq!(owner.snapshot().used, 8);
|
||||
owner.set_sampled_limit(0, Duration::ZERO);
|
||||
buffer.grow(8, 4).expect("reuse retained storage after expiry");
|
||||
assert_eq!(buffer.as_slice(), b"data\0\0\0\0");
|
||||
|
||||
let bytes = buffer.take_bytes(4);
|
||||
drop(buffer);
|
||||
assert_eq!(bytes.as_ref(), b"data");
|
||||
assert_eq!(owner.snapshot().used, 8);
|
||||
drop(bytes);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn whole_file_growth_denial_and_source_failure_refund_storage() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(35, Duration::from_secs(30));
|
||||
let error = BackgroundBuffer::new(Some(owner.clone()))
|
||||
.read_to_end(&mut &b"abcdefgh"[..], 4)
|
||||
.expect_err("old four plus new thirty-two bytes exceed capacity");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
|
||||
struct FailedRead(u8);
|
||||
impl io::Read for FailedRead {
|
||||
fn read(&mut self, bytes: &mut [u8]) -> io::Result<usize> {
|
||||
self.0 += 1;
|
||||
match self.0 {
|
||||
1 => Err(io::ErrorKind::Interrupted.into()),
|
||||
2 => {
|
||||
bytes[..2].copy_from_slice(b"ab");
|
||||
Ok(2)
|
||||
}
|
||||
_ => Err(io::ErrorKind::PermissionDenied.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
let error = BackgroundBuffer::new(Some(owner.clone()))
|
||||
.read_to_end(&mut FailedRead(0), 8)
|
||||
.expect_err("failure after partial output is not successful EOF");
|
||||
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn shrink_and_expiry_keep_retained_storage_charged() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
assert!(owner.reserve(1).is_err());
|
||||
owner.set_sampled_limit(16, Duration::from_secs(3));
|
||||
let mut buffer = BackgroundBuffer::new(Some(owner.clone()));
|
||||
buffer.ensure_len(8).expect("initial allocation");
|
||||
let pointer = buffer.as_slice().as_ptr();
|
||||
owner.set_sampled_limit(4, Duration::from_secs(3));
|
||||
buffer.ensure_len(8).expect("existing storage can be reused after shrink");
|
||||
assert_eq!(buffer.as_slice().as_ptr(), pointer);
|
||||
assert_eq!(owner.snapshot(), BackgroundMemorySnapshot { limit: 4, used: 8 });
|
||||
assert_eq!(buffer.ensure_len(9).expect_err("growth must be denied").kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(owner.snapshot().used, 8, "failed growth preserves the existing reservation");
|
||||
tokio::time::advance(Duration::from_secs(3)).await;
|
||||
assert_eq!(owner.snapshot(), BackgroundMemorySnapshot { limit: 0, used: 8 });
|
||||
assert!(owner.reserve(1).is_err(), "an expired sampler cannot grant memory");
|
||||
drop(buffer);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn growth_accounts_for_both_allocations_and_refunds_failed_attempts() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(12, Duration::from_secs(3));
|
||||
let mut buffer = BackgroundBuffer::new(Some(owner.clone()));
|
||||
buffer.ensure_len(4).expect("first allocation");
|
||||
buffer.as_mut_slice().copy_from_slice(b"data");
|
||||
assert!(buffer.ensure_len(9).is_err(), "four old plus nine new bytes exceed the cap");
|
||||
assert_eq!(buffer.as_slice(), b"data");
|
||||
buffer.ensure_len(8).expect("four old plus eight new bytes fit");
|
||||
assert_eq!(owner.snapshot().used, 8);
|
||||
assert_eq!(buffer.as_slice(), &[0; 8]);
|
||||
drop(buffer);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn concurrent_reservations_share_one_cap() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(1024, Duration::from_secs(30));
|
||||
let ready = Arc::new(std::sync::Barrier::new(17));
|
||||
let release = Arc::new(std::sync::Barrier::new(17));
|
||||
// Start every worker before the first barrier; lazy spawning would deadlock.
|
||||
let mut workers = Vec::with_capacity(16);
|
||||
for _ in 0..16 {
|
||||
let owner = owner.clone();
|
||||
let ready = ready.clone();
|
||||
let release = release.clone();
|
||||
workers.push(std::thread::spawn(move || {
|
||||
let reservation = owner.reserve(128).ok();
|
||||
ready.wait();
|
||||
release.wait();
|
||||
reservation.is_some()
|
||||
}));
|
||||
}
|
||||
ready.wait();
|
||||
let retained = owner.snapshot().used;
|
||||
release.wait();
|
||||
let granted = workers
|
||||
.into_iter()
|
||||
.map(|worker| worker.join().expect("reservation worker"))
|
||||
.filter(|granted| *granted)
|
||||
.count();
|
||||
assert_eq!(retained, 1024);
|
||||
assert_eq!(granted, 8);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,443 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::fs::File;
|
||||
use std::io::{self, Read};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const MAX_TOPOLOGY_NODES: usize = 256;
|
||||
const MAX_TOPOLOGY_DEPTH: usize = 32;
|
||||
const MAX_ATTRIBUTE_BYTES: u64 = 128;
|
||||
|
||||
/// Kernel device numbers can be reused after removal. `diskseq` distinguishes
|
||||
/// those incarnations; this identity does not authorize storage mutations.
|
||||
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
pub(super) struct PhysicalDeviceIdentity {
|
||||
pub(super) major: u32,
|
||||
pub(super) minor: u32,
|
||||
pub(super) generation: u64,
|
||||
}
|
||||
|
||||
fn invalid(message: &'static str) -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, message)
|
||||
}
|
||||
|
||||
pub(super) fn attribute(path: &Path) -> io::Result<String> {
|
||||
let mut value = String::new();
|
||||
File::open(path)?.take(MAX_ATTRIBUTE_BYTES + 1).read_to_string(&mut value)?;
|
||||
if value.len() > MAX_ATTRIBUTE_BYTES as usize {
|
||||
return Err(invalid("physical device attribute exceeds its bound"));
|
||||
}
|
||||
Ok(value.trim().to_owned())
|
||||
}
|
||||
|
||||
fn number(value: &str) -> io::Result<u64> {
|
||||
if value.is_empty() || !value.bytes().all(|byte| byte.is_ascii_digit()) {
|
||||
return Err(invalid("invalid physical device number"));
|
||||
}
|
||||
value.parse().map_err(|_| invalid("physical device number overflow"))
|
||||
}
|
||||
|
||||
fn identity(path: &Path) -> io::Result<PhysicalDeviceIdentity> {
|
||||
let generation = number(&attribute(&path.join("diskseq"))?)?;
|
||||
if generation == 0 {
|
||||
return Err(invalid("physical device generation is missing"));
|
||||
}
|
||||
let device = attribute(&path.join("dev"))?;
|
||||
let (major, minor) = device
|
||||
.split_once(':')
|
||||
.ok_or_else(|| invalid("invalid physical device identity"))?;
|
||||
let major = u32::try_from(number(major)?).map_err(|_| invalid("physical major number overflow"))?;
|
||||
let minor = u32::try_from(number(minor)?).map_err(|_| invalid("physical minor number overflow"))?;
|
||||
if number(&attribute(&path.join("diskseq"))?)? != generation {
|
||||
return Err(invalid("physical device changed during discovery"));
|
||||
}
|
||||
Ok(PhysicalDeviceIdentity {
|
||||
major,
|
||||
minor,
|
||||
generation,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Debug, Eq, PartialEq)]
|
||||
enum NodeTopology {
|
||||
Partition {
|
||||
device: String,
|
||||
number: u64,
|
||||
parent: PathBuf,
|
||||
},
|
||||
Whole {
|
||||
identity: PhysicalDeviceIdentity,
|
||||
children: Vec<PathBuf>,
|
||||
},
|
||||
}
|
||||
|
||||
fn canonical_beneath(path: &Path, root: &Path) -> io::Result<PathBuf> {
|
||||
let path = path.canonicalize()?;
|
||||
if !path.starts_with(root) {
|
||||
return Err(invalid("physical device topology escapes sysfs"));
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
fn node_topology(path: &Path, root: &Path) -> io::Result<NodeTopology> {
|
||||
if canonical_beneath(path, root)? != path {
|
||||
return Err(invalid("physical device path changed during discovery"));
|
||||
}
|
||||
match attribute(&path.join("partition")) {
|
||||
Ok(partition) => {
|
||||
let number = number(&partition)?;
|
||||
if number == 0 {
|
||||
return Err(invalid("physical partition number is missing"));
|
||||
}
|
||||
let parent = path.parent().ok_or_else(|| invalid("physical partition has no parent"))?;
|
||||
return Ok(NodeTopology::Partition {
|
||||
device: attribute(&path.join("dev"))?,
|
||||
number,
|
||||
parent: canonical_beneath(parent, root)?,
|
||||
});
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
let identity = identity(path)?;
|
||||
let mut children = Vec::new();
|
||||
for entry in std::fs::read_dir(path.join("slaves"))? {
|
||||
if children.len() >= MAX_TOPOLOGY_NODES {
|
||||
return Err(invalid("physical device topology fanout exceeds its bound"));
|
||||
}
|
||||
children.push(canonical_beneath(&entry?.path(), root)?);
|
||||
}
|
||||
children.sort_unstable();
|
||||
children.dedup();
|
||||
Ok(NodeTopology::Whole { identity, children })
|
||||
}
|
||||
|
||||
struct TopologySnapshot {
|
||||
root: PathBuf,
|
||||
entry: PathBuf,
|
||||
first: PathBuf,
|
||||
nodes: BTreeMap<PathBuf, NodeTopology>,
|
||||
leaves: BTreeSet<PhysicalDeviceIdentity>,
|
||||
}
|
||||
|
||||
impl TopologySnapshot {
|
||||
fn discover(sysfs_root: &Path, major: u64, minor: u64) -> io::Result<Self> {
|
||||
let root = sysfs_root.canonicalize()?;
|
||||
let entry = root.join(format!("dev/block/{major}:{minor}"));
|
||||
let first = canonical_beneath(&entry, &root)?;
|
||||
if attribute(&first.join("dev"))? != format!("{major}:{minor}") {
|
||||
return Err(invalid("physical device entry does not match its device number"));
|
||||
}
|
||||
let mut stack = vec![(first.clone(), 0, false)];
|
||||
let mut active = BTreeSet::new();
|
||||
let mut nodes = BTreeMap::new();
|
||||
let mut leaves = BTreeMap::<PhysicalDeviceIdentity, PathBuf>::new();
|
||||
while let Some((path, depth, exiting)) = stack.pop() {
|
||||
if exiting {
|
||||
active.remove(&path);
|
||||
continue;
|
||||
}
|
||||
if depth > MAX_TOPOLOGY_DEPTH {
|
||||
return Err(invalid("physical device topology is too deep"));
|
||||
}
|
||||
if active.contains(&path) {
|
||||
return Err(invalid("physical device topology contains a cycle"));
|
||||
}
|
||||
if nodes.contains_key(&path) {
|
||||
continue;
|
||||
}
|
||||
if nodes.len() >= MAX_TOPOLOGY_NODES {
|
||||
return Err(invalid("physical device topology exceeds its node bound"));
|
||||
}
|
||||
let topology = node_topology(&path, &root)?;
|
||||
active.insert(path.clone());
|
||||
stack.push((path.clone(), depth, true));
|
||||
match &topology {
|
||||
NodeTopology::Partition { parent, .. } => stack.push((parent.clone(), depth + 1, false)),
|
||||
NodeTopology::Whole { identity, children } => {
|
||||
if stack.len() + children.len() > MAX_TOPOLOGY_NODES * 2 {
|
||||
return Err(invalid("physical device topology fanout exceeds its bound"));
|
||||
}
|
||||
for child in children {
|
||||
stack.push((child.clone(), depth + 1, false));
|
||||
}
|
||||
if children.is_empty()
|
||||
&& let Some(previous) = leaves.insert(*identity, path.clone())
|
||||
&& previous != path
|
||||
{
|
||||
return Err(invalid("physical device identity is ambiguous"));
|
||||
}
|
||||
}
|
||||
}
|
||||
nodes.insert(path, topology);
|
||||
}
|
||||
if leaves.is_empty() {
|
||||
return Err(invalid("physical device topology has no leaves"));
|
||||
}
|
||||
Ok(Self {
|
||||
root,
|
||||
entry,
|
||||
first,
|
||||
nodes,
|
||||
leaves: leaves.into_keys().collect(),
|
||||
})
|
||||
}
|
||||
|
||||
fn validate(&self) -> io::Result<()> {
|
||||
if canonical_beneath(&self.entry, &self.root)? != self.first {
|
||||
return Err(invalid("physical device entry changed during discovery"));
|
||||
}
|
||||
for (path, expected) in &self.nodes {
|
||||
if &node_topology(path, &self.root)? != expected {
|
||||
return Err(invalid("physical device topology changed during discovery"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve a bounded sysfs topology once, before installing an admission owner.
|
||||
/// Missing topology or generation evidence is unknown, never a name-based owner.
|
||||
/// Validate intermediate mappings too: stable leaves alone do not prove that a
|
||||
/// device-mapper table still points to those leaves. This is an observation, not
|
||||
/// a kernel topology lock; the admission owner must expire and refresh it.
|
||||
pub(super) fn resolve_physical_devices(sysfs_root: &Path, major: u64, minor: u64) -> io::Result<Vec<PhysicalDeviceIdentity>> {
|
||||
let snapshot = TopologySnapshot::discover(sysfs_root, major, minor)?;
|
||||
snapshot.validate()?;
|
||||
Ok(snapshot.leaves.into_iter().collect())
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::os::unix::fs::symlink;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
struct Fixture(PathBuf);
|
||||
|
||||
impl Fixture {
|
||||
fn new() -> Self {
|
||||
static NEXT: AtomicU64 = AtomicU64::new(0);
|
||||
let stamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.expect("clock")
|
||||
.as_nanos();
|
||||
let root = std::env::temp_dir().join(format!(
|
||||
"physical-identity-{}-{stamp}-{}",
|
||||
std::process::id(),
|
||||
NEXT.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
std::fs::create_dir(&root).expect("exclusive test root");
|
||||
std::fs::create_dir_all(root.join("sys/dev/block")).expect("sysfs fixture");
|
||||
Self(root)
|
||||
}
|
||||
|
||||
fn root(&self) -> PathBuf {
|
||||
self.0.join("sys")
|
||||
}
|
||||
|
||||
fn device(&self, name: &str, major: u32, minor: u32, generation: u64) -> PathBuf {
|
||||
let path = self.root().join("devices").join(name);
|
||||
std::fs::create_dir_all(path.join("slaves")).expect("device fixture");
|
||||
std::fs::write(path.join("dev"), format!("{major}:{minor}\n")).expect("device number");
|
||||
std::fs::write(path.join("diskseq"), format!("{generation}\n")).expect("device generation");
|
||||
symlink(&path, self.root().join(format!("dev/block/{major}:{minor}"))).expect("sysfs link");
|
||||
path
|
||||
}
|
||||
|
||||
fn link(&self, parent: &Path, child: &Path, name: &str) {
|
||||
symlink(child, parent.join("slaves").join(name)).expect("slave link");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Fixture {
|
||||
fn drop(&mut self) {
|
||||
std::fs::remove_dir_all(&self.0).expect("remove owned fixture");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partitions_share_the_leaf_generation_and_replacement_changes_identity() {
|
||||
let fixture = Fixture::new();
|
||||
let leaf = fixture.device("sda", 8, 0, 41);
|
||||
let first = fixture.device("sda/sda1", 8, 1, 41);
|
||||
let second = fixture.device("sda/sda2", 8, 2, 41);
|
||||
std::fs::write(first.join("partition"), "1\n").expect("first partition");
|
||||
std::fs::write(second.join("partition"), "2\n").expect("second partition");
|
||||
let old = resolve_physical_devices(&fixture.root(), 8, 1).expect("first leaf");
|
||||
assert_eq!(old, resolve_physical_devices(&fixture.root(), 8, 2).expect("alias leaf"));
|
||||
assert_eq!(
|
||||
old,
|
||||
vec![PhysicalDeviceIdentity {
|
||||
major: 8,
|
||||
minor: 0,
|
||||
generation: 41
|
||||
}]
|
||||
);
|
||||
std::fs::write(leaf.join("diskseq"), "42\n").expect("replacement generation");
|
||||
let new = resolve_physical_devices(&fixture.root(), 8, 1).expect("replacement leaf");
|
||||
assert_ne!(old, new);
|
||||
assert_eq!(new[0].generation, 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overlapping_stacks_deduplicate_shared_physical_leaves() {
|
||||
let fixture = Fixture::new();
|
||||
let a = fixture.device("a", 8, 0, 1);
|
||||
let b = fixture.device("b", 8, 16, 2);
|
||||
let inner = fixture.device("dm-0", 253, 0, 3);
|
||||
let outer = fixture.device("dm-1", 253, 1, 4);
|
||||
fixture.link(&inner, &a, "a");
|
||||
fixture.link(&inner, &b, "b");
|
||||
fixture.link(&outer, &inner, "dm-0");
|
||||
fixture.link(&outer, &a, "a-again");
|
||||
assert_eq!(
|
||||
resolve_physical_devices(&fixture.root(), 253, 0).expect("inner"),
|
||||
resolve_physical_devices(&fixture.root(), 253, 1).expect("outer")
|
||||
);
|
||||
assert_eq!(resolve_physical_devices(&fixture.root(), 253, 1).expect("leaves").len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_or_invalid_generation_never_falls_back_to_a_device_name() {
|
||||
let fixture = Fixture::new();
|
||||
let leaf = fixture.device("a", 8, 0, 1);
|
||||
for value in ["0", "-1", "18446744073709551616", "+1"] {
|
||||
std::fs::write(leaf.join("diskseq"), value).expect("bad generation");
|
||||
assert!(resolve_physical_devices(&fixture.root(), 8, 0).is_err(), "accepted {value}");
|
||||
}
|
||||
std::fs::remove_file(leaf.join("diskseq")).expect("missing generation");
|
||||
assert_eq!(
|
||||
resolve_physical_devices(&fixture.root(), 8, 0)
|
||||
.expect_err("unknown generation")
|
||||
.kind(),
|
||||
io::ErrorKind::NotFound
|
||||
);
|
||||
assert!(resolve_physical_devices(&fixture.root(), 99, 99).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_cycles_and_escaping_links_are_rejected() {
|
||||
let fixture = Fixture::new();
|
||||
let a = fixture.device("a", 8, 0, 1);
|
||||
let b = fixture.device("b", 8, 16, 2);
|
||||
fixture.link(&a, &b, "b");
|
||||
fixture.link(&b, &a, "a");
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 8, 0)
|
||||
.expect_err("cycle")
|
||||
.to_string()
|
||||
.contains("cycle")
|
||||
);
|
||||
std::fs::remove_file(b.join("slaves/a")).expect("remove cycle edge");
|
||||
fixture.link(&b, &fixture.0, "outside");
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 8, 0)
|
||||
.expect_err("escape")
|
||||
.to_string()
|
||||
.contains("escapes")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn intermediate_mapping_changes_are_rejected_even_when_old_leaves_survive() {
|
||||
let fixture = Fixture::new();
|
||||
let a = fixture.device("a", 8, 0, 1);
|
||||
let b = fixture.device("b", 8, 16, 2);
|
||||
let mapper = fixture.device("dm-0", 253, 0, 3);
|
||||
fixture.link(&mapper, &a, "backing");
|
||||
let snapshot = TopologySnapshot::discover(&fixture.root(), 253, 0).expect("initial mapping");
|
||||
snapshot.validate().expect("unchanged topology");
|
||||
std::fs::remove_file(mapper.join("slaves/backing")).expect("replace mapping");
|
||||
fixture.link(&mapper, &b, "backing");
|
||||
assert_eq!(identity(&a).expect("old leaf still exists").generation, 1);
|
||||
assert!(snapshot.validate().is_err(), "stable old leaves cannot certify a changed mapper");
|
||||
let current = resolve_physical_devices(&fixture.root(), 253, 0).expect("new complete mapping");
|
||||
assert_eq!(current, vec![identity(&b).expect("new leaf")]);
|
||||
let snapshot = TopologySnapshot::discover(&fixture.root(), 253, 0).expect("new mapping");
|
||||
std::fs::write(mapper.join("diskseq"), "4\n").expect("replace mapper incarnation");
|
||||
assert!(
|
||||
snapshot.validate().is_err(),
|
||||
"an intermediate generation change also invalidates discovery"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entry_and_partition_changes_are_not_hidden_by_an_unchanged_parent() {
|
||||
let fixture = Fixture::new();
|
||||
fixture.device("a", 8, 0, 1);
|
||||
let partition = fixture.device("a/p1", 8, 1, 1);
|
||||
std::fs::write(partition.join("partition"), "1\n").expect("partition index");
|
||||
let snapshot = TopologySnapshot::discover(&fixture.root(), 8, 1).expect("partition topology");
|
||||
std::fs::write(partition.join("partition"), "2\n").expect("changed partition");
|
||||
assert!(snapshot.validate().is_err());
|
||||
let b = fixture.device("b", 8, 16, 2);
|
||||
let snapshot = TopologySnapshot::discover(&fixture.root(), 8, 0).expect("first entry");
|
||||
let entry = fixture.root().join("dev/block/8:0");
|
||||
std::fs::remove_file(&entry).expect("replace entry");
|
||||
symlink(&b, &entry).expect("changed entry target");
|
||||
assert!(snapshot.validate().is_err());
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 8, 0).is_err(),
|
||||
"entry device numbers must match"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_slave_directory_is_unknown_not_a_physical_leaf() {
|
||||
let fixture = Fixture::new();
|
||||
let mapper = fixture.device("dm-0", 253, 0, 1);
|
||||
std::fs::remove_dir(mapper.join("slaves")).expect("missing topology evidence");
|
||||
assert!(resolve_physical_devices(&fixture.root(), 253, 0).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_depth_and_attribute_sizes_are_bounded() {
|
||||
let fixture = Fixture::new();
|
||||
let first = fixture.device("d0", 253, 0, 1);
|
||||
let mut previous = first.clone();
|
||||
for index in 1..=MAX_TOPOLOGY_DEPTH + 1 {
|
||||
let child = fixture.device(&format!("d{index}"), 253, u32::try_from(index).expect("bounded index"), 1);
|
||||
fixture.link(&previous, &child, "next");
|
||||
previous = child;
|
||||
}
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 253, 0)
|
||||
.expect_err("depth bound")
|
||||
.to_string()
|
||||
.contains("deep")
|
||||
);
|
||||
std::fs::remove_file(first.join("slaves/next")).expect("make first node a leaf");
|
||||
std::fs::write(first.join("diskseq"), "1".repeat(129)).expect("oversized attribute");
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 253, 0)
|
||||
.expect_err("attribute bound")
|
||||
.to_string()
|
||||
.contains("attribute")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_node_bound_passes_and_one_more_node_is_rejected() {
|
||||
let fixture = Fixture::new();
|
||||
let root = fixture.device("root", 253, 0, 1);
|
||||
for index in 0..MAX_TOPOLOGY_NODES - 1 {
|
||||
let leaf = fixture.device(&format!("leaf-{index}"), 8, u32::try_from(index).expect("bounded index"), 2);
|
||||
fixture.link(&root, &leaf, &format!("leaf-{index}"));
|
||||
}
|
||||
assert_eq!(
|
||||
resolve_physical_devices(&fixture.root(), 253, 0)
|
||||
.expect("exact node bound")
|
||||
.len(),
|
||||
MAX_TOPOLOGY_NODES - 1
|
||||
);
|
||||
let extra = fixture.device("extra", 9, 0, 3);
|
||||
fixture.link(&root, &extra, "extra");
|
||||
assert!(
|
||||
resolve_physical_devices(&fixture.root(), 253, 0).is_err(),
|
||||
"reject a graph above the node bound"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,268 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::io;
|
||||
use std::num::NonZeroU16;
|
||||
use std::sync::{Arc, Weak};
|
||||
use std::time::Duration;
|
||||
|
||||
use rustfs_concurrency::adaptive::AdaptiveAdmission;
|
||||
#[cfg(test)]
|
||||
use rustfs_concurrency::adaptive::{AdaptiveAdmissionSnapshot, AdaptivePermit};
|
||||
|
||||
use super::physical_identity::PhysicalDeviceIdentity;
|
||||
|
||||
const MAX_PHYSICAL_OWNERS: usize = 256;
|
||||
|
||||
/// Resolve aliases outside object work. The sampling owner serializes changes;
|
||||
/// attached handles and native permits never consult this registry.
|
||||
pub(super) struct PhysicalOwnerRegistry {
|
||||
maximum: NonZeroU16,
|
||||
owners: BTreeMap<PhysicalDeviceIdentity, Weak<AdaptiveAdmission>>,
|
||||
}
|
||||
|
||||
/// One attachment to an existing physical capacity owner, not a reservation.
|
||||
/// The gate itself owns permit slots, so native permit clones keep a registry
|
||||
/// entry alive even after the last logical disk attachment disappears.
|
||||
pub(super) struct PhysicalDeviceAdmission {
|
||||
#[cfg(test)]
|
||||
identity: PhysicalDeviceIdentity,
|
||||
gate: Arc<AdaptiveAdmission>,
|
||||
}
|
||||
|
||||
impl PhysicalOwnerRegistry {
|
||||
pub(super) fn new(maximum: NonZeroU16) -> Self {
|
||||
Self {
|
||||
maximum,
|
||||
owners: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Old topology descriptors may still reference an unused gate. Close it
|
||||
/// before reclaiming its registry slot; only admitted I/O keeps an absent
|
||||
/// incarnation alive, and its final release still reaches the same owner.
|
||||
pub(super) fn retire_absent(&mut self, current: &BTreeSet<PhysicalDeviceIdentity>) {
|
||||
self.owners.retain(|identity, owner| {
|
||||
let Some(gate) = owner.upgrade() else {
|
||||
return false;
|
||||
};
|
||||
if current.contains(identity) {
|
||||
return true;
|
||||
}
|
||||
gate.set_sampled_limit(0, Duration::ZERO);
|
||||
gate.snapshot().active != 0
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn bind(&mut self, identities: &[PhysicalDeviceIdentity]) -> io::Result<Vec<PhysicalDeviceAdmission>> {
|
||||
if identities.is_empty() || identities.len() > MAX_PHYSICAL_OWNERS || identities.iter().any(|id| id.generation == 0) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid physical capacity identities"));
|
||||
}
|
||||
let identities = identities.iter().copied().collect::<BTreeSet<_>>();
|
||||
self.owners.retain(|_, owner| owner.strong_count() != 0);
|
||||
let new_owners = identities
|
||||
.iter()
|
||||
.filter(|identity| !self.owners.contains_key(identity))
|
||||
.count();
|
||||
if new_owners > MAX_PHYSICAL_OWNERS - self.owners.len() {
|
||||
return Err(io::Error::new(io::ErrorKind::WouldBlock, "physical capacity registry is full"));
|
||||
}
|
||||
Ok(identities
|
||||
.into_iter()
|
||||
.map(|identity| {
|
||||
let gate = self.owners.get(&identity).and_then(Weak::upgrade).unwrap_or_else(|| {
|
||||
let gate = AdaptiveAdmission::new(self.maximum, 0);
|
||||
self.owners.insert(identity, Arc::downgrade(&gate));
|
||||
gate
|
||||
});
|
||||
PhysicalDeviceAdmission {
|
||||
#[cfg(test)]
|
||||
identity,
|
||||
gate,
|
||||
}
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Only the sampling owner publishes capacity. New or expired identities
|
||||
/// remain closed until a fresh observation establishes a bounded limit.
|
||||
#[cfg(test)]
|
||||
pub(super) fn publish(&mut self, identity: PhysicalDeviceIdentity, limit: u32, valid_for: Duration) -> bool {
|
||||
let Some(gate) = self.owners.get(&identity).and_then(Weak::upgrade) else {
|
||||
return false;
|
||||
};
|
||||
gate.set_sampled_limit(limit, valid_for);
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl PhysicalDeviceAdmission {
|
||||
pub(super) fn gate(&self) -> &Arc<AdaptiveAdmission> {
|
||||
&self.gate
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn identity(&self) -> PhysicalDeviceIdentity {
|
||||
self.identity
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn snapshot(&self) -> AdaptiveAdmissionSnapshot {
|
||||
self.gate.snapshot()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn try_acquire(&self) -> Option<AdaptivePermit> {
|
||||
self.gate.try_acquire()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn device(minor: u32, generation: u64) -> PhysicalDeviceIdentity {
|
||||
PhysicalDeviceIdentity {
|
||||
major: 259,
|
||||
minor,
|
||||
generation,
|
||||
}
|
||||
}
|
||||
|
||||
fn registry() -> PhysicalOwnerRegistry {
|
||||
PhysicalOwnerRegistry::new(NonZeroU16::new(2).expect("two units"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn aliases_and_overlapping_stacks_share_one_physical_capacity() {
|
||||
let mut registry = registry();
|
||||
let a = device(1, 11);
|
||||
let b = device(2, 12);
|
||||
let single = registry.bind(&[a]).expect("single device");
|
||||
let stack = registry.bind(&[b, a, a]).expect("overlapping stack");
|
||||
assert_eq!(stack.len(), 2, "duplicates cannot reserve additional units");
|
||||
assert_eq!(stack[0].identity(), a);
|
||||
assert!(single[0].try_acquire().is_none(), "unobserved capacity is closed");
|
||||
assert!(registry.publish(a, 1, Duration::from_secs(60)));
|
||||
let held = single[0].try_acquire().expect("physical unit");
|
||||
assert!(stack[0].try_acquire().is_none(), "the alias cannot double the capacity");
|
||||
assert_eq!(stack[0].snapshot().active, 1);
|
||||
drop(held);
|
||||
assert!(stack[0].try_acquire().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_tail_keeps_the_owner_alive_after_all_attachments_are_gone() {
|
||||
let mut registry = registry();
|
||||
let id = device(1, 11);
|
||||
let attachment = registry.bind(&[id]).expect("logical disk");
|
||||
assert!(registry.publish(id, 1, Duration::from_secs(60)));
|
||||
let waiter = attachment[0].try_acquire().expect("admitted work");
|
||||
let native = waiter.clone();
|
||||
drop(waiter);
|
||||
drop(attachment);
|
||||
let reconnected = registry.bind(&[id]).expect("reconnect during native I/O");
|
||||
assert_eq!(reconnected[0].snapshot().active, 1);
|
||||
assert!(reconnected[0].try_acquire().is_none());
|
||||
drop(native);
|
||||
assert!(reconnected[0].try_acquire().is_some(), "only the final native release returns capacity");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replacement_generation_does_not_receive_old_native_refunds() {
|
||||
let mut registry = registry();
|
||||
let old_id = device(1, 11);
|
||||
let new_id = device(1, 12);
|
||||
let old = registry.bind(&[old_id]).expect("old incarnation");
|
||||
assert!(registry.publish(old_id, 1, Duration::from_secs(60)));
|
||||
let native = old[0].try_acquire().expect("old native I/O");
|
||||
drop(old);
|
||||
let new = registry.bind(&[new_id]).expect("replacement incarnation");
|
||||
assert_eq!(new[0].snapshot().active, 0);
|
||||
assert!(new[0].try_acquire().is_none());
|
||||
assert!(registry.publish(new_id, 1, Duration::from_secs(60)));
|
||||
let current = new[0].try_acquire().expect("new work");
|
||||
drop(native);
|
||||
assert_eq!(new[0].snapshot().active, 1, "old completion cannot refund the replacement");
|
||||
assert!(new[0].try_acquire().is_none());
|
||||
drop(current);
|
||||
assert_eq!(new[0].snapshot().active, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shrink_and_expiry_preserve_active_units_and_leave_other_disks_available() {
|
||||
let mut registry = registry();
|
||||
let a = device(1, 11);
|
||||
let b = device(2, 12);
|
||||
let disks = registry.bind(&[a, b]).expect("two physical disks");
|
||||
assert!(registry.publish(a, 2, Duration::from_secs(60)));
|
||||
assert!(registry.publish(b, 1, Duration::from_secs(60)));
|
||||
let first = disks[0].try_acquire().expect("first unit");
|
||||
let second = disks[0].try_acquire().expect("second unit");
|
||||
assert!(registry.publish(a, 1, Duration::from_secs(60)));
|
||||
assert_eq!(disks[0].snapshot().active, 2);
|
||||
assert!(disks[0].try_acquire().is_none());
|
||||
assert!(disks[1].try_acquire().is_some(), "a stalled disk cannot consume another disk's capacity");
|
||||
assert!(registry.publish(a, 2, Duration::ZERO));
|
||||
assert_eq!(disks[0].snapshot().limit, 0);
|
||||
assert_eq!(disks[0].snapshot().active, 2);
|
||||
drop((first, second));
|
||||
assert!(disks[0].try_acquire().is_none(), "expired observations do not reopen after completion");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn registry_bound_preserves_live_native_owners_and_reclaims_only_dead_entries() {
|
||||
let mut registry = registry();
|
||||
let ids = (0..MAX_PHYSICAL_OWNERS)
|
||||
.map(|minor| device(u32::try_from(minor).expect("bounded minor"), 1))
|
||||
.collect::<Vec<_>>();
|
||||
let attachments = registry.bind(&ids).expect("exact registry bound");
|
||||
let extra = device(999, 1);
|
||||
assert!(matches!(registry.bind(&[extra]), Err(error) if error.kind() == io::ErrorKind::WouldBlock));
|
||||
assert_eq!(registry.owners.len(), MAX_PHYSICAL_OWNERS, "failed binding must not add partial owners");
|
||||
assert!(registry.publish(ids[0], 1, Duration::from_secs(60)));
|
||||
let native = attachments[0].try_acquire().expect("native tail");
|
||||
drop(attachments);
|
||||
let replacement = registry.bind(&[extra]).expect("dead owners reclaimed");
|
||||
assert_eq!(registry.owners.len(), 2, "the native tail still owns its original entry");
|
||||
let original = registry.bind(&[ids[0]]).expect("original native owner");
|
||||
assert_eq!(original[0].snapshot().active, 1);
|
||||
drop((native, original, replacement));
|
||||
assert!(!registry.publish(extra, 1, Duration::from_secs(60)), "a dead owner cannot be refreshed");
|
||||
assert!(registry.bind(&[]).is_err());
|
||||
assert!(registry.bind(&[device(1, 0)]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retirement_reclaims_idle_descriptors_but_keeps_active_native_incarnations() {
|
||||
let mut registry = registry();
|
||||
let ids = (0..MAX_PHYSICAL_OWNERS)
|
||||
.map(|minor| device(u32::try_from(minor).expect("bounded minor"), 1))
|
||||
.collect::<Vec<_>>();
|
||||
let descriptors = registry.bind(&ids).expect("full registry");
|
||||
assert!(registry.publish(ids[0], 1, Duration::from_secs(60)));
|
||||
let native = descriptors[0].try_acquire().expect("active old incarnation");
|
||||
let mut current = ids[2..].iter().copied().collect::<BTreeSet<_>>();
|
||||
registry.retire_absent(¤t);
|
||||
assert_eq!(registry.owners.len(), MAX_PHYSICAL_OWNERS - 1);
|
||||
assert_eq!(descriptors[0].snapshot().active, 1);
|
||||
assert!(descriptors[0].try_acquire().is_none());
|
||||
assert!(descriptors[1].try_acquire().is_none());
|
||||
let first = device(0, 2);
|
||||
let second = device(1, 2);
|
||||
let replacement = registry.bind(&[first]).expect("idle descriptor was reclaimed");
|
||||
current.insert(first);
|
||||
assert!(matches!(registry.bind(&[second]), Err(error) if error.kind() == io::ErrorKind::WouldBlock));
|
||||
drop(native);
|
||||
registry.retire_absent(¤t);
|
||||
let next = registry.bind(&[second]).expect("final native release permits reclamation");
|
||||
assert_eq!(replacement[0].snapshot().active, 0);
|
||||
assert_eq!(next[0].snapshot().active, 0);
|
||||
assert_eq!(descriptors[0].snapshot().active, 0);
|
||||
assert!(
|
||||
!registry.publish(ids[0], 1, Duration::from_secs(60)),
|
||||
"retired identity cannot be refreshed"
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,309 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::{PhysicalDiskBinding, ResourceIoUnit};
|
||||
use rustfs_concurrency::adaptive::{
|
||||
NodeAdmission,
|
||||
permit_set::{NodeResourceCompletion, NodeResourcePermit},
|
||||
};
|
||||
use std::cell::{Cell, RefCell};
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::sync::Arc;
|
||||
|
||||
tokio::task_local! {
|
||||
static REMOTE_RESOURCES: RemoteBackgroundIo;
|
||||
}
|
||||
|
||||
/// Receiver-owned admission for an authenticated background RPC. The owner must
|
||||
/// be the same node controller used by local Scanner and Heal work.
|
||||
pub struct RemoteBackgroundIo {
|
||||
node: Arc<NodeAdmission>,
|
||||
admitted: RefCell<Option<NodeResourcePermit>>,
|
||||
progressed: Cell<bool>,
|
||||
finished: Cell<bool>,
|
||||
}
|
||||
|
||||
impl RemoteBackgroundIo {
|
||||
pub fn new(node: Arc<NodeAdmission>) -> Self {
|
||||
Self {
|
||||
node,
|
||||
admitted: RefCell::new(None),
|
||||
progressed: Cell::new(false),
|
||||
finished: Cell::new(false),
|
||||
}
|
||||
}
|
||||
|
||||
/// Move the request's reservations into its existing stream producer.
|
||||
/// Only physical subsets may be cloned into native I/O; never its node unit.
|
||||
pub fn take_current() -> Option<Self> {
|
||||
REMOTE_RESOURCES
|
||||
.try_with(|context| Self {
|
||||
node: context.node.clone(),
|
||||
admitted: RefCell::new(context.admitted.take()),
|
||||
progressed: Cell::new(context.progressed.replace(false)),
|
||||
finished: Cell::new(false),
|
||||
})
|
||||
.ok()
|
||||
}
|
||||
|
||||
/// Defer a response's progress without keeping async admission alive while
|
||||
/// its body streams. The reader retains its own native resource ownership.
|
||||
pub fn take_current_completion() -> Option<NodeResourceCompletion> {
|
||||
REMOTE_RESOURCES
|
||||
.try_with(|context| {
|
||||
let admitted = context.admitted.take();
|
||||
if context.progressed.replace(false) {
|
||||
admitted.map(NodeResourcePermit::into_completion)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
/// A failed response or cancelled producer must not credit earlier phases.
|
||||
pub fn discard_current_progress() {
|
||||
let _ = REMOTE_RESOURCES.try_with(|context| context.progressed.set(false));
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RemoteBackgroundIo {
|
||||
fn drop(&mut self) {
|
||||
if self.finished.get()
|
||||
&& self.progressed.get()
|
||||
&& let Some(permit) = self.admitted.get_mut().take()
|
||||
{
|
||||
permit.complete();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Scope a verified request or its stream producer. An interrupted request
|
||||
/// returns node capacity while native jobs retain their selected physical units.
|
||||
pub async fn with_remote_background_io<F: Future>(context: Option<RemoteBackgroundIo>, operation: F) -> F::Output {
|
||||
match context {
|
||||
Some(context) => {
|
||||
REMOTE_RESOURCES
|
||||
.scope(context, async {
|
||||
let result = operation.await;
|
||||
REMOTE_RESOURCES.with(|context| context.finished.set(true));
|
||||
result
|
||||
})
|
||||
.await
|
||||
}
|
||||
None => operation.await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn active() -> bool {
|
||||
REMOTE_RESOURCES.try_with(|_| ()).is_ok()
|
||||
}
|
||||
|
||||
pub(crate) fn resources() -> Option<ResourceIoUnit> {
|
||||
REMOTE_RESOURCES
|
||||
.try_with(|context| {
|
||||
context
|
||||
.admitted
|
||||
.borrow()
|
||||
.as_ref()
|
||||
.map(|permit| ResourceIoUnit::from(permit.resources().clone()))
|
||||
})
|
||||
.ok()
|
||||
.flatten()
|
||||
}
|
||||
|
||||
pub(crate) fn record_progress() {
|
||||
let _ = REMOTE_RESOURCES.try_with(|context| context.progressed.set(true));
|
||||
}
|
||||
|
||||
pub(crate) fn admit(binding: &PhysicalDiskBinding) -> io::Result<()> {
|
||||
REMOTE_RESOURCES
|
||||
.try_with(|context| {
|
||||
let deferred = || io::Error::from(io::ErrorKind::WouldBlock);
|
||||
let plan = binding.plan().ok_or_else(deferred)?;
|
||||
let mut admitted = context.admitted.borrow_mut();
|
||||
if let Some(permit) = admitted.as_ref() {
|
||||
// Reuse the request's receipt for append/read/rename phases at
|
||||
// capacity one. Additional attachments never wait under locks.
|
||||
let native = plan.capture_or_acquire_native(permit.resources()).map_err(|_| deferred())?;
|
||||
drop(native);
|
||||
if !plan.is_current() {
|
||||
return Err(deferred());
|
||||
}
|
||||
} else {
|
||||
*admitted = Some(plan.try_acquire(&context.node).ok_or_else(deferred)?);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap_or(Ok(()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_concurrency::adaptive::{
|
||||
AdaptiveAdmission, FeedbackController, FeedbackObservation, ResourcePressure, permit_set::AdaptivePermitSetOwner,
|
||||
};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
fn node() -> Arc<NodeAdmission> {
|
||||
let node = NodeAdmission::new(NonZeroU16::MIN);
|
||||
node.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
node
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_reuses_one_receipt_across_phases_and_releases_only_native_subsets() {
|
||||
let node = node();
|
||||
let a = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let b = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let shared = AdaptivePermitSetOwner::new(NonZeroU16::MIN, vec![a.clone(), b.clone()]).expect("device catalog");
|
||||
let first = PhysicalDiskBinding::default();
|
||||
let second = PhysicalDiskBinding::default();
|
||||
first
|
||||
.set_test_shared_capacity(shared.clone(), vec![a.clone()])
|
||||
.expect("device A");
|
||||
second.set_test_shared_capacity(shared, vec![b.clone()]).expect("device B");
|
||||
|
||||
let native = with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
assert!(resources().is_none(), "no physical work before target selection");
|
||||
admit(&first).expect("first phase");
|
||||
let admitted = resources().expect("admitted resources");
|
||||
let native = first
|
||||
.plan()
|
||||
.expect("current plan")
|
||||
.capture_native(admitted.as_set().expect("resource set"))
|
||||
.expect("escaped native read");
|
||||
admit(&first).expect("second phase reuses capacity while the first reader lives");
|
||||
admit(&second).expect("additional selected device");
|
||||
assert_eq!([node.snapshot().active, a.snapshot().active, b.snapshot().active], [1, 1, 1]);
|
||||
record_progress();
|
||||
native
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, a.snapshot().active, b.snapshot().active], [0, 1, 0]);
|
||||
assert_eq!([node.completed(), a.completed(), b.completed()], [Some(1), Some(1), Some(1)]);
|
||||
drop(native);
|
||||
assert_eq!(a.snapshot().active, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_cancellation_returns_node_but_not_escaped_native_capacity() {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
|
||||
let (sender, receiver) = tokio::sync::oneshot::channel();
|
||||
let task = tokio::spawn(with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async move {
|
||||
admit(&binding).expect("receiver admission");
|
||||
let admitted = resources().expect("receiver resources");
|
||||
let native = binding
|
||||
.plan()
|
||||
.expect("plan")
|
||||
.capture_native(admitted.as_set().expect("set"))
|
||||
.expect("native subset");
|
||||
sender.send(native).expect("native owner receiver");
|
||||
record_progress();
|
||||
std::future::pending::<()>().await;
|
||||
}));
|
||||
let native = receiver.await.expect("native read ownership");
|
||||
task.abort();
|
||||
assert!(task.await.expect_err("cancel request").is_cancelled());
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 1]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
drop(native);
|
||||
assert_eq!(gate.snapshot().active, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_defers_stream_progress_without_retaining_node_capacity() {
|
||||
for (progressed, succeeds) in [(false, true), (true, false), (true, true)] {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
|
||||
let completion = with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
admit(&binding).expect("receiver admission");
|
||||
if progressed {
|
||||
record_progress();
|
||||
}
|
||||
let completion = RemoteBackgroundIo::take_current_completion();
|
||||
assert!(RemoteBackgroundIo::take_current_completion().is_none(), "single progress owner");
|
||||
assert!(resources().is_none());
|
||||
completion
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
assert_eq!(completion.is_some(), progressed);
|
||||
if succeeds && let Some(completion) = completion {
|
||||
completion.complete();
|
||||
}
|
||||
let expected = u64::from(progressed && succeeds);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(expected), Some(expected)]);
|
||||
}
|
||||
assert!(
|
||||
RemoteBackgroundIo::take_current_completion().is_none(),
|
||||
"foreground has no progress owner"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_failed_completion_does_not_credit_successful_preparation() {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
admit(&binding).expect("receiver admission");
|
||||
record_progress();
|
||||
RemoteBackgroundIo::discard_current_progress();
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_stream_transfer_moves_node_ownership_and_defers_unavailable_devices() {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 0);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
|
||||
let context = with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
assert_eq!(admit(&binding).expect_err("closed device").kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
gate.set_limit(1);
|
||||
admit(&binding).expect("opened device");
|
||||
let context = RemoteBackgroundIo::take_current().expect("stream context");
|
||||
assert!(resources().is_none(), "request cannot keep a second node owner");
|
||||
context
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [1, 1]);
|
||||
assert_eq!(node.completed(), Some(0));
|
||||
with_remote_background_io(Some(context), async {
|
||||
admit(&binding).expect("stream reuses the request's receipt");
|
||||
binding
|
||||
.set_test_capacity(vec![AdaptiveAdmission::new(NonZeroU16::MIN, 1)])
|
||||
.expect("replace physical device");
|
||||
assert_eq!(
|
||||
admit(&binding).expect_err("old receipt cannot cross topology").kind(),
|
||||
io::ErrorKind::WouldBlock
|
||||
);
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!(node.completed(), Some(0), "admission alone is not completed work");
|
||||
}
|
||||
}
|
||||
+3591
-238
File diff suppressed because it is too large
Load Diff
@@ -28,7 +28,7 @@ use super::{
|
||||
should_remove_staged_meta_before_commit, skip_access_checks, tmp_staging_mutation_path,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::{run_inline_before_file_sync_admission, run_owned_file_write_before_open, run_rename_data_after_first_publication};
|
||||
use super::{run_inline_before_file_sync_admission, run_owned_file_before_open, run_rename_data_after_first_publication};
|
||||
use crate::crash_inject::{self, CrashPoint};
|
||||
use crate::disk::{
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RenameDataResp, STORAGE_FORMAT_FILE, STORAGE_FORMAT_FILE_BACKUP, SnapshotLeaseToken,
|
||||
@@ -329,6 +329,38 @@ pub(super) async fn lock_rename_commit_directories(
|
||||
result.map_err(to_file_error).map_err(DiskError::from)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn prepare_rename_commit_and_destination_metadata_std(
|
||||
create_source_parent: bool,
|
||||
source_parent: &Path,
|
||||
destination_parent: &Path,
|
||||
destination_metadata: &Path,
|
||||
base_dir: &Path,
|
||||
publication_root: &os::PublicationRoot,
|
||||
) -> std::io::Result<(os::RenameCommitGuard, Option<Bytes>)> {
|
||||
if create_source_parent {
|
||||
std::fs::create_dir_all(source_parent)?;
|
||||
}
|
||||
let guard = match os::prepare_rename_commit_guard(source_parent, destination_parent, base_dir, publication_root) {
|
||||
Ok(guard) => guard,
|
||||
Err(err) => {
|
||||
return Err(match std::fs::symlink_metadata(base_dir) {
|
||||
Err(base_err) if base_err.kind() == ErrorKind::NotFound => base_err,
|
||||
_ => err,
|
||||
});
|
||||
}
|
||||
};
|
||||
let data = os::read_destination_file_with_commit_guard(destination_metadata, &guard)?;
|
||||
Ok((guard, data.map(Bytes::from)))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
enum FusedCommitAttempt {
|
||||
Committed(RenameDataResp),
|
||||
CommitErr(DiskError),
|
||||
Fallback(os::RenameCommitGuard, Option<Bytes>, FileInfo),
|
||||
}
|
||||
|
||||
async fn prepare_rename_commit_and_destination_metadata(
|
||||
create_source_parent: bool,
|
||||
source_parent: &Path,
|
||||
@@ -571,6 +603,9 @@ impl LocalDisk {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
if dst_volume == crate::disk::RUSTFS_META_BUCKET {
|
||||
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
|
||||
}
|
||||
if fi.is_legacy_indexed_delete_marker() {
|
||||
fi.erasure.index = 0;
|
||||
}
|
||||
@@ -698,6 +733,216 @@ impl LocalDisk {
|
||||
let dst_file_parent = dst_file_path
|
||||
.parent()
|
||||
.ok_or_else(|| DiskError::other("missing object metadata parent"))?;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let (rename_commit_guard, has_dst_buf) = if !preserve_upload_metadata
|
||||
&& multipart_publication.is_none()
|
||||
&& pool_meta_condition.is_none()
|
||||
&& !fi_healing
|
||||
&& !os::dst_dir_fsync_group_commit_enabled()
|
||||
&& !os::file_fdatasync_group_commit_enabled()
|
||||
&& !super::has_rename_data_test_failpoints(dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterDataRename, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterBackupBeforeMetaCommit, dst_path)
|
||||
&& !crash_inject::is_armed(CrashPoint::RenameAfterMetaCommit, dst_path)
|
||||
&& !super::has_delete_version_test_failpoints()
|
||||
{
|
||||
#[cfg(test)]
|
||||
if !no_inline {
|
||||
run_inline_before_file_sync_admission(dst_path);
|
||||
}
|
||||
let sync_data_shards = no_inline && durability.syncs_data_shards();
|
||||
let sync_commit_metadata = durability.syncs_commit_metadata();
|
||||
let file_sync_admission = if sync_data_shards || sync_commit_metadata {
|
||||
Some(
|
||||
os::acquire_file_sync_admission(self.file_sync_permits.clone())
|
||||
.await
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from)?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let src_parent_owned = src_file_parent.to_path_buf();
|
||||
let dst_parent_owned = dst_file_parent.to_path_buf();
|
||||
let src_file_path_owned = src_file_path.clone();
|
||||
let dst_file_path_owned = dst_file_path.clone();
|
||||
let dst_volume_dir_owned = dst_volume_dir.clone();
|
||||
let publication_root_owned = self.publication_root.clone();
|
||||
let has_data_dir_path_owned = has_data_dir_path.clone();
|
||||
let dst_volume_str = dst_volume.to_string();
|
||||
let dst_path_str = dst_path.to_string();
|
||||
let fi_moved = fi;
|
||||
#[cfg(test)]
|
||||
let root_owned = self.root.clone();
|
||||
|
||||
let fused_attempt = move |is_cancelled: &dyn Fn() -> bool| -> std::io::Result<FusedCommitAttempt> {
|
||||
let (guard, has_dst_buf) = prepare_rename_commit_and_destination_metadata_std(
|
||||
!no_inline,
|
||||
&src_parent_owned,
|
||||
&dst_parent_owned,
|
||||
&dst_file_path_owned,
|
||||
&dst_volume_dir_owned,
|
||||
&publication_root_owned,
|
||||
)?;
|
||||
// An existing xl.meta can share versions or require rollback
|
||||
// restoration when a post-rename directory sync fails.
|
||||
if is_cancelled() || (sync_commit_metadata && has_dst_buf.is_some()) {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
}
|
||||
let mut xlmeta = FileMeta::new();
|
||||
let mut dst_meta_unparsable = false;
|
||||
if let Some(ref dst_buf) = has_dst_buf {
|
||||
if FileMeta::is_xl2_v1_format(dst_buf)
|
||||
&& let Ok(nmeta) = FileMeta::load(dst_buf)
|
||||
{
|
||||
xlmeta = nmeta;
|
||||
} else {
|
||||
dst_meta_unparsable = true;
|
||||
}
|
||||
}
|
||||
let version_id = fi_moved.version_id.unwrap_or_default();
|
||||
let has_old_data_dir = xlmeta.find_unshared_data_dir_for_version(Some(version_id));
|
||||
let rollback_data_dir = has_old_data_dir.or_else(|| {
|
||||
if has_dst_buf.is_some() && xlmeta.find_version(Some(version_id)).is_ok() {
|
||||
Some(inline_metadata_rollback_dir(version_id, &xlmeta))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
if rollback_data_dir.is_some() {
|
||||
return Ok(FusedCommitAttempt::Fallback(guard, has_dst_buf, fi_moved));
|
||||
}
|
||||
let (old_current_size, old_current_source_checked, old_current_source) = observe_old_current(
|
||||
has_dst_buf.is_some(),
|
||||
dst_meta_unparsable,
|
||||
&xlmeta,
|
||||
&dst_volume_str,
|
||||
&dst_path_str,
|
||||
version_id,
|
||||
);
|
||||
xlmeta.add_version(fi_moved)?;
|
||||
let version_signature = rename_data_versions_signature(&xlmeta);
|
||||
let new_dst_buf = xlmeta.marshal_msg().map_err(std::io::Error::other)?;
|
||||
let destination_was_absent = has_dst_buf.is_none();
|
||||
let mut skip_parent = dst_volume_dir_owned.clone();
|
||||
if has_dst_buf.is_some() {
|
||||
skip_parent = dst_parent_owned.clone();
|
||||
}
|
||||
|
||||
let mut prepared_metadata_source =
|
||||
os::create_prepared_rename_source_with_commit_guard(&src_file_path_owned, &dst_file_path_owned, &guard)?;
|
||||
prepared_metadata_source.write_all(&new_dst_buf, sync_commit_metadata)?;
|
||||
|
||||
if no_inline {
|
||||
if sync_data_shards && let Some((src_data_path, _)) = has_data_dir_path_owned.as_ref() {
|
||||
let sync_result = if super::take_presynced_tmp_data_dir(src_data_path) {
|
||||
// The payload was synced, but its part.1 directory entry
|
||||
// must also be persisted before moving the data directory.
|
||||
os::fsync_dir_std(src_data_path)
|
||||
} else {
|
||||
os::sync_dir_files_std(src_data_path)
|
||||
};
|
||||
if let Err(err) = sync_result
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
if let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref()
|
||||
&& let Err(err) = os::rename_all_with_commit_guard_on_thread(src_data_path, dst_data_path, &guard)
|
||||
{
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
#[cfg(test)]
|
||||
if has_data_dir_path_owned.is_some() {
|
||||
run_rename_data_after_first_publication(&root_owned, &dst_volume_str, &dst_path_str);
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(err) = os::rename_prepared_source_on_thread(
|
||||
prepared_metadata_source,
|
||||
&src_file_path_owned,
|
||||
&dst_file_path_owned,
|
||||
&skip_parent,
|
||||
&publication_root_owned,
|
||||
&guard,
|
||||
) {
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
}
|
||||
return Ok(FusedCommitAttempt::CommitErr(err));
|
||||
}
|
||||
|
||||
if sync_commit_metadata
|
||||
&& let Err(err) =
|
||||
fsync_inline_commit_std(&dst_file_path_owned, &dst_volume_dir_owned, destination_was_absent, None, &guard)
|
||||
{
|
||||
if no_inline && let Some((src_data_path, dst_data_path)) = has_data_dir_path_owned.as_ref() {
|
||||
let _ = std::fs::rename(dst_data_path, src_data_path);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
if let Some(parent) = src_file_path_owned.parent() {
|
||||
let _ = std::fs::remove_dir(parent);
|
||||
}
|
||||
|
||||
Ok(FusedCommitAttempt::Committed(RenameDataResp {
|
||||
multipart_source: None,
|
||||
old_data_dir: None,
|
||||
rollback_data_dir: None,
|
||||
cleanup_data_dir: None,
|
||||
sign: version_signature,
|
||||
old_current_size,
|
||||
old_current_source_checked,
|
||||
old_current_source,
|
||||
}))
|
||||
};
|
||||
|
||||
let attempt_res = if let Some(admission) = file_sync_admission.as_ref() {
|
||||
os::run_cancellable_blocking_namespace_file_sync_operation(mutation_lease.clone(), admission, fused_attempt).await
|
||||
} else {
|
||||
os::run_cancellable_blocking_namespace_operation(mutation_lease.clone(), fused_attempt).await
|
||||
}
|
||||
.map_err(to_file_error)
|
||||
.map_err(DiskError::from);
|
||||
drop(file_sync_admission);
|
||||
match attempt_res {
|
||||
Ok(FusedCommitAttempt::Committed(resp)) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
#[cfg(unix)]
|
||||
if let Some(receipt_mutation) = receipt_mutation.as_mut() {
|
||||
receipt_mutation.complete();
|
||||
}
|
||||
return Ok(resp);
|
||||
}
|
||||
Ok(FusedCommitAttempt::CommitErr(err)) | Err(err) => {
|
||||
for part_path in &invalidate_part_paths {
|
||||
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
Ok(FusedCommitAttempt::Fallback(guard, dst_buf, restored_fi)) => {
|
||||
fi = restored_fi;
|
||||
(guard, dst_buf)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
prepare_rename_commit_and_destination_metadata(
|
||||
!no_inline && !preserve_upload_metadata,
|
||||
src_file_parent,
|
||||
dst_file_parent,
|
||||
&dst_file_path,
|
||||
&dst_volume_dir,
|
||||
&self.publication_root,
|
||||
mutation_lease.clone(),
|
||||
)
|
||||
.await?
|
||||
};
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
// Acquire the common trees before reading destination metadata. On
|
||||
// Windows this pins the object directory identity across publication.
|
||||
// Multipart authority already created its source tree. Recreating it
|
||||
@@ -840,7 +1085,7 @@ impl LocalDisk {
|
||||
async move {
|
||||
os::run_blocking_namespace_operation(owned_mutation_lease, move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&src_file_path);
|
||||
run_owned_file_before_open(&src_file_path);
|
||||
let mut prepared_metadata_source = os::create_prepared_rename_source_with_commit_guard(
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
@@ -1026,7 +1271,7 @@ impl LocalDisk {
|
||||
// deterministic rollback backup.
|
||||
let write_result = os::run_blocking_namespace_operation(commit.mutation_lease.clone(), move || {
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&backup_path);
|
||||
run_owned_file_before_open(&backup_path);
|
||||
backup_path_guard.write_file_for_path_access(
|
||||
&backup_path,
|
||||
backup_bytes.as_ref(),
|
||||
|
||||
@@ -1317,7 +1317,7 @@ impl LocalDisk {
|
||||
continue;
|
||||
};
|
||||
#[cfg(test)]
|
||||
run_owned_file_write_before_open(&record_destination.join(name));
|
||||
run_owned_file_before_open(&record_destination.join(name));
|
||||
let mut file = std::fs::File::create(staged.path().join(name))?;
|
||||
std::io::Write::write_all(&mut file, bytes)?;
|
||||
if sync && !bytes.is_empty() {
|
||||
@@ -1795,7 +1795,7 @@ impl LocalDisk {
|
||||
self.write_all_meta_with_namespace_owner(
|
||||
volume,
|
||||
&format!("{object}/{STORAGE_FORMAT_FILE}"),
|
||||
&target,
|
||||
target,
|
||||
true,
|
||||
owner.clone(),
|
||||
)
|
||||
@@ -2090,6 +2090,474 @@ impl LocalDisk {
|
||||
pub(super) async fn delete_recovery_barrier(&self, volume: &str, object: &str, transaction: Uuid, stage: &str) -> Result<()> {
|
||||
barrier(volume, object, transaction, stage, &self.root).await
|
||||
}
|
||||
|
||||
pub(super) async fn try_fused_recoverable_delete_version(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
fi: &FileInfo,
|
||||
opts: &DeleteOptions,
|
||||
transaction: Uuid,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<Option<()>> {
|
||||
if !opts.recoverable_delete
|
||||
|| opts.undo_write
|
||||
|| opts.expected_delete_marker.is_some()
|
||||
|| pool_meta_native::is_object(&self.root, volume, object)
|
||||
|| has_delete_version_test_failpoints()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
self.wait_for_startup_cleanup().await;
|
||||
|
||||
let (_, file_path) = self.io_get_bucket_and_object_path(volume, object)?;
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
let xl_path = path_join(&[file_path.as_path(), Path::new(STORAGE_FORMAT_FILE)]);
|
||||
let destination = file_path.join(transaction.to_string());
|
||||
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let incarnation = opts.delete_recovery_bucket_incarnation;
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&destination, owner).await;
|
||||
let fi_clone = fi.clone();
|
||||
let volume_str = volume.to_owned();
|
||||
let object_str = object.to_owned();
|
||||
|
||||
let res = os::run_blocking_namespace_operation_with_sync(
|
||||
lease,
|
||||
sync,
|
||||
move || -> std::io::Result<Result<Option<Option<Uuid>>>> {
|
||||
let buf = match std::fs::read(&xl_path) {
|
||||
Ok(buf) => buf,
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let (old_dir, journal_raw) = match (|| -> Result<Option<(Option<Uuid>, Vec<u8>)>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let deleted_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| v.header.version_id.unwrap_or_default())
|
||||
.collect();
|
||||
let original_versions: Vec<_> = meta
|
||||
.versions
|
||||
.iter()
|
||||
.map(|v| generation(v.into_fileinfo(&volume_str, &object_str, true)?))
|
||||
.collect::<Result<_>>()?;
|
||||
let old_dir = meta.delete_version(&fi_clone)?;
|
||||
if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
if let Some(uuid) = old_dir {
|
||||
meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
|
||||
let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
|
||||
check_path_length(old_path.to_string_lossy().as_ref())?;
|
||||
if old_path.join(STORAGE_FORMAT_FILE).exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
if !meta.versions.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut predecessors = BTreeSet::new();
|
||||
if let Ok(entries) = std::fs::read_dir(&file_path) {
|
||||
for (visited, entry) in entries.enumerate() {
|
||||
if visited >= 65536 {
|
||||
return Err(DiskError::DiskOngoingReq);
|
||||
}
|
||||
let entry = entry.map_err(to_file_error)?;
|
||||
let Some(id) = entry
|
||||
.file_name()
|
||||
.to_str()
|
||||
.and_then(|name| Uuid::parse_str(name).ok())
|
||||
.filter(|id| !id.is_nil() && *id != transaction)
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
if !entry.file_type().map_err(to_file_error)?.is_dir() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let prepared = regular_bytes(&entry.path().join(PREPARED))?;
|
||||
let receipt = regular_bytes(&entry.path().join(DELETE_QUORUM_COMMITTED_FILE))?;
|
||||
let mut records = Vec::new();
|
||||
if let Some(bytes) = prepared {
|
||||
records.push(
|
||||
serde_json::from_slice::<DeleteRecoveryJournal>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?,
|
||||
);
|
||||
}
|
||||
if let Some(bytes) = receipt.filter(|bytes| !bytes.is_empty()) {
|
||||
records.push(
|
||||
serde_json::from_slice::<CommitReceipt>(&bytes)
|
||||
.map_err(|_| DiskError::FileCorrupt)?
|
||||
.journal,
|
||||
);
|
||||
}
|
||||
for record in records {
|
||||
if record.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
|| record.volume != volume_str
|
||||
|| record.object != object_str
|
||||
|| record.transaction != id
|
||||
|| incarnation.zip(record.bucket_incarnation).is_some_and(|(a, b)| a != b)
|
||||
|| record.predecessors.iter().any(|before| before.is_nil() || *before == id)
|
||||
{
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
predecessors.insert(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut sparse = fi_clone;
|
||||
sparse.data = None;
|
||||
let journal = DeleteRecoveryJournal {
|
||||
protocol_version: DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
volume: volume_str,
|
||||
object: object_str,
|
||||
transaction,
|
||||
bucket_incarnation: incarnation,
|
||||
original_digest: Some(digest(&buf)),
|
||||
target_digest: None,
|
||||
data_dirs: old_dir.into_iter().collect(),
|
||||
deleted_versions,
|
||||
requests: vec![sparse],
|
||||
original_versions,
|
||||
predecessors: predecessors.into_iter().collect(),
|
||||
};
|
||||
Ok(Some((old_dir, journal_bytes(&journal)?)))
|
||||
})() {
|
||||
Ok(Some(pair)) => pair,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
Err(err) => return Ok(Err(err)),
|
||||
};
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let staged = tempfile::Builder::new()
|
||||
.prefix("delete-preparation-")
|
||||
.tempdir_in(&staging_root)?;
|
||||
for (name, bytes) in [
|
||||
(STORAGE_FORMAT_FILE_BACKUP, buf.as_slice()),
|
||||
(PREPARED, journal_raw.as_slice()),
|
||||
] {
|
||||
let mut file = std::fs::File::create(staged.path().join(name))?;
|
||||
std::io::Write::write_all(&mut file, bytes)?;
|
||||
if sync && !bytes.is_empty() {
|
||||
file.sync_data()?;
|
||||
}
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(staged.path())?;
|
||||
}
|
||||
match std::fs::symlink_metadata(&destination) {
|
||||
Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) if is_dir_not_empty_error(&err) => return Ok(Err(DiskError::OutdatedXLMeta)),
|
||||
Err(err) => return Err(err),
|
||||
},
|
||||
Ok(_) => return Ok(Err(DiskError::OutdatedXLMeta)),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
if let Some(parent) = destination.parent() {
|
||||
std::fs::create_dir_all(parent)?;
|
||||
}
|
||||
std::fs::rename(staged.path(), &destination)?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&file_path)?;
|
||||
}
|
||||
let mut reserved_data_path = None;
|
||||
if let Some(uuid) = old_dir {
|
||||
let old_path = file_path.join(uuid.to_string());
|
||||
if old_path.is_dir() {
|
||||
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
let marker = std::fs::File::create(&reserved_marker)?;
|
||||
if sync {
|
||||
marker.sync_all()?;
|
||||
os::fsync_dir_std(&old_path)?;
|
||||
}
|
||||
reserved_data_path = Some((old_path, reserved_marker));
|
||||
}
|
||||
}
|
||||
if let Err(err) = std::fs::remove_file(&xl_path)
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
if let Some((_, reserved_marker)) = &reserved_data_path {
|
||||
let _ = std::fs::remove_file(reserved_marker);
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(&file_path)?;
|
||||
}
|
||||
if let Some((old_path, reserved_marker)) = reserved_data_path {
|
||||
let recovery_marker = old_path.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
match std::fs::rename(&reserved_marker, &recovery_marker) {
|
||||
Ok(()) if sync => os::fsync_dir_std(&old_path)?,
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Ok(Ok(Some(old_dir)))
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(to_file_error)??;
|
||||
|
||||
let Some(old_dir) = res else {
|
||||
return Ok(None);
|
||||
};
|
||||
if let Some(uuid) = old_dir {
|
||||
self.io_backend
|
||||
.invalidate_cached_fds_under(volume, &format!("{object}/{uuid}"));
|
||||
}
|
||||
Ok(Some(()))
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if !safe_path(object, false) || transaction.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let object_path = self.get_object_path(volume, object)?;
|
||||
let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
|
||||
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
|
||||
|
||||
let root = self.io_get_object_path(volume, object)?;
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let staging_root = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let volume_str = volume.to_owned();
|
||||
let object_str = object.to_owned();
|
||||
|
||||
os::run_blocking_namespace_operation_with_sync(
|
||||
lease,
|
||||
sync,
|
||||
move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
|
||||
let prepared = match regular_bytes(&rollback.join(PREPARED)) {
|
||||
Ok(Some(bytes)) => bytes,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
Err(e) => return Ok(Err(e)),
|
||||
};
|
||||
let Ok(journal) = serde_json::from_slice::<DeleteRecoveryJournal>(&prepared) else {
|
||||
return Ok(Err(DiskError::FileCorrupt));
|
||||
};
|
||||
if journal.protocol_version != DELETE_RECOVERY_PROTOCOL_VERSION
|
||||
|| journal.volume != volume_str
|
||||
|| journal.object != object_str
|
||||
|| journal.transaction != transaction
|
||||
|| journal.transaction.is_nil()
|
||||
|| journal.data_dirs.iter().any(|id| id.is_nil() || *id == transaction)
|
||||
{
|
||||
return Ok(Err(DiskError::FileCorrupt));
|
||||
}
|
||||
let current_exists = match regular_file_exists(&root.join(STORAGE_FORMAT_FILE)) {
|
||||
Ok(exists) => exists,
|
||||
Err(e) => return Ok(Err(e)),
|
||||
};
|
||||
let unversioned_target_absent = journal.target_digest.is_none() && !current_exists;
|
||||
if !unversioned_target_absent {
|
||||
return Ok(Ok(None));
|
||||
}
|
||||
let mut sole_transaction = journal.predecessors.is_empty();
|
||||
if sole_transaction {
|
||||
match std::fs::read_dir(&root) {
|
||||
Ok(entries) => {
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
let Ok(name) = entry.file_name().into_string() else {
|
||||
sole_transaction = false;
|
||||
break;
|
||||
};
|
||||
if let Some(id) = canonical_uuid(&name).filter(|id| !id.is_nil())
|
||||
&& id != transaction
|
||||
&& !journal.data_dirs.contains(&id)
|
||||
{
|
||||
sole_transaction = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
let receipt = CommitReceipt {
|
||||
journal: journal.clone(),
|
||||
finished: false,
|
||||
};
|
||||
let receipt_bytes = serde_json::to_vec(&receipt).map_err(std::io::Error::other)?;
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let mut staged = tempfile::NamedTempFile::new_in(&staging_root)?;
|
||||
std::io::Write::write_all(staged.as_file_mut(), &receipt_bytes)?;
|
||||
if sync && !receipt_bytes.is_empty() {
|
||||
staged.as_file().sync_data()?;
|
||||
}
|
||||
std::fs::rename(staged.path(), &receipt_path)?;
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
Ok(Ok(Some((journal, sole_transaction, unversioned_target_absent))))
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(to_file_error)?
|
||||
}
|
||||
|
||||
pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<bool> {
|
||||
fn ignore_not_found(res: std::io::Result<()>) -> std::io::Result<()> {
|
||||
match res {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if !safe_path(object, false) || transaction.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let object_path = self.get_object_path(volume, object)?;
|
||||
let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
|
||||
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
|
||||
|
||||
let mut claims: Vec<SnapshotLeaseDeleteClaim> = Vec::with_capacity(journal.data_dirs.len());
|
||||
for data_dir in &journal.data_dirs {
|
||||
let key = SnapshotLeaseKey {
|
||||
volume: volume.to_owned(),
|
||||
path: format!("{object}/{data_dir}"),
|
||||
};
|
||||
let opts = DeleteOptions {
|
||||
recursive: true,
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
};
|
||||
let Some(claim) = self.claim_snapshot_delete(key, SnapshotPendingDelete::Ordinary(opts)) else {
|
||||
for prior in claims {
|
||||
prior.finish(false);
|
||||
}
|
||||
return Ok(false);
|
||||
};
|
||||
claims.push(claim);
|
||||
}
|
||||
|
||||
let bucket_path = self.io_get_bucket_path(volume)?;
|
||||
let root = self.io_get_object_path(volume, object)?;
|
||||
let rollback = root.join(transaction.to_string());
|
||||
let receipt_path = rollback.join(DELETE_QUORUM_COMMITTED_FILE);
|
||||
let sync = effective_durability(volume).syncs_commit_metadata();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&receipt_path, owner).await;
|
||||
let data_dirs = journal.data_dirs.clone();
|
||||
let prune_paths: Vec<_> = rollback
|
||||
.ancestors()
|
||||
.take_while(|dir| *dir != bucket_path.as_path() && dir.starts_with(&bucket_path))
|
||||
.map(Path::to_path_buf)
|
||||
.collect();
|
||||
|
||||
let res = os::run_blocking_namespace_operation_with_sync(lease, sync, move || -> std::io::Result<()> {
|
||||
let committed = format!("{DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
for data_dir in &data_dirs {
|
||||
for base in [&root, &rollback] {
|
||||
let dir_path = base.join(data_dir.to_string());
|
||||
match std::fs::read_dir(&dir_path) {
|
||||
Ok(entries) => {
|
||||
let mut removable = Vec::new();
|
||||
let mut preserve_dir = false;
|
||||
for entry in entries {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let Some(name) = name.to_str() else {
|
||||
preserve_dir = true;
|
||||
continue;
|
||||
};
|
||||
if entry.file_type()?.is_file()
|
||||
&& (delete_recovery_part_file(name)
|
||||
|| name == committed
|
||||
|| name == recovery_committed
|
||||
|| name == reserved
|
||||
|| is_delete_transaction_marker(name, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX))
|
||||
{
|
||||
removable.push(entry.path());
|
||||
} else {
|
||||
preserve_dir = true;
|
||||
}
|
||||
}
|
||||
for path in removable {
|
||||
ignore_not_found(std::fs::remove_file(path))?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(&dir_path)?;
|
||||
}
|
||||
if !preserve_dir {
|
||||
match std::fs::remove_dir(&dir_path) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound || is_dir_not_empty_error(&err) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
if !data_dirs.is_empty() && sync && root.is_dir() {
|
||||
os::fsync_dir_std(&root)?;
|
||||
}
|
||||
for name in [
|
||||
STORAGE_FORMAT_FILE_BACKUP,
|
||||
DELETE_MARKER_ROLLBACK_FILE,
|
||||
TARGET,
|
||||
PREPARED,
|
||||
FINISHED,
|
||||
] {
|
||||
ignore_not_found(std::fs::remove_file(rollback.join(name)))?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
ignore_not_found(std::fs::remove_file(&receipt_path))?;
|
||||
for dir_path in &prune_paths {
|
||||
if let Err(err) = std::fs::remove_dir(dir_path) {
|
||||
if is_dir_not_empty_error(&err) {
|
||||
break;
|
||||
}
|
||||
if !is_benign_object_rmdir_error(&err) {
|
||||
return Err(std::io::Error::other(FileAccessDeniedWithContext {
|
||||
path: dir_path.clone(),
|
||||
source: err,
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await;
|
||||
|
||||
let ok = res.is_ok();
|
||||
for claim in claims {
|
||||
claim.finish(ok);
|
||||
}
|
||||
res.map(|()| true).map_err(|err| to_file_error(err).into())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "e2e-test-hooks"))]
|
||||
@@ -3038,12 +3506,12 @@ mod tests {
|
||||
let exit = native.exit_signal.as_ref().expect("fixture cleanup lifecycle");
|
||||
exit.send(()).expect("stop the existing idle cleanup task");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while exit.receiver_count() != 0 || root_file.strong_count() != 2 {
|
||||
while exit.receiver_count() != 0 || root_file.strong_count() != 3 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("only LocalDisk's publication and mount fields may retain the initial root FD");
|
||||
.expect("only LocalDisk's publication, mount and I/O backend fields may retain the initial root FD");
|
||||
let transaction = Uuid::new_v4();
|
||||
let applied = apply(&native, transaction).await;
|
||||
let committed = native
|
||||
@@ -3309,6 +3777,171 @@ mod tests {
|
||||
inspected
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fused_delete_recovery_survives_interrupted_prepare_and_publication() {
|
||||
for published in [false, true] {
|
||||
let (_temp, disk, _info, original) = fixture().await;
|
||||
let transaction = Uuid::new_v4();
|
||||
let prepared = disk
|
||||
.try_fused_recoverable_delete_version(
|
||||
"recovery",
|
||||
"object",
|
||||
&request(),
|
||||
&DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
},
|
||||
transaction,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("prepare fused deletion");
|
||||
assert_eq!(prepared, Some(()), "the test must exercise the fused path");
|
||||
|
||||
if published {
|
||||
let (_, sole, target_absent) = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("publish fused receipt")
|
||||
.expect("fused publication must apply");
|
||||
assert!(sole && target_absent);
|
||||
}
|
||||
|
||||
let mut state = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("inspect persisted state after interruption");
|
||||
if published {
|
||||
assert_eq!(state.state, DeleteRecoveryState::QuorumCommitted);
|
||||
for action in [DeleteRecoveryAction::FinishCommit, DeleteRecoveryAction::FinalizeCommit] {
|
||||
state = disk
|
||||
.settle_delete_transaction("recovery", "object", transaction, state.fingerprint, action)
|
||||
.await
|
||||
.expect("finish published deletion through recovery");
|
||||
}
|
||||
assert_eq!(state.state, DeleteRecoveryState::Absent);
|
||||
assert!(disk.read_all("recovery", "object/xl.meta").await.is_err());
|
||||
} else {
|
||||
assert_eq!(state.state, DeleteRecoveryState::NewPrepared);
|
||||
state = disk
|
||||
.settle_delete_transaction(
|
||||
"recovery",
|
||||
"object",
|
||||
transaction,
|
||||
state.fingerprint,
|
||||
DeleteRecoveryAction::Rollback,
|
||||
)
|
||||
.await
|
||||
.expect("roll back uncommitted fused deletion");
|
||||
assert_eq!(state.state, DeleteRecoveryState::Absent);
|
||||
assert_eq!(
|
||||
disk.read_all("recovery", "object/xl.meta")
|
||||
.await
|
||||
.expect("restore original metadata")
|
||||
.as_ref(),
|
||||
original
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn fused_delete_recovery_preserves_or_removes_shard_after_interruption() {
|
||||
use crate::erasure::coding::BitrotWriter;
|
||||
|
||||
for mode in 0..3 {
|
||||
let published = mode != 0;
|
||||
let (_temp, disk, _info, _) = fixture().await;
|
||||
let data_dir = Uuid::new_v4();
|
||||
let payload = Bytes::from_static(b"original shard");
|
||||
let mut info = test_file_info("object", Uuid::nil(), Some(data_dir), None);
|
||||
info.size = i64::try_from(payload.len()).expect("payload size");
|
||||
info.erasure.block_size = payload.len();
|
||||
info.parts[0].size = payload.len();
|
||||
info.parts[0].actual_size = info.size;
|
||||
let original = test_meta(info.clone());
|
||||
let mut writer = BitrotWriter::new(
|
||||
std::io::Cursor::new(Vec::new()),
|
||||
info.erasure.shard_size(),
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
);
|
||||
writer.write(&payload).await.expect("encode shard");
|
||||
writer.shutdown().await.expect("finish shard encoding");
|
||||
let encoded = writer.into_inner().into_inner();
|
||||
let root = disk.io_get_object_path("recovery", "object").expect("object path");
|
||||
fs::write(root.join(STORAGE_FORMAT_FILE), &original)
|
||||
.await
|
||||
.expect("write shard metadata");
|
||||
let shard_dir = root.join(data_dir.to_string());
|
||||
fs::create_dir_all(&shard_dir).await.expect("create shard directory");
|
||||
fs::write(shard_dir.join("part.1"), &encoded)
|
||||
.await
|
||||
.expect("write original shard");
|
||||
let transaction = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
disk.try_fused_recoverable_delete_version(
|
||||
"recovery",
|
||||
"object",
|
||||
&request(),
|
||||
&DeleteOptions {
|
||||
recoverable_delete: true,
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
},
|
||||
transaction,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("prepare fused shard deletion"),
|
||||
Some(()),
|
||||
"the test must exercise the fused path"
|
||||
);
|
||||
if published {
|
||||
let (journal, sole, target_absent) = disk
|
||||
.try_publish_prepared_delete_local("recovery", "object", transaction, None)
|
||||
.await
|
||||
.expect("publish fused receipt")
|
||||
.expect("fused publication must apply");
|
||||
assert!(sole && target_absent);
|
||||
if mode == 2 {
|
||||
assert!(
|
||||
disk.try_finish_and_finalize_prepared_delete_local("recovery", "object", transaction, &journal, None)
|
||||
.await
|
||||
.expect("finish fused deletion"),
|
||||
"the fused finalization must apply"
|
||||
);
|
||||
assert!(!shard_dir.exists(), "fused finalization must remove the shard");
|
||||
assert!(!root.join(STORAGE_FORMAT_FILE).exists());
|
||||
assert!(!root.join(transaction.to_string()).exists());
|
||||
continue;
|
||||
}
|
||||
}
|
||||
let mut state = disk
|
||||
.inspect_delete_transaction("recovery", "object", transaction)
|
||||
.await
|
||||
.expect("inspect persisted state");
|
||||
let actions = if published {
|
||||
vec![DeleteRecoveryAction::FinishCommit, DeleteRecoveryAction::FinalizeCommit]
|
||||
} else {
|
||||
vec![DeleteRecoveryAction::Rollback]
|
||||
};
|
||||
for action in actions {
|
||||
state = disk
|
||||
.settle_delete_transaction("recovery", "object", transaction, state.fingerprint, action)
|
||||
.await
|
||||
.expect("recover shard deletion");
|
||||
}
|
||||
assert_eq!(state.state, DeleteRecoveryState::Absent);
|
||||
if published {
|
||||
assert!(!shard_dir.exists(), "committed deletion must remove the shard");
|
||||
} else {
|
||||
assert_eq!(fs::read(root.join(STORAGE_FORMAT_FILE)).await.expect("restore metadata"), original);
|
||||
assert_eq!(fs::read(shard_dir.join("part.1")).await.expect("restore shard"), encoded);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_uncommitted_inline_rolls_back_exact_bytes() {
|
||||
let (_temp, disk, _info, original) = fixture().await;
|
||||
|
||||
@@ -604,7 +604,7 @@ impl LocalDisk {
|
||||
let publication = os::acquire_namespace_mutation_lease_with_owner(&destination, Some(owner)).await;
|
||||
let tmp_volume = self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?;
|
||||
let temporary = self.io_get_object_path(RUSTFS_META_TMP_BUCKET, &Uuid::new_v4().to_string())?;
|
||||
self.write_all_internal(&temporary, InternalBuf::Ref(&replacement), SyncMode::FileOnly, &tmp_volume)
|
||||
self.write_all_internal(&temporary, replacement, SyncMode::FileOnly, &tmp_volume)
|
||||
.await?;
|
||||
os::rename_metadata_and_sync_with_lease(
|
||||
temporary,
|
||||
|
||||
@@ -30,15 +30,21 @@ pub mod local;
|
||||
pub mod multipart_source;
|
||||
pub mod os;
|
||||
|
||||
pub(crate) use io_admission::memory as background_memory;
|
||||
pub(crate) use io_admission::remote as remote_io_admission;
|
||||
pub use io_admission::{
|
||||
BackgroundMemoryBudget, PhysicalDiskBinding, PhysicalDiskPlan, PhysicalDiskSample, PhysicalDiskSampler, PhysicalResourcePlan,
|
||||
RemoteBackgroundIo, spawn_blocking_with_resource_ownership, with_background_disk_io, with_background_resource_set,
|
||||
with_background_resource_unit, with_remote_background_io,
|
||||
};
|
||||
pub use local::pool_meta_native::{PoolMetaNativeCasCondition, PoolMetaNativeFileIdentity};
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) use io_admission::DiskIoClass;
|
||||
pub(crate) use io_admission::{DiskIoClass, test_resource_units};
|
||||
pub(crate) use io_admission::{
|
||||
acquire_context_disk_io_permit, current_disk_io_context, current_resource_unit, with_optional_disk_io_context,
|
||||
with_optional_resource_unit,
|
||||
PhysicalHealBinding, acquire_context_disk_io_permit, current_disk_io_context, current_resource_unit, has_resource_io_scope,
|
||||
with_optional_disk_io_context, with_optional_resource_unit,
|
||||
};
|
||||
pub use io_admission::{with_background_disk_io, with_background_resource_unit};
|
||||
|
||||
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
|
||||
pub const MIGRATING_META_BUCKET: &str = ".minio.sys";
|
||||
@@ -305,6 +311,14 @@ pub enum Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// Physical ownership belongs to the node hosting the local attachment.
|
||||
pub fn physical_admission(&self) -> Option<Arc<PhysicalDiskBinding>> {
|
||||
match self {
|
||||
Self::Local(disk) => Some(disk.physical_admission()),
|
||||
Self::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// A read-only probe; the physical delete repeats the same leaf checks.
|
||||
pub async fn is_plain_orphan_data_dir(&self, volume: &str, path: &str) -> Result<bool> {
|
||||
match self {
|
||||
@@ -387,6 +401,39 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn try_publish_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_publish_prepared_delete_local(volume, object, transaction, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn try_finish_and_finalize_prepared_delete_local(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
transaction: Uuid,
|
||||
journal: &DeleteRecoveryJournal,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<bool> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.try_finish_and_finalize_prepared_delete_local(volume, object, transaction, journal, namespace_owner)
|
||||
.await
|
||||
}
|
||||
Self::Remote(_) => Ok(false),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_version_recoverable_with_namespace_owner(
|
||||
&self,
|
||||
volume: &str,
|
||||
|
||||
+130
-59
@@ -286,6 +286,11 @@ mod dir_fsync_failures {
|
||||
|
||||
static FAILURES: LazyLock<Mutex<HashMap<PathBuf, io::ErrorKind>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(super) fn has_any() -> bool {
|
||||
!FAILURES.lock().is_empty()
|
||||
}
|
||||
|
||||
pub(super) fn set(dir: &Path, kind: io::ErrorKind) {
|
||||
FAILURES.lock().insert(dir.to_path_buf(), kind);
|
||||
}
|
||||
@@ -323,7 +328,12 @@ pub fn set_local_dir_fsync_failure(disk: &crate::disk::Disk, volume: &str, path:
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Pause a real namespace mutation inside its physical executor.
|
||||
#[cfg(all(target_os = "linux", any(test, feature = "test-util")))]
|
||||
pub(crate) fn has_dir_fsync_test_failures() -> bool {
|
||||
dir_fsync_failures::has_any()
|
||||
}
|
||||
|
||||
/// Pause a real native filesystem operation inside its physical executor.
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
pub(crate) mod prepared_publication_test_hooks {
|
||||
use super::*;
|
||||
@@ -333,6 +343,7 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
NativeReadOpen,
|
||||
MetadataReadAfterOpen,
|
||||
MetadataReadAfterRead,
|
||||
#[cfg(test)]
|
||||
@@ -372,6 +383,17 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_any_hooks() -> bool {
|
||||
if !BEFORE_PUBLICATION.lock().is_empty() {
|
||||
return true;
|
||||
}
|
||||
#[cfg(test)]
|
||||
if !RENAME_DESTINATIONS.lock().is_empty() || !HARDLINK_DESTINATIONS.lock().is_empty() {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
type RenameDestinationHook = Box<dyn FnOnce(&Path) + Send>;
|
||||
#[cfg(test)]
|
||||
@@ -459,6 +481,8 @@ pub enum LocalPublicationStage {
|
||||
PreparedRename,
|
||||
Rename,
|
||||
Remove,
|
||||
/// Pause an accepted native read/open before its original syscall.
|
||||
NativeReadOpen,
|
||||
MetadataReadAfterOpen,
|
||||
MetadataReadAfterRead,
|
||||
}
|
||||
@@ -473,6 +497,7 @@ impl LocalPublicationPause {
|
||||
LocalPublicationStage::PreparedRename => prepared_publication_test_hooks::Stage::PreparedRename,
|
||||
LocalPublicationStage::Rename => prepared_publication_test_hooks::Stage::Rename,
|
||||
LocalPublicationStage::Remove => prepared_publication_test_hooks::Stage::Remove,
|
||||
LocalPublicationStage::NativeReadOpen => prepared_publication_test_hooks::Stage::NativeReadOpen,
|
||||
LocalPublicationStage::MetadataReadAfterOpen => prepared_publication_test_hooks::Stage::MetadataReadAfterOpen,
|
||||
LocalPublicationStage::MetadataReadAfterRead => prepared_publication_test_hooks::Stage::MetadataReadAfterRead,
|
||||
};
|
||||
@@ -758,7 +783,7 @@ fn set_file_fdatasync_group_commit_wait_for_test(wait_micros: u64) {
|
||||
file_fdatasync_group_commit_override::set_wait_micros(wait_micros);
|
||||
}
|
||||
|
||||
fn file_fdatasync_group_commit_enabled() -> bool {
|
||||
pub(crate) fn file_fdatasync_group_commit_enabled() -> bool {
|
||||
#[cfg(test)]
|
||||
if let Some(enabled) = file_fdatasync_group_commit_override::get() {
|
||||
return enabled;
|
||||
@@ -859,13 +884,13 @@ impl OpenedDstDirFsyncGroup {
|
||||
|
||||
struct DstDirFsyncWaiter {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
resource_unit: Option<rustfs_concurrency::adaptive::AdaptivePermit>,
|
||||
resource_unit: Option<super::io_admission::ResourceIoUnit>,
|
||||
result_tx: oneshot::Sender<SharedDstDirFsyncResult>,
|
||||
}
|
||||
|
||||
struct DstDirFsyncOwner {
|
||||
_namespace: Option<Arc<dyn Send + Sync>>,
|
||||
_resource: Option<rustfs_concurrency::adaptive::AdaptivePermit>,
|
||||
_resource: Option<super::io_admission::ResourceIoUnit>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -1307,7 +1332,7 @@ type FileFdatasyncGroupKey = (usize, bool);
|
||||
|
||||
struct FileFdatasyncWaiter {
|
||||
files: Vec<PathBuf>,
|
||||
resource_unit: Option<rustfs_concurrency::adaptive::AdaptivePermit>,
|
||||
resource_unit: Option<super::io_admission::ResourceIoUnit>,
|
||||
enqueued_at: Option<Instant>,
|
||||
wait_role: &'static str,
|
||||
result_tx: oneshot::Sender<SharedFileFdatasyncResult>,
|
||||
@@ -2392,6 +2417,10 @@ pub(crate) mod file_sync_probe {
|
||||
configure(root, None, false)
|
||||
}
|
||||
|
||||
pub(crate) fn is_active() -> bool {
|
||||
!ROOTS.read().expect("file sync probe lock poisoned").is_empty()
|
||||
}
|
||||
|
||||
pub(super) fn set_failing(root: &Path) -> ProbeGuard {
|
||||
configure(root, Some(1), false)
|
||||
}
|
||||
@@ -2553,18 +2582,25 @@ fn regular_files(dir: &Path) -> io::Result<Vec<PathBuf>> {
|
||||
|
||||
/// Fdatasync every regular file directly inside `dir`, then fsync the directory
|
||||
/// itself.
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "reached only through sync_dir_files, whose callers are tests (backlog#1823)"
|
||||
#[cfg_attr(
|
||||
not(target_os = "linux"),
|
||||
allow(
|
||||
dead_code,
|
||||
reason = "reached only through sync_dir_files, whose callers are tests (backlog#1823)"
|
||||
)
|
||||
)]
|
||||
pub fn sync_dir_files_std(dir: impl AsRef<Path>) -> io::Result<()> {
|
||||
for entry in std::fs::read_dir(dir.as_ref())? {
|
||||
let entry = entry?;
|
||||
if entry.file_type()?.is_file() {
|
||||
sync_file(&entry.path())?;
|
||||
}
|
||||
}
|
||||
fsync_dir_std(dir)
|
||||
let dir = dir.as_ref();
|
||||
let files = regular_files(dir)?;
|
||||
rustfs_io_metrics::record_put_rename_fdatasync_batch(rustfs_io_metrics::PUT_RENAME_FDATASYNC_BATCH_MODE_SERIAL, files.len());
|
||||
sync_files(&files)?;
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
let result = fsync_dir_std(dir);
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC,
|
||||
fsync_started,
|
||||
);
|
||||
result
|
||||
}
|
||||
|
||||
/// Async wrapper around [`sync_dir_files_std`]. Large directories flush files
|
||||
@@ -3808,6 +3844,23 @@ pub(crate) async fn rename_all_with_prepared_source(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
pub(crate) fn rename_all_with_commit_guard_on_thread(source: &Path, destination: &Path, guard: &RenameCommitGuard) -> Result<()> {
|
||||
guard.directories.prepare_destination().map_err(to_file_error)?;
|
||||
#[cfg(test)]
|
||||
prepared_publication_test_hooks::run_rename_destination(source, destination);
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
{
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, source);
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Rename, destination);
|
||||
}
|
||||
#[cfg(test)]
|
||||
multipart_source_syscall_failures::check(multipart_source_syscall_failures::Stage::Rename, destination)
|
||||
.map_err(to_file_error)?;
|
||||
guard.directories.rename(source, destination, None).map_err(to_file_error)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
pub(crate) async fn rename_all_with_commit_guard(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
@@ -6483,6 +6536,7 @@ pub(crate) fn sync_multipart_owned_record_parents(
|
||||
if !directory.starts_with(source) {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidInput, "native record barrier escaped source"));
|
||||
}
|
||||
#[cfg_attr(not(unix), allow(unused_mut))]
|
||||
let mut pinned = pin_multipart_source_directory(directory, base, publication_root)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
@@ -11100,54 +11154,71 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn resource_native_dispatch_retains_each_unit_after_caller_cancellation() {
|
||||
use crate::disk::io_admission::with_background_resource_unit;
|
||||
use rustfs_concurrency::adaptive::AdaptiveAdmission;
|
||||
use std::num::NonZeroU16;
|
||||
use crate::disk::io_admission::{test_resource_units, with_optional_resource_unit};
|
||||
|
||||
let mut retained = Vec::new();
|
||||
for lane in [None, Some(DiskBlockingLane::Foreground), Some(DiskBlockingLane::Fsync)] {
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::new(1).expect("one unit"), 1);
|
||||
let unit = gate.acquire().await;
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let (finished_tx, finished_rx) = oneshot::channel();
|
||||
let worker = tokio::spawn(with_background_resource_unit(unit, async move {
|
||||
let operation = move || {
|
||||
started_tx.send(()).expect("native operation entered");
|
||||
let _ = release_rx.blocking_recv();
|
||||
let _ = finished_tx.send(());
|
||||
Ok(())
|
||||
};
|
||||
match lane {
|
||||
Some(lane) => {
|
||||
DiskBlockingQueue::new()
|
||||
.run_on_lane(
|
||||
lane,
|
||||
"native operation failed",
|
||||
DiskBlockingQueue::test_stages_for_lane(lane),
|
||||
move |_| operation(),
|
||||
)
|
||||
.await
|
||||
for physical_count in [1, 2] {
|
||||
for lane in [
|
||||
None,
|
||||
Some(DiskBlockingLane::Foreground),
|
||||
Some(DiskBlockingLane::Isolated),
|
||||
Some(DiskBlockingLane::Fsync),
|
||||
] {
|
||||
let (unit, gates) = test_resource_units(physical_count);
|
||||
let (started_tx, started_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
let (finished_tx, finished_rx) = oneshot::channel();
|
||||
let worker = tokio::spawn(with_optional_resource_unit(Some(unit), async move {
|
||||
let operation = move || {
|
||||
started_tx.send(()).expect("native operation entered");
|
||||
let _ = release_rx.blocking_recv();
|
||||
let _ = finished_tx.send(());
|
||||
Ok(())
|
||||
};
|
||||
match lane {
|
||||
Some(lane) => {
|
||||
DiskBlockingQueue::new()
|
||||
.run_on_lane(
|
||||
lane,
|
||||
"native operation failed",
|
||||
DiskBlockingQueue::test_stages_for_lane(lane),
|
||||
move |_| operation(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
None => fsync_spawn_blocking(operation).await.expect("native operation joined"),
|
||||
}
|
||||
None => fsync_spawn_blocking(operation).await.expect("native operation joined"),
|
||||
}
|
||||
}));
|
||||
started_rx.await.expect("native operation started");
|
||||
worker.abort();
|
||||
assert!(worker.await.expect_err("caller cancelled").is_cancelled());
|
||||
retained.push(gate.snapshot().active);
|
||||
// Drain native work before assertions, including in the negative control.
|
||||
release_tx.send(()).expect("release native operation");
|
||||
finished_rx.await.expect("native operation finished");
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while gate.snapshot().active != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("native completion returns the unit");
|
||||
}));
|
||||
started_rx.await.expect("native operation started");
|
||||
worker.abort();
|
||||
assert!(worker.await.expect_err("caller cancelled").is_cancelled());
|
||||
retained.push(gates.iter().map(|gate| gate.snapshot().active).collect::<Vec<_>>());
|
||||
// Drain native work before assertions, including in the negative control.
|
||||
release_tx.send(()).expect("release native operation");
|
||||
finished_rx.await.expect("native operation finished");
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while gates.iter().any(|gate| gate.snapshot().active != 0) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("native completion returns the unit");
|
||||
}
|
||||
}
|
||||
assert_eq!(retained, [1, 1, 1], "fsync and both queue lanes must retain their own producer's unit");
|
||||
assert_eq!(
|
||||
retained,
|
||||
[
|
||||
vec![1],
|
||||
vec![1],
|
||||
vec![1],
|
||||
vec![1],
|
||||
vec![1, 1],
|
||||
vec![1, 1],
|
||||
vec![1, 1],
|
||||
vec![1, 1]
|
||||
],
|
||||
"native dispatch and all queue lanes must retain their own producer's units"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -76,14 +76,14 @@ impl DriverReadBudget {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn start_driver(
|
||||
pub(super) fn start_driver<O: Send + Unpin + 'static>(
|
||||
&self,
|
||||
entries: u32,
|
||||
shards: usize,
|
||||
) -> Result<rustfs_uring::UringDriver, rustfs_uring::ProbeFailure> {
|
||||
) -> Result<rustfs_uring::OwnedUringDriver<O>, rustfs_uring::ProbeFailure> {
|
||||
match self {
|
||||
Self::Disabled => rustfs_uring::UringDriver::probe_and_start_sharded(entries, shards),
|
||||
Self::Limited { pool, per_driver } => rustfs_uring::UringDriver::probe_and_start_with_shared_budget(
|
||||
Self::Disabled => rustfs_uring::OwnedUringDriver::probe_and_start_sharded(entries, shards),
|
||||
Self::Limited { pool, per_driver } => rustfs_uring::OwnedUringDriver::probe_and_start_with_shared_budget(
|
||||
entries,
|
||||
shards,
|
||||
rustfs_uring::ReadLimits {
|
||||
|
||||
@@ -98,6 +98,9 @@ pin_project! {
|
||||
// contiguous on-disk `[hash][data]` block so both are pulled in a single
|
||||
// pass; grown lazily and never shrunk.
|
||||
buf: Vec<u8>,
|
||||
// Free scratch storage before refunding its allocation.
|
||||
buf_memory: Option<crate::disk::background_memory::Reservation>,
|
||||
memory_budget: Option<std::sync::Arc<crate::disk::BackgroundMemoryBudget>>,
|
||||
// Reused owned chunk vector for the remote HTTP fast path. Keeping the
|
||||
// allocation with the reader avoids allocating once per bitrot block.
|
||||
chunks: Vec<bytes::Bytes>,
|
||||
@@ -118,6 +121,8 @@ where
|
||||
hash_algo: algo,
|
||||
shard_size,
|
||||
buf: Vec::new(),
|
||||
buf_memory: None,
|
||||
memory_budget: crate::disk::current_resource_unit().and_then(|unit| unit.memory_budget()),
|
||||
chunks: Vec::new(),
|
||||
skip_verify,
|
||||
last_verify_duration: Duration::ZERO,
|
||||
@@ -209,6 +214,7 @@ where
|
||||
/// On `Ok` the block is `self.buf[..need]`; the caller re-borrows it so the
|
||||
/// verification can also borrow `self` immutably.
|
||||
async fn read_scratch_block(&mut self, need: usize, want: usize) -> std::io::Result<()> {
|
||||
reserve_bitrot_scratch(&mut self.buf, &mut self.buf_memory, self.memory_budget.as_ref(), need)?;
|
||||
if self.buf.len() < need {
|
||||
self.buf.resize(need, 0);
|
||||
}
|
||||
@@ -229,6 +235,28 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Scratch growth happens only before a complete block or an empty chunk tail.
|
||||
// Old contents are consumed; keep both allocations charged until replacement.
|
||||
fn reserve_bitrot_scratch(
|
||||
buf: &mut Vec<u8>,
|
||||
reservation: &mut Option<crate::disk::background_memory::Reservation>,
|
||||
budget: Option<&std::sync::Arc<crate::disk::BackgroundMemoryBudget>>,
|
||||
capacity: usize,
|
||||
) -> std::io::Result<()> {
|
||||
if let Some(budget) = budget
|
||||
&& buf.capacity() < capacity
|
||||
{
|
||||
let memory = budget.reserve(capacity)?;
|
||||
let mut replacement = Vec::new();
|
||||
replacement
|
||||
.try_reserve_exact(capacity)
|
||||
.map_err(|_| std::io::Error::from(std::io::ErrorKind::OutOfMemory))?;
|
||||
*buf = replacement;
|
||||
*reservation = Some(memory);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A truncated shard is `UnexpectedEof`, not a short success (backlog#799 B2).
|
||||
fn short_shard_read(got: usize, want: usize) -> std::io::Error {
|
||||
error!(
|
||||
@@ -339,6 +367,8 @@ where
|
||||
let inner = &mut self.inner;
|
||||
let chunks = &mut self.chunks;
|
||||
let tail_buf = &mut self.buf;
|
||||
let tail_memory = &mut self.buf_memory;
|
||||
let memory_budget = self.memory_budget.as_ref();
|
||||
let mut received = 0usize;
|
||||
poll_fn(|cx| {
|
||||
for _ in 0..MAX_CHUNK_POLLS_PER_YIELD {
|
||||
@@ -381,7 +411,9 @@ where
|
||||
|
||||
if chunks.len() == MAX_RETAINED_CHUNKS_PER_BLOCK {
|
||||
if tail_buf.is_empty() {
|
||||
tail_buf.reserve_exact(need - (received - chunk.len()));
|
||||
let tail_capacity = need - (received - chunk.len());
|
||||
reserve_bitrot_scratch(tail_buf, tail_memory, memory_budget, tail_capacity)?;
|
||||
tail_buf.reserve_exact(tail_capacity);
|
||||
}
|
||||
tail_buf.extend_from_slice(&chunk);
|
||||
} else {
|
||||
@@ -657,7 +689,12 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
|
||||
algo: HashAlgorithm,
|
||||
mut shard_size: usize,
|
||||
) -> std::io::Result<()> {
|
||||
let mut hash_buf = vec![0; algo.size()];
|
||||
// Every supported digest fits on the stack; only shard payloads need leases.
|
||||
let mut hash = [0; 64];
|
||||
let hash_buf = &mut hash[..algo.size()];
|
||||
let mut background_buffer = crate::disk::current_resource_unit()
|
||||
.and_then(|unit| unit.memory_budget())
|
||||
.map(|budget| crate::disk::background_memory::BackgroundBuffer::new(Some(budget)));
|
||||
let mut left = want_size;
|
||||
|
||||
if left != bitrot_shard_file_size(part_size, shard_size, algo.clone()) {
|
||||
@@ -665,21 +702,30 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
|
||||
}
|
||||
|
||||
while left > 0 {
|
||||
let n = r.read_exact(&mut hash_buf).await?;
|
||||
let n = r.read_exact(hash_buf).await?;
|
||||
left -= n;
|
||||
|
||||
if left < shard_size {
|
||||
shard_size = left;
|
||||
}
|
||||
|
||||
let mut buf = get_ec_buffer(shard_size);
|
||||
buf.resize(shard_size, 0);
|
||||
let read = r.read_exact(&mut buf).await?;
|
||||
let mut pooled = None;
|
||||
let buf = if let Some(buffer) = background_buffer.as_mut() {
|
||||
buffer.ensure_len(shard_size)?;
|
||||
&mut buffer.as_mut_slice()[..shard_size]
|
||||
} else {
|
||||
let buffer = pooled.insert(get_ec_buffer(shard_size));
|
||||
buffer.resize(shard_size, 0);
|
||||
buffer.as_mut_slice()
|
||||
};
|
||||
let read = r.read_exact(buf).await?;
|
||||
|
||||
let actual_hash = algo.hash_encode(&buf);
|
||||
let actual_hash = algo.hash_encode(buf);
|
||||
let hash_ok = actual_hash.as_ref() == &hash_buf[0..n];
|
||||
drop(actual_hash); // 释放借用
|
||||
return_ec_buffer(buf);
|
||||
if let Some(buf) = pooled {
|
||||
return_ec_buffer(buf);
|
||||
}
|
||||
if !hash_ok {
|
||||
return Err(std::io::Error::other("bitrot hash mismatch"));
|
||||
}
|
||||
@@ -698,15 +744,74 @@ pub async fn bitrot_verify<R: AsyncRead + Unpin + Send>(
|
||||
/// Custom writer enum that supports inline buffer storage
|
||||
pub enum CustomWriter {
|
||||
/// Inline buffer writer - stores data in memory
|
||||
InlineBuffer(Vec<u8>),
|
||||
InlineBuffer(InlineBuffer),
|
||||
/// Disk-based writer using tokio file
|
||||
Other(Box<dyn AsyncWrite + Unpin + Send + Sync>),
|
||||
}
|
||||
|
||||
/// Inline payload storage retains its background allocation through export.
|
||||
pub struct InlineBuffer {
|
||||
foreground: Vec<u8>,
|
||||
background: Option<crate::disk::background_memory::BackgroundBuffer>,
|
||||
filled: usize,
|
||||
}
|
||||
|
||||
impl InlineBuffer {
|
||||
fn new() -> Self {
|
||||
let background = crate::disk::current_resource_unit()
|
||||
.and_then(|unit| unit.memory_budget())
|
||||
.map(|budget| crate::disk::background_memory::BackgroundBuffer::new(Some(budget)));
|
||||
Self {
|
||||
foreground: Vec::new(),
|
||||
background,
|
||||
filled: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn as_slice(&self) -> &[u8] {
|
||||
match &self.background {
|
||||
Some(buffer) => &buffer.as_slice()[..self.filled],
|
||||
None => &self.foreground,
|
||||
}
|
||||
}
|
||||
|
||||
fn append(&mut self, bufs: &[IoSlice<'_>]) -> std::io::Result<usize> {
|
||||
let total = bufs
|
||||
.iter()
|
||||
.try_fold(0usize, |length, buf| length.checked_add(buf.len()).ok_or(std::io::ErrorKind::OutOfMemory))?;
|
||||
if let Some(buffer) = &mut self.background {
|
||||
let length = self.filled.checked_add(total).ok_or(std::io::ErrorKind::OutOfMemory)?;
|
||||
if length > buffer.as_slice().len() {
|
||||
let capacity = buffer.as_slice().len().checked_mul(2).unwrap_or(length).max(length);
|
||||
// Admission includes both allocations until the prefix has been copied.
|
||||
buffer.grow(capacity, self.filled)?;
|
||||
}
|
||||
let mut offset = self.filled;
|
||||
for buf in bufs {
|
||||
buffer.as_mut_slice()[offset..offset + buf.len()].copy_from_slice(buf);
|
||||
offset += buf.len();
|
||||
}
|
||||
self.filled = length;
|
||||
} else {
|
||||
for buf in bufs {
|
||||
self.foreground.extend_from_slice(buf);
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
fn into_bytes(self) -> bytes::Bytes {
|
||||
match self.background {
|
||||
Some(mut buffer) => buffer.take_bytes(self.filled),
|
||||
None => bytes::Bytes::from(self.foreground),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CustomWriter {
|
||||
/// Create a new inline buffer writer
|
||||
pub fn new_inline_buffer() -> Self {
|
||||
Self::InlineBuffer(Vec::new())
|
||||
Self::InlineBuffer(InlineBuffer::new())
|
||||
}
|
||||
|
||||
/// Create a new disk writer from any AsyncWrite implementation
|
||||
@@ -720,15 +825,15 @@ impl CustomWriter {
|
||||
/// Get the inline buffer data if this is an inline buffer writer
|
||||
pub fn get_inline_data(&self) -> Option<&[u8]> {
|
||||
match self {
|
||||
Self::InlineBuffer(data) => Some(data),
|
||||
Self::InlineBuffer(data) => Some(data.as_slice()),
|
||||
Self::Other(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the inline buffer data, consuming the writer
|
||||
pub fn into_inline_data(self) -> Option<Vec<u8>> {
|
||||
pub fn into_inline_data(self) -> Option<bytes::Bytes> {
|
||||
match self {
|
||||
Self::InlineBuffer(data) => Some(data),
|
||||
Self::InlineBuffer(data) => Some(data.into_bytes()),
|
||||
Self::Other(_) => None,
|
||||
}
|
||||
}
|
||||
@@ -741,10 +846,7 @@ impl AsyncWrite for CustomWriter {
|
||||
buf: &[u8],
|
||||
) -> std::task::Poll<std::io::Result<usize>> {
|
||||
match self.get_mut() {
|
||||
Self::InlineBuffer(data) => {
|
||||
data.extend_from_slice(buf);
|
||||
std::task::Poll::Ready(Ok(buf.len()))
|
||||
}
|
||||
Self::InlineBuffer(data) => std::task::Poll::Ready(data.append(&[IoSlice::new(buf)])),
|
||||
Self::Other(writer) => {
|
||||
let pinned_writer = std::pin::Pin::new(writer.as_mut());
|
||||
pinned_writer.poll_write(cx, buf)
|
||||
@@ -778,13 +880,7 @@ impl AsyncWrite for CustomWriter {
|
||||
bufs: &[IoSlice<'_>],
|
||||
) -> std::task::Poll<std::io::Result<usize>> {
|
||||
match self.get_mut() {
|
||||
Self::InlineBuffer(data) => {
|
||||
let total = bufs.iter().map(|buf| buf.len()).sum::<usize>();
|
||||
for buf in bufs {
|
||||
data.extend_from_slice(buf);
|
||||
}
|
||||
std::task::Poll::Ready(Ok(total))
|
||||
}
|
||||
Self::InlineBuffer(data) => std::task::Poll::Ready(data.append(bufs)),
|
||||
Self::Other(writer) => {
|
||||
let pinned_writer = std::pin::Pin::new(writer.as_mut());
|
||||
pinned_writer.poll_write_vectored(cx, bufs)
|
||||
@@ -859,7 +955,7 @@ impl BitrotWriterWrapper {
|
||||
}
|
||||
|
||||
/// Extract the inline buffer data, consuming the wrapper
|
||||
pub fn into_inline_data(self) -> Option<Vec<u8>> {
|
||||
pub fn into_inline_data(self) -> Option<bytes::Bytes> {
|
||||
match self.writer_type {
|
||||
WriterType::InlineBuffer => {
|
||||
let writer = self.bitrot_writer.into_inner();
|
||||
@@ -1506,6 +1602,184 @@ mod tests {
|
||||
assert_eq!(data, &out[..]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_reader_scratch_budget_preserves_growth_and_reuse() {
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let mut encoded = Vec::new();
|
||||
for data in [&b"abcd"[..], &b"abcdefgh"[..], &b"ijklmnop"[..]] {
|
||||
encoded.extend_from_slice(algo.hash_encode(data).as_ref());
|
||||
encoded.extend_from_slice(data);
|
||||
}
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut reader = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BitrotReader::new(Cursor::new(encoded), 8, algo, false)
|
||||
})
|
||||
.await;
|
||||
let mut first = [0; 4];
|
||||
assert_eq!(
|
||||
reader.read(&mut first).await.expect_err("closed scratch budget").kind(),
|
||||
io::ErrorKind::WouldBlock
|
||||
);
|
||||
assert_eq!(reader.inner.position(), 0, "allocation denial must not consume a frame");
|
||||
memory.set_sampled_limit(36, Duration::from_secs(30));
|
||||
reader.read(&mut first).await.expect("first scratch allocation fits");
|
||||
assert_eq!(&first, b"abcd");
|
||||
assert_eq!(memory.snapshot().used, 36);
|
||||
memory.set_sampled_limit(75, Duration::from_secs(30));
|
||||
let mut next = [0; 8];
|
||||
assert_eq!(
|
||||
reader
|
||||
.read(&mut next)
|
||||
.await
|
||||
.expect_err("growth requires old36 plus new40")
|
||||
.kind(),
|
||||
io::ErrorKind::WouldBlock
|
||||
);
|
||||
assert_eq!(reader.inner.position(), 36);
|
||||
assert_eq!(memory.snapshot().used, 36);
|
||||
memory.set_sampled_limit(76, Duration::from_secs(30));
|
||||
reader.read(&mut next).await.expect("growth now fits");
|
||||
assert_eq!(&next, b"abcdefgh");
|
||||
assert_eq!(memory.snapshot().used, 40);
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
reader.read(&mut next).await.expect("reuse after hot shrink");
|
||||
assert_eq!(&next, b"ijklmnop");
|
||||
assert_eq!(memory.snapshot().used, 40);
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_chunk_tail_allocation_stays_charged_until_reader_drop() {
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let payload = vec![37; 256];
|
||||
let encoded = encode_one_block(&payload, payload.len(), algo.clone()).await;
|
||||
let tail_capacity = encoded.len() - MAX_RETAINED_CHUNKS_PER_BLOCK;
|
||||
for fits in [false, true] {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(if fits { tail_capacity as u64 } else { 0 }, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut reader = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
BitrotReader::new(GeneratedChunkSource::new(encoded.clone(), 1), payload.len(), algo.clone(), false)
|
||||
})
|
||||
.await;
|
||||
let mut output = Vec::with_capacity(payload.len());
|
||||
let result = reader.read_appending(&mut output, payload.len()).await;
|
||||
if fits {
|
||||
assert_eq!(result.expect("accounted tail fits"), payload.len());
|
||||
assert_eq!(output, payload);
|
||||
assert_eq!(memory.snapshot().used, tail_capacity as u64);
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
assert_eq!(memory.snapshot().used, tail_capacity as u64);
|
||||
} else {
|
||||
assert_eq!(result.expect_err("closed tail budget").kind(), io::ErrorKind::WouldBlock);
|
||||
assert!(output.is_empty(), "an unverified fragment must not reach output");
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_verify_background_memory_is_bounded_and_reused() {
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
|
||||
struct ShrinkingReader {
|
||||
data: Cursor<Vec<u8>>,
|
||||
memory: Arc<BackgroundMemoryBudget>,
|
||||
first_block_end: u64,
|
||||
peak: Arc<std::sync::atomic::AtomicU64>,
|
||||
}
|
||||
impl AsyncRead for ShrinkingReader {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, out: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
self.peak.fetch_max(self.memory.snapshot().used, Ordering::SeqCst);
|
||||
let result = Pin::new(&mut self.data).poll_read(cx, out);
|
||||
if self.data.position() >= self.first_block_end {
|
||||
self.memory.set_sampled_limit(0, Duration::ZERO);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
for algo in [
|
||||
HashAlgorithm::None,
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
HashAlgorithm::HighwayHash256SLegacy,
|
||||
] {
|
||||
let payload = [37; 17];
|
||||
let mut writer = BitrotWriter::new(Cursor::new(Vec::new()), 8, algo.clone());
|
||||
for chunk in payload.chunks(8) {
|
||||
writer.write(chunk).await.expect("encoded bitrot block");
|
||||
}
|
||||
writer.shutdown().await.expect("complete fixture");
|
||||
let encoded = writer.into_inner().into_inner();
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let error = with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
bitrot_verify(Cursor::new(&encoded), encoded.len(), payload.len(), algo.clone(), 8),
|
||||
)
|
||||
.await
|
||||
.expect_err("a closed background budget must reject bitrot payload allocation");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
memory.set_sampled_limit(8, Duration::from_secs(30));
|
||||
let peak = Arc::new(std::sync::atomic::AtomicU64::new(0));
|
||||
let reader = ShrinkingReader {
|
||||
data: Cursor::new(encoded.clone()),
|
||||
memory: memory.clone(),
|
||||
first_block_end: (algo.size() + 8) as u64,
|
||||
peak: peak.clone(),
|
||||
};
|
||||
let (resources, _) = test_resource_units(1);
|
||||
with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
bitrot_verify(reader, encoded.len(), payload.len(), algo, 8),
|
||||
)
|
||||
.await
|
||||
.expect("reuse the same allocation after shrink, including the short tail");
|
||||
assert_eq!(peak.load(Ordering::SeqCst), 8);
|
||||
assert_eq!(memory.snapshot().limit, 0);
|
||||
assert_eq!(
|
||||
memory.snapshot().used,
|
||||
0,
|
||||
"the verifier frees its payload instead of retaining it in an uncharged pool"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_verify_cancelled_payload_read_refunds_memory() {
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
|
||||
let algo = HashAlgorithm::HighwayHash256S;
|
||||
let (mut sender, reader) = tokio::io::duplex(64);
|
||||
sender
|
||||
.write_all(algo.hash_encode(&[37; 8]).as_ref())
|
||||
.await
|
||||
.expect("send only hash");
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(8, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut verify = Box::pin(with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
bitrot_verify(reader, algo.size() + 8, 8, algo, 8),
|
||||
));
|
||||
assert!(futures::poll!(&mut verify).is_pending(), "the payload read waits after the complete hash");
|
||||
let retained = memory.snapshot().used;
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
drop(verify);
|
||||
drop(sender);
|
||||
assert_eq!(retained, 8, "pending verification owns its payload allocation");
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bitrot_verify_accepts_valid_shard_file_and_rejects_size_or_hash_mismatch() {
|
||||
let data = b"bitrot verify covers every shard";
|
||||
@@ -2403,4 +2677,128 @@ mod tests {
|
||||
assert_eq!(output, vec![9u8]);
|
||||
}
|
||||
}
|
||||
|
||||
mod inline_memory_regressions {
|
||||
use super::*;
|
||||
use crate::disk::background_memory::is_memory_pressure;
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
async fn writer_for(memory: Arc<BackgroundMemoryBudget>) -> CustomWriter {
|
||||
let (resources, _) = test_resource_units(1);
|
||||
with_optional_resource_unit(Some(resources.with_memory_budget(memory)), async { CustomWriter::new_inline_buffer() })
|
||||
.await
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_writer_rejects_closed_and_expired_owners_before_first_allocation() {
|
||||
for expired in [false, true] {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
if expired {
|
||||
memory.set_sampled_limit(128, Duration::ZERO);
|
||||
}
|
||||
let mut writer = writer_for(memory.clone()).await;
|
||||
let error = writer.write_all(&[7; 64]).await.expect_err("no live grant");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert!(is_memory_pressure(&error));
|
||||
assert_eq!(writer.get_inline_data(), Some(&[][..]));
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_writer_growth_charges_old_plus_new_and_is_atomic_for_vectored_writes() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(128, Duration::from_secs(30));
|
||||
let mut writer = writer_for(memory.clone()).await;
|
||||
writer.write_all(&[1; 64]).await.expect("first allocation");
|
||||
assert_eq!(memory.snapshot().used, 64);
|
||||
let second = [2; 32];
|
||||
let third = [3; 32];
|
||||
let chunks = [IoSlice::new(&second), IoSlice::new(&third)];
|
||||
let error = writer.write_vectored(&chunks).await.expect_err("64 old + 128 new exceed 128");
|
||||
assert!(is_memory_pressure(&error));
|
||||
assert_eq!(writer.get_inline_data(), Some(&[1; 64][..]));
|
||||
assert_eq!(memory.snapshot().used, 64);
|
||||
memory.set_sampled_limit(192, Duration::from_secs(30));
|
||||
assert_eq!(writer.write_vectored(&chunks).await.expect("old and new both fit"), 64);
|
||||
assert_eq!(memory.snapshot().used, 128);
|
||||
let data = writer.get_inline_data().expect("inline payload");
|
||||
assert_eq!(&data[..64], &[1; 64]);
|
||||
assert_eq!(&data[64..96], &second);
|
||||
assert_eq!(&data[96..], &third);
|
||||
drop(writer);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_writer_expiry_preserves_charged_storage_but_blocks_growth() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(192, Duration::from_secs(30));
|
||||
let mut writer = writer_for(memory.clone()).await;
|
||||
writer.write_all(&[5; 64]).await.expect("existing storage");
|
||||
memory.set_sampled_limit(192, Duration::ZERO);
|
||||
let error = writer
|
||||
.write_all(&[9])
|
||||
.await
|
||||
.expect_err("expired sample cannot authorize replacement");
|
||||
assert!(is_memory_pressure(&error));
|
||||
assert_eq!(writer.get_inline_data(), Some(&[5; 64][..]));
|
||||
assert_eq!(memory.snapshot().used, 64);
|
||||
writer.write_all(&[]).await.expect("empty write needs no allocation");
|
||||
writer.flush().await.expect("flush does not revoke storage");
|
||||
writer.shutdown().await.expect("shutdown does not revoke storage");
|
||||
let bytes = writer.into_inline_data().expect("export after expiry");
|
||||
assert_eq!(bytes.as_ref(), &[5; 64]);
|
||||
assert_eq!(memory.snapshot().used, 64);
|
||||
drop(bytes);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_writer_export_keeps_receipt_through_clones_and_the_last_slice() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(64, Duration::from_secs(30));
|
||||
let mut writer = writer_for(memory.clone()).await;
|
||||
writer.write_all(&[11; 64]).await.expect("allocate payload");
|
||||
let bytes = writer.into_inline_data().expect("export bytes");
|
||||
let clone = bytes.clone();
|
||||
let slice = bytes.slice(3..7);
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
drop(bytes);
|
||||
drop(clone);
|
||||
assert_eq!(slice.as_ref(), &[11; 4]);
|
||||
assert_eq!(memory.snapshot().used, 64);
|
||||
drop(slice);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_writer_keeps_the_constructing_owner_after_leaving_its_scope() {
|
||||
let first = BackgroundMemoryBudget::new();
|
||||
first.set_sampled_limit(64, Duration::from_secs(30));
|
||||
let mut writer = writer_for(first.clone()).await;
|
||||
let second = BackgroundMemoryBudget::new();
|
||||
let (resources, _) = test_resource_units(1);
|
||||
with_optional_resource_unit(Some(resources.with_memory_budget(second.clone())), async {
|
||||
writer.write_all(&[13; 64]).await.expect("writer still owns the first grant");
|
||||
})
|
||||
.await;
|
||||
assert_eq!(first.snapshot().used, 64);
|
||||
assert_eq!(second.snapshot().used, 0);
|
||||
drop(writer);
|
||||
assert_eq!(first.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn foreground_inline_writer_preserves_payload_and_empty_writes() {
|
||||
let mut writer = CustomWriter::new_inline_buffer();
|
||||
writer.write_all(&[]).await.expect("empty foreground write");
|
||||
writer.write_all(b"inline body").await.expect("foreground write");
|
||||
let bytes = writer.into_inline_data().expect("foreground export");
|
||||
assert_eq!(bytes.as_ref(), b"inline body");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,8 +175,10 @@ const DEFAULT_RUSTFS_GET_BITROT_DECODE_OVERLAP_ENABLE: bool = false;
|
||||
/// stripe via their stripe handle — only when a data shard is missing or dies.
|
||||
/// This halves per-GET read bytes, IOPS, and bitrot hashing on healthy
|
||||
/// objects with 2+2 layouts.
|
||||
/// Default: false (current behavior: every live shard reader is read on every
|
||||
/// stripe).
|
||||
/// Default: false for raw `Erasure::decode` calls without deferred parity
|
||||
/// handles; `DataBlocksFirst` reader setup attaches deferred parity handles and
|
||||
/// leaves those parity reserves unengaged by default unless this env var is set
|
||||
/// to `false`.
|
||||
const ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE: &str = "RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE";
|
||||
const DEFAULT_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE: bool = false;
|
||||
|
||||
@@ -189,6 +191,13 @@ pub(crate) fn get_lockstep_data_shards_only_enabled() -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
/// Whether `DataBlocksFirst` reader setup and `with_deferred_parity_handles`
|
||||
/// should keep parity shards deferred until needed on healthy GETs.
|
||||
pub(crate) fn get_lockstep_data_shards_only_with_deferred_parity_enabled() -> bool {
|
||||
matches!(decode_read_policy(), DecodeReadPolicy::DemandBound)
|
||||
|| rustfs_utils::get_env_bool(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, true)
|
||||
}
|
||||
|
||||
/// Get whether bitrot-decode overlap is enabled.
|
||||
fn is_bitrot_decode_overlap_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(
|
||||
@@ -832,6 +841,13 @@ where
|
||||
pub(crate) fn with_deferred_parity_handles(mut self, mut handles: Vec<Option<DeferredReaderStripeHandle>>) -> Self {
|
||||
handles.resize_with(self.readers.len(), || None);
|
||||
handles.truncate(self.readers.len());
|
||||
if get_lockstep_data_shards_only_with_deferred_parity_enabled() {
|
||||
for (index, handle) in handles.iter().enumerate().skip(self.data_shards) {
|
||||
if handle.is_some() {
|
||||
self.engaged[index] = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
self.deferred_handles = handles;
|
||||
self
|
||||
}
|
||||
@@ -2596,7 +2612,8 @@ impl Erasure {
|
||||
// the ordinary GET read quorum. Only the opt-in GET rollout requires a
|
||||
// surplus source; available surplus shards and integrity proofs are
|
||||
// still verified before any reconstructed copy bytes are emitted.
|
||||
let require_surplus_source = reader.demand_bound_lockstep && matches!(decode_read_policy(), DecodeReadPolicy::Default);
|
||||
let require_surplus_source =
|
||||
get_lockstep_data_shards_only_enabled() && matches!(decode_read_policy(), DecodeReadPolicy::Default);
|
||||
|
||||
let start = offset / self.block_size;
|
||||
let end = end_offset.saturating_sub(1) / self.block_size;
|
||||
@@ -4367,9 +4384,9 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Compatibility lock: with the gate off (default), the lockstep path
|
||||
/// keeps the pre-backlog#923 behavior and reads every live shard —
|
||||
/// including parity — on every stripe.
|
||||
/// Compatibility lock: with the gate off (default), raw `Erasure::decode`
|
||||
/// without deferred parity handles keeps the pre-backlog#923 behavior and
|
||||
/// reads every live shard — including parity — on every stripe.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_lockstep_default_reads_all_shards_per_stripe() {
|
||||
|
||||
@@ -16,6 +16,8 @@
|
||||
//! Supports legacy (reed-solomon-simd) for reading/healing old-version files.
|
||||
//!
|
||||
|
||||
use crate::disk::background_memory::{BackgroundBuffer, Reservation};
|
||||
use crate::disk::{BackgroundMemoryBudget, current_resource_unit};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use reed_solomon_simd;
|
||||
use rustfs_erasure_codec::galois_8::ReedSolomon;
|
||||
@@ -144,12 +146,54 @@ pub fn calc_shard_size_legacy(block_size: usize, data_shards: usize) -> usize {
|
||||
|
||||
/// Reed-Solomon encoder for legacy (main branch) format using reed-solomon-simd.
|
||||
/// Used when decoding/encoding files with uses_legacy_checksum == true.
|
||||
struct LegacyWorkspace<T> {
|
||||
codec: T,
|
||||
// The cached payload is freed before its reservation.
|
||||
memory: Option<Reservation>,
|
||||
payload_bound: usize,
|
||||
}
|
||||
|
||||
impl<T> LegacyWorkspace<T> {
|
||||
fn prepare(
|
||||
cached: Option<Self>,
|
||||
payload_bound: usize,
|
||||
budget: Option<&Arc<BackgroundMemoryBudget>>,
|
||||
create: impl FnOnce() -> io::Result<T>,
|
||||
reset: impl FnOnce(&mut T) -> io::Result<()>,
|
||||
) -> io::Result<Self> {
|
||||
if let Some(mut cached) = cached {
|
||||
let reusable = match (&cached.memory, budget) {
|
||||
(None, None) => true,
|
||||
(Some(memory), budget) => {
|
||||
payload_bound <= cached.payload_bound && budget.is_none_or(|budget| memory.belongs_to(budget))
|
||||
}
|
||||
// Uncharged foreground caches can have larger retained Vec
|
||||
// capacities after reset; start a measured background workspace.
|
||||
(None, Some(_)) => false,
|
||||
};
|
||||
if reusable && reset(&mut cached.codec).is_ok() {
|
||||
return Ok(cached);
|
||||
}
|
||||
// A fresh allocation avoids hidden geometric Vec growth. No payload
|
||||
// references escape codec calls, so dropping this cache is safe.
|
||||
drop(cached);
|
||||
}
|
||||
let memory = budget.map(|budget| budget.reserve(payload_bound)).transpose()?;
|
||||
let codec = create()?;
|
||||
Ok(Self {
|
||||
codec,
|
||||
memory,
|
||||
payload_bound,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct LegacyReedSolomonEncoder {
|
||||
data_shards: usize,
|
||||
parity_shards: usize,
|
||||
cache_workspaces: bool,
|
||||
encoder_cache: RwLock<Option<reed_solomon_simd::ReedSolomonEncoder>>,
|
||||
decoder_cache: RwLock<Option<reed_solomon_simd::ReedSolomonDecoder>>,
|
||||
encoder_cache: RwLock<Option<LegacyWorkspace<reed_solomon_simd::ReedSolomonEncoder>>>,
|
||||
decoder_cache: RwLock<Option<LegacyWorkspace<reed_solomon_simd::ReedSolomonDecoder>>>,
|
||||
}
|
||||
|
||||
impl LegacyReedSolomonEncoder {
|
||||
@@ -199,20 +243,29 @@ impl LegacyReedSolomonEncoder {
|
||||
.write()
|
||||
.map_err(|_| io::Error::other("Failed to acquire encoder cache lock"))?
|
||||
.take();
|
||||
let mut encoder = {
|
||||
match cached_encoder {
|
||||
Some(mut cached) => {
|
||||
if cached.reset(self.data_shards, self.parity_shards, shard_len).is_err() {
|
||||
reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
|
||||
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?
|
||||
} else {
|
||||
cached
|
||||
}
|
||||
}
|
||||
None => reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
|
||||
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))?,
|
||||
}
|
||||
};
|
||||
let budget = current_resource_unit().and_then(|resources| resources.memory_budget());
|
||||
// reed-solomon-simd 3.1.0 Shards::resize allocates 64-byte units. A
|
||||
// fresh Vec has at least four units; the decoder work-count bound also
|
||||
// covers both encoder rate strategies.
|
||||
let payload_bound = self
|
||||
.logical_shard_bytes_upper_bound(shard_len)
|
||||
.ok_or(io::ErrorKind::OutOfMemory)?
|
||||
.max(256);
|
||||
let mut workspace = LegacyWorkspace::prepare(
|
||||
cached_encoder,
|
||||
payload_bound,
|
||||
budget.as_ref(),
|
||||
|| {
|
||||
reed_solomon_simd::ReedSolomonEncoder::new(self.data_shards, self.parity_shards, shard_len)
|
||||
.map_err(|e| io::Error::other(format!("Failed to create SIMD encoder: {e:?}")))
|
||||
},
|
||||
|encoder| {
|
||||
encoder
|
||||
.reset(self.data_shards, self.parity_shards, shard_len)
|
||||
.map_err(|e| io::Error::other(format!("Failed to reset SIMD encoder: {e:?}")))
|
||||
},
|
||||
)?;
|
||||
let encoder = &mut workspace.codec;
|
||||
for (i, shard) in shards.iter().enumerate().take(self.data_shards) {
|
||||
encoder
|
||||
.add_original_shard(shard)
|
||||
@@ -233,7 +286,7 @@ impl LegacyReedSolomonEncoder {
|
||||
.write()
|
||||
.map_err(|_| io::Error::other("Failed to return encoder to cache"))?;
|
||||
if cache.is_none() {
|
||||
*cache = Some(encoder);
|
||||
*cache = Some(workspace);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -254,21 +307,26 @@ impl LegacyReedSolomonEncoder {
|
||||
.write()
|
||||
.map_err(|_| io::Error::other("Failed to acquire decoder cache lock"))?
|
||||
.take();
|
||||
let mut decoder = {
|
||||
match cached_decoder {
|
||||
Some(mut cached_decoder) => {
|
||||
if let Err(e) = cached_decoder.reset(self.data_shards, self.parity_shards, shard_len) {
|
||||
warn!("Failed to reset SIMD decoder: {:?}, creating new one", e);
|
||||
reed_solomon_simd::ReedSolomonDecoder::new(self.data_shards, self.parity_shards, shard_len)
|
||||
.map_err(|e| io::Error::other(format!("Failed to create SIMD decoder: {e:?}")))?
|
||||
} else {
|
||||
cached_decoder
|
||||
}
|
||||
}
|
||||
None => reed_solomon_simd::ReedSolomonDecoder::new(self.data_shards, self.parity_shards, shard_len)
|
||||
.map_err(|e| io::Error::other(format!("Failed to create SIMD decoder: {e:?}")))?,
|
||||
}
|
||||
};
|
||||
let budget = current_resource_unit().and_then(|resources| resources.memory_budget());
|
||||
let payload_bound = self
|
||||
.logical_shard_bytes_upper_bound(shard_len)
|
||||
.ok_or(io::ErrorKind::OutOfMemory)?
|
||||
.max(256);
|
||||
let mut workspace = LegacyWorkspace::prepare(
|
||||
cached_decoder,
|
||||
payload_bound,
|
||||
budget.as_ref(),
|
||||
|| {
|
||||
reed_solomon_simd::ReedSolomonDecoder::new(self.data_shards, self.parity_shards, shard_len)
|
||||
.map_err(|e| io::Error::other(format!("Failed to create SIMD decoder: {e:?}")))
|
||||
},
|
||||
|decoder| {
|
||||
decoder
|
||||
.reset(self.data_shards, self.parity_shards, shard_len)
|
||||
.map_err(|e| io::Error::other(format!("Failed to reset SIMD decoder: {e:?}")))
|
||||
},
|
||||
)?;
|
||||
let decoder = &mut workspace.codec;
|
||||
|
||||
for (i, shard_opt) in shards.iter().enumerate() {
|
||||
if let Some(shard) = shard_opt {
|
||||
@@ -308,7 +366,7 @@ impl LegacyReedSolomonEncoder {
|
||||
.write()
|
||||
.map_err(|_| io::Error::other("Failed to return decoder to cache"))?;
|
||||
if cache.is_none() {
|
||||
*cache = Some(decoder);
|
||||
*cache = Some(workspace);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -333,6 +391,13 @@ impl LegacyReedSolomonEncoder {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
let expected_bytes = shards.iter().try_fold(0usize, |total, shard| {
|
||||
total
|
||||
.checked_add(shard.len())
|
||||
.ok_or_else(|| io::Error::from(io::ErrorKind::OutOfMemory))
|
||||
})?;
|
||||
let memory = current_resource_unit().and_then(|resources| resources.memory_budget());
|
||||
let _expected_memory = memory.as_ref().map(|memory| memory.reserve(expected_bytes)).transpose()?;
|
||||
let mut expected = shards.iter().map(|shard| Some(shard.to_vec())).collect::<Vec<_>>();
|
||||
self.encode_parity(&mut expected)?;
|
||||
|
||||
@@ -431,8 +496,18 @@ impl ReedSolomonEncoder {
|
||||
}
|
||||
|
||||
if let Some(ref rs) = self.encoder {
|
||||
rs.verify(shards)
|
||||
.map_err(|e| io::Error::other(format!("Reed-Solomon verify failed: {e:?}")))
|
||||
let result = if let Some(memory) = current_resource_unit().and_then(|resources| resources.memory_budget())
|
||||
&& let Some(shard_len) = shards.first().map(|shard| shard.len()).filter(|len| *len != 0)
|
||||
{
|
||||
let scratch_len = shard_len.checked_mul(self.parity_shards).ok_or(io::ErrorKind::OutOfMemory)?;
|
||||
let mut scratch = BackgroundBuffer::new(Some(memory));
|
||||
scratch.ensure_len(scratch_len)?;
|
||||
let mut buffers: SmallVec<[&mut [u8]; 16]> = scratch.as_mut_slice().chunks_mut(shard_len).collect();
|
||||
rs.verify_with_buffer(shards, &mut buffers)
|
||||
} else {
|
||||
rs.verify(shards)
|
||||
};
|
||||
result.map_err(|e| io::Error::other(format!("Reed-Solomon verify failed: {e:?}")))
|
||||
} else {
|
||||
Ok(true)
|
||||
}
|
||||
@@ -1200,6 +1275,105 @@ impl Erasure {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_legacy_workspaces_keep_their_memory_in_cache() {
|
||||
use crate::disk::{test_resource_units, with_optional_resource_unit};
|
||||
use std::time::Duration;
|
||||
|
||||
let codec = LegacyReedSolomonEncoder::with_workspace_cache(2, 1, true).expect("isolated cached codec");
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(512, Duration::from_secs(30));
|
||||
let mut shards = [vec![7; 8], vec![9; 8], vec![0; 8]];
|
||||
let (resources, _) = test_resource_units(1);
|
||||
with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
codec
|
||||
.encode(shards.iter_mut().map(Vec::as_mut_slice).collect())
|
||||
.expect("accounted encoder");
|
||||
})
|
||||
.await;
|
||||
assert_eq!(memory.snapshot().used, 256, "fresh SIMD work storage uses at most four 64-byte units");
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
let (resources, _) = test_resource_units(1);
|
||||
with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
codec
|
||||
.encode(shards.iter_mut().map(Vec::as_mut_slice).collect())
|
||||
.expect("cached payload survives shrink");
|
||||
})
|
||||
.await;
|
||||
assert_eq!(memory.snapshot().used, 256);
|
||||
|
||||
memory.set_sampled_limit(512, Duration::from_secs(30));
|
||||
let mut decoded = vec![None, Some(shards[1].clone()), Some(shards[2].clone())];
|
||||
let (resources, _) = test_resource_units(1);
|
||||
with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
codec.reconstruct_data(&mut decoded).expect("accounted decoder");
|
||||
})
|
||||
.await;
|
||||
assert_eq!(decoded[0].as_deref(), Some(shards[0].as_slice()));
|
||||
assert_eq!(memory.snapshot().used, 512, "both cached codec payloads remain charged");
|
||||
drop(codec);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_legacy_cache_cannot_reuse_a_different_memory_owner() {
|
||||
use crate::disk::{test_resource_units, with_optional_resource_unit};
|
||||
use std::time::Duration;
|
||||
|
||||
let codec = LegacyReedSolomonEncoder::with_workspace_cache(2, 1, true).expect("isolated cached codec");
|
||||
let first = BackgroundMemoryBudget::new();
|
||||
first.set_sampled_limit(256, Duration::from_secs(30));
|
||||
let mut shards = [vec![7; 8], vec![9; 8], vec![0; 8]];
|
||||
let (resources, _) = test_resource_units(1);
|
||||
with_optional_resource_unit(Some(resources.with_memory_budget(first.clone())), async {
|
||||
codec
|
||||
.encode(shards.iter_mut().map(Vec::as_mut_slice).collect())
|
||||
.expect("first owner");
|
||||
})
|
||||
.await;
|
||||
let second = BackgroundMemoryBudget::new();
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let error = with_optional_resource_unit(Some(resources.with_memory_budget(second.clone())), async {
|
||||
codec.encode(shards.iter_mut().map(Vec::as_mut_slice).collect())
|
||||
})
|
||||
.await
|
||||
.expect_err("a foreign cache must not bypass a closed owner");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
assert_eq!(first.snapshot().used, 0);
|
||||
assert_eq!(second.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_modern_verification_charges_scratch_and_preserves_invalid_input_errors() {
|
||||
use crate::disk::{test_resource_units, with_optional_resource_unit};
|
||||
use std::time::Duration;
|
||||
|
||||
let codec = ReedSolomonEncoder::new(2, 1).expect("modern codec");
|
||||
let mut shards = [vec![7; 8], vec![9; 8], vec![0; 8]];
|
||||
codec
|
||||
.encode(shards.iter_mut().map(Vec::as_mut_slice).collect())
|
||||
.expect("valid parity");
|
||||
let refs: Vec<_> = shards.iter().map(Vec::as_slice).collect();
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let error =
|
||||
with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async { codec.verify(&refs) })
|
||||
.await
|
||||
.expect_err("closed owner rejects verification scratch");
|
||||
assert_eq!(error.kind(), io::ErrorKind::WouldBlock);
|
||||
memory.set_sampled_limit(8, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
assert!(codec.verify(&refs).expect("exact parity scratch fits"));
|
||||
assert!(
|
||||
codec.verify(&[&[], &[9; 8], &[0; 8]]).is_err(),
|
||||
"empty first shard remains an error instead of a panic"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
use proptest::collection::{btree_set, vec};
|
||||
use proptest::prelude::*;
|
||||
use std::pin::Pin;
|
||||
|
||||
@@ -12,8 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::background_memory::{Reservation, is_memory_pressure};
|
||||
use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::{BackgroundMemoryBudget, current_resource_unit};
|
||||
use crate::erasure::coding::BitrotReader;
|
||||
use crate::erasure::coding::BitrotWriterWrapper;
|
||||
use crate::erasure::coding::encode::MultiWriter;
|
||||
@@ -22,6 +24,7 @@ use futures::StreamExt;
|
||||
use futures::stream::FuturesUnordered;
|
||||
use std::io;
|
||||
use std::io::ErrorKind;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::{trace, warn};
|
||||
@@ -30,21 +33,53 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_ERASURE: &str = "erasure";
|
||||
const EVENT_ERASURE_HEAL_STARTED: &str = "erasure_heal_started";
|
||||
|
||||
struct HealShards {
|
||||
// Drop payloads before refunding their reservations on every error path.
|
||||
shards: Vec<Option<Vec<u8>>>,
|
||||
memory: Vec<Option<Reservation>>,
|
||||
budget: Option<Arc<BackgroundMemoryBudget>>,
|
||||
}
|
||||
|
||||
impl HealShards {
|
||||
fn reserve_missing(&mut self, shard_size: usize) -> io::Result<()> {
|
||||
if let Some(budget) = &self.budget {
|
||||
for (shard, memory) in self.shards.iter().zip(&mut self.memory) {
|
||||
if shard.is_none() {
|
||||
*memory = Some(budget.reserve(shard_size)?);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn into_bytes(self) -> Vec<Bytes> {
|
||||
self.shards
|
||||
.into_iter()
|
||||
.zip(self.memory)
|
||||
.map(|(shard, memory)| Reservation::wrap(Bytes::from(shard.unwrap_or_default()), memory))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_heal_shards<R>(
|
||||
readers: &mut [Option<BitrotReader<R>>],
|
||||
shard_size: usize,
|
||||
read_timeout: Duration,
|
||||
) -> (Vec<Option<Vec<u8>>>, Vec<Option<Error>>)
|
||||
) -> Result<(HealShards, Vec<Option<Error>>)>
|
||||
where
|
||||
R: AsyncRead + Unpin + Send + Sync,
|
||||
{
|
||||
let num_readers = readers.len();
|
||||
let mut shards = vec![None; num_readers];
|
||||
let mut shards = HealShards {
|
||||
shards: vec![None; num_readers],
|
||||
memory: std::iter::repeat_with(|| None).take(num_readers).collect(),
|
||||
budget: current_resource_unit().and_then(|resources| resources.memory_budget()),
|
||||
};
|
||||
let mut errs = vec![None; num_readers];
|
||||
let mut retire_readers = Vec::new();
|
||||
|
||||
if shard_size == 0 {
|
||||
return (shards, errs);
|
||||
return Ok((shards, errs));
|
||||
}
|
||||
|
||||
{
|
||||
@@ -55,8 +90,17 @@ where
|
||||
continue;
|
||||
};
|
||||
|
||||
let budget = shards.budget.clone();
|
||||
futures.push(Box::pin(async move {
|
||||
let mut buf = vec![0; shard_size];
|
||||
let memory = match budget.as_ref().map(|budget| budget.reserve(shard_size)).transpose() {
|
||||
Ok(memory) => memory,
|
||||
Err(error) => return (index, Err(Error::from(error)), false),
|
||||
};
|
||||
let mut buf = Vec::new();
|
||||
if buf.try_reserve_exact(shard_size).is_err() {
|
||||
return (index, Err(Error::from(io::Error::from(ErrorKind::OutOfMemory))), false);
|
||||
}
|
||||
buf.resize(shard_size, 0);
|
||||
let read_result = if read_timeout.is_zero() {
|
||||
reader.read(&mut buf).await
|
||||
} else {
|
||||
@@ -75,7 +119,7 @@ where
|
||||
match read_result {
|
||||
Ok(n) => {
|
||||
buf.truncate(n);
|
||||
(index, Ok(buf), false)
|
||||
(index, Ok((buf, memory)), false)
|
||||
}
|
||||
Err(err) => {
|
||||
let should_retire = err.kind() == ErrorKind::TimedOut;
|
||||
@@ -87,10 +131,14 @@ where
|
||||
|
||||
while let Some((index, result, should_retire)) = futures.next().await {
|
||||
match result {
|
||||
Ok(shard) => {
|
||||
shards[index] = Some(shard);
|
||||
Ok((shard, memory)) => {
|
||||
shards.shards[index] = Some(shard);
|
||||
shards.memory[index] = memory;
|
||||
}
|
||||
Err(err) => {
|
||||
if matches!(&err, Error::Io(error) if is_memory_pressure(error)) {
|
||||
return Err(err);
|
||||
}
|
||||
errs[index] = Some(err);
|
||||
if should_retire {
|
||||
retire_readers.push(index);
|
||||
@@ -105,7 +153,7 @@ where
|
||||
warn!(shard_index = index, "retiring timed-out heal shard reader");
|
||||
}
|
||||
|
||||
(shards, errs)
|
||||
Ok((shards, errs))
|
||||
}
|
||||
|
||||
fn heal_writer_failure(writers: &mut MultiWriter<'_>, error: io::Error) -> Error {
|
||||
@@ -164,7 +212,7 @@ impl super::Erasure {
|
||||
for block_index in start_block..end_block {
|
||||
let shard_offset = block_index * self.shard_size();
|
||||
let shard_size = self.shard_size().min(shard_file_size.saturating_sub(shard_offset));
|
||||
let (mut shards, errs) = read_heal_shards(&mut readers, shard_size, read_timeout).await;
|
||||
let (mut owned_shards, errs) = read_heal_shards(&mut readers, shard_size, read_timeout).await?;
|
||||
|
||||
// Every source shard is already bitrot-verified by its reader, so any
|
||||
// data_shards survivors are sufficient to reconstruct — requiring more
|
||||
@@ -183,6 +231,24 @@ impl super::Erasure {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
owned_shards.reserve_missing(shard_size)?;
|
||||
let source_parity_bytes =
|
||||
owned_shards
|
||||
.shards
|
||||
.iter()
|
||||
.skip(self.data_shards)
|
||||
.flatten()
|
||||
.try_fold(0usize, |total, shard| {
|
||||
total
|
||||
.checked_add(shard.len())
|
||||
.ok_or_else(|| io::Error::from(ErrorKind::OutOfMemory))
|
||||
})?;
|
||||
let _source_memory = owned_shards
|
||||
.budget
|
||||
.as_ref()
|
||||
.map(|budget| budget.reserve(source_parity_bytes))
|
||||
.transpose()?;
|
||||
let shards = &mut owned_shards.shards;
|
||||
let source_parity = shards
|
||||
.iter()
|
||||
.enumerate()
|
||||
@@ -190,24 +256,23 @@ impl super::Erasure {
|
||||
.filter_map(|(index, shard)| shard.as_ref().map(|shard| (index, shard.clone())))
|
||||
.collect::<Vec<_>>();
|
||||
if self.parity_shards > 0 {
|
||||
self.decode_data_and_parity(&mut shards)?;
|
||||
if !source_parity.is_empty() && !self.verify_data_and_parity(&shards)? {
|
||||
self.decode_data_and_parity(shards)?;
|
||||
if !source_parity.is_empty() && !self.verify_data_and_parity(shards)? {
|
||||
return Err(Error::other("can not reconstruct data: inconsistent heal source shards"));
|
||||
}
|
||||
for (index, source) in source_parity {
|
||||
let Some(rebuilt) = shards[index].as_ref() else {
|
||||
for (index, source) in &source_parity {
|
||||
let Some(rebuilt) = shards[*index].as_ref() else {
|
||||
return Err(Error::other("can not reconstruct data: missing rebuilt parity shard"));
|
||||
};
|
||||
if rebuilt != &source {
|
||||
if rebuilt != source {
|
||||
return Err(Error::other("can not reconstruct data: inconsistent heal source shards"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let shards = shards
|
||||
.into_iter()
|
||||
.map(|s| Bytes::from(s.unwrap_or_default()))
|
||||
.collect::<Vec<_>>();
|
||||
drop(source_parity);
|
||||
drop(_source_memory);
|
||||
let shards = owned_shards.into_bytes();
|
||||
|
||||
if let Err(error) = writers.write(shards).await {
|
||||
return Err(heal_writer_failure(&mut writers, error));
|
||||
@@ -306,6 +371,90 @@ mod tests {
|
||||
BitrotWriterWrapper::new(CustomWriter::new_inline_buffer(), shard_size, HashAlgorithm::None)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_shards_retain_memory_through_missing_outputs_and_last_slice() {
|
||||
use crate::disk::{test_resource_units, with_optional_resource_unit};
|
||||
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(24, Duration::from_secs(30));
|
||||
let mut readers = vec![
|
||||
Some(BitrotReader::new(Cursor::new(vec![7; 8]), 8, HashAlgorithm::None, false)),
|
||||
None,
|
||||
Some(BitrotReader::new(Cursor::new(vec![9; 8]), 8, HashAlgorithm::None, false)),
|
||||
];
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let (mut shards, _) = with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
read_heal_shards(&mut readers, 8, Duration::ZERO),
|
||||
)
|
||||
.await
|
||||
.expect("read two accounted shards");
|
||||
assert_eq!(memory.snapshot().used, 16);
|
||||
shards
|
||||
.reserve_missing(8)
|
||||
.expect("reserve the reconstructed output before allocation");
|
||||
shards.shards[1] = Some(vec![11; 8]);
|
||||
assert_eq!(memory.snapshot().used, 24);
|
||||
let bytes = shards.into_bytes();
|
||||
let slice = bytes[1].slice(1..3);
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
drop(bytes);
|
||||
assert_eq!(slice.as_ref(), &[11; 2]);
|
||||
assert_eq!(memory.snapshot().used, 8);
|
||||
drop(slice);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_memory_denial_is_not_replaced_with_read_quorum() {
|
||||
use crate::disk::{test_resource_units, with_optional_resource_unit};
|
||||
|
||||
let erasure = Erasure::new(2, 1, 16);
|
||||
let encoded = erasure.encode_data(&[37; 16]).expect("valid source shards");
|
||||
for limit in [0, 23, 31, 39] {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(limit, Duration::from_secs(30));
|
||||
let readers = encoded
|
||||
.iter()
|
||||
.map(|shard| Some(BitrotReader::new(Cursor::new(shard.clone()), 8, HashAlgorithm::None, false)))
|
||||
.collect();
|
||||
let mut writers = vec![Some(inline_writer(8)), None, None];
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let result = with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
erasure.heal(&mut writers, readers, 16, &[]),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(result, Err(Error::Io(error)) if error.kind() == ErrorKind::WouldBlock),
|
||||
"limit={limit}"
|
||||
);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_heal_shard_reads_release_allocated_payloads() {
|
||||
use crate::disk::{test_resource_units, with_optional_resource_unit};
|
||||
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(16, Duration::from_secs(30));
|
||||
let mut readers = vec![
|
||||
Some(BitrotReader::new(PendingReader, 8, HashAlgorithm::None, false)),
|
||||
Some(BitrotReader::new(PendingReader, 8, HashAlgorithm::None, false)),
|
||||
];
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut read = Box::pin(with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
read_heal_shards(&mut readers, 8, Duration::ZERO),
|
||||
));
|
||||
assert!(futures::poll!(&mut read).is_pending());
|
||||
assert_eq!(memory.snapshot().used, 16);
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
drop(read);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_heal_shards_returns_empty_slots_for_zero_sized_shards() {
|
||||
let mut readers = vec![
|
||||
@@ -313,9 +462,11 @@ mod tests {
|
||||
None,
|
||||
];
|
||||
|
||||
let (shards, errs) = read_heal_shards(&mut readers, 0, Duration::ZERO).await;
|
||||
let (shards, errs) = read_heal_shards(&mut readers, 0, Duration::ZERO)
|
||||
.await
|
||||
.expect("empty read succeeds");
|
||||
|
||||
assert_eq!(shards, vec![None, None]);
|
||||
assert_eq!(shards.shards, vec![None, None]);
|
||||
assert_eq!(errs, vec![None, None]);
|
||||
}
|
||||
|
||||
@@ -323,9 +474,11 @@ mod tests {
|
||||
async fn read_heal_shards_records_reader_errors_and_retiring_timeouts() {
|
||||
let mut readers = vec![Some(BitrotReader::new(FailingReader, 4, HashAlgorithm::None, false))];
|
||||
|
||||
let (shards, errs) = read_heal_shards(&mut readers, 4, Duration::ZERO).await;
|
||||
let (shards, errs) = read_heal_shards(&mut readers, 4, Duration::ZERO)
|
||||
.await
|
||||
.expect("ordinary failures remain per-shard");
|
||||
|
||||
assert!(shards[0].is_none());
|
||||
assert!(shards.shards[0].is_none());
|
||||
assert!(
|
||||
errs[0]
|
||||
.as_ref()
|
||||
@@ -334,9 +487,11 @@ mod tests {
|
||||
assert!(readers[0].is_some(), "ordinary read errors must not retire readers");
|
||||
|
||||
let mut timeout_readers = vec![Some(BitrotReader::new(PendingReader, 4, HashAlgorithm::None, false))];
|
||||
let (shards, errs) = read_heal_shards(&mut timeout_readers, 4, Duration::from_millis(1)).await;
|
||||
let (shards, errs) = read_heal_shards(&mut timeout_readers, 4, Duration::from_millis(1))
|
||||
.await
|
||||
.expect("timeouts remain per-shard");
|
||||
|
||||
assert_eq!(shards, vec![None]);
|
||||
assert_eq!(shards.shards, vec![None]);
|
||||
assert!(errs[0].as_ref().is_some_and(|err| err.to_string().contains("timed out")));
|
||||
assert!(timeout_readers[0].is_none(), "timed-out readers are retired");
|
||||
}
|
||||
|
||||
@@ -1536,7 +1536,7 @@ mod tests {
|
||||
writer.write(chunk).await.expect("write chunk");
|
||||
}
|
||||
|
||||
let inline_data = Bytes::from(writer.into_inline_data().expect("inline buffer"));
|
||||
let inline_data = writer.into_inline_data().expect("inline buffer");
|
||||
let mut reader = create_bitrot_reader_from_bytes(
|
||||
Some(inline_data),
|
||||
None,
|
||||
@@ -1584,7 +1584,7 @@ mod tests {
|
||||
|
||||
let inline_data = writer.into_inline_data().expect("inline buffer");
|
||||
let mut reader = create_deferred_bitrot_reader(
|
||||
Some(inline_data.into()),
|
||||
Some(inline_data),
|
||||
None,
|
||||
"test-bucket",
|
||||
"test-path",
|
||||
@@ -1626,7 +1626,7 @@ mod tests {
|
||||
|
||||
for stripe in 0..payload.len() / shard_size {
|
||||
let (mut reader, handle) = create_deferred_bitrot_reader_with_stripe_handle(
|
||||
Some(inline_data.clone().into()),
|
||||
Some(inline_data.clone()),
|
||||
None,
|
||||
"test-bucket",
|
||||
"test-path",
|
||||
@@ -1671,7 +1671,7 @@ mod tests {
|
||||
for chunk in payload.chunks(shard_size) {
|
||||
writer.write(chunk).await.expect("write chunk");
|
||||
}
|
||||
let mut inline_data = writer.into_inline_data().expect("inline buffer");
|
||||
let mut inline_data = writer.into_inline_data().expect("inline buffer").to_vec();
|
||||
|
||||
// Corrupt one payload byte of block 2 (blocks are hash + data).
|
||||
let block_len = algo.size() + shard_size;
|
||||
@@ -1718,7 +1718,7 @@ mod tests {
|
||||
let inline_data = writer.into_inline_data().expect("inline buffer");
|
||||
|
||||
let (mut reader, handle) = create_deferred_bitrot_reader_with_stripe_handle(
|
||||
Some(inline_data.into()),
|
||||
Some(inline_data),
|
||||
None,
|
||||
"test-bucket",
|
||||
"test-path",
|
||||
@@ -1845,4 +1845,122 @@ mod tests {
|
||||
|
||||
assert_eq!(output, data);
|
||||
}
|
||||
|
||||
mod inline_memory_regressions {
|
||||
use super::*;
|
||||
use crate::disk::background_memory::Reservation;
|
||||
use crate::disk::{BackgroundMemoryBudget, test_resource_units, with_optional_resource_unit};
|
||||
use std::time::Duration;
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_inline_reader_retains_existing_receipt_without_a_new_grant() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(64, Duration::from_secs(30));
|
||||
let receipt = memory.reserve(64).expect("existing metadata payload grant");
|
||||
let bytes = Reservation::wrap(Bytes::from(vec![7; 64]), Some(receipt));
|
||||
memory.set_sampled_limit(0, Duration::ZERO);
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut reader = with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
create_bitrot_reader_from_bytes(
|
||||
Some(bytes.clone()),
|
||||
None,
|
||||
"bucket",
|
||||
"inline",
|
||||
0,
|
||||
64,
|
||||
64,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
false,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("owned data requires no allocation")
|
||||
.expect("inline reader");
|
||||
drop(bytes);
|
||||
assert_eq!(memory.snapshot().used, 64, "the source owner must remain attached to the reader");
|
||||
let mut output = [0; 64];
|
||||
assert_eq!(reader.read(&mut output).await.expect("read existing payload"), 64);
|
||||
assert_eq!(output, [7; 64]);
|
||||
assert_eq!(memory.snapshot().used, 64);
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accounted_inline_writer_round_trips_bitrot_and_preserves_export_lifetime() {
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
memory.set_sampled_limit(192, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let bytes = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
let mut writer = create_bitrot_writer(true, None, "bucket", "inline", 64, 64, HashAlgorithm::HighwayHash256S)
|
||||
.await
|
||||
.expect("inline writer");
|
||||
assert_eq!(writer.write(&[19; 64]).await.expect("complete shard"), 64);
|
||||
writer.shutdown().await.expect("shutdown");
|
||||
writer.into_inline_data().expect("bitrot bytes")
|
||||
})
|
||||
.await;
|
||||
assert_eq!(bytes.len(), 96);
|
||||
assert_eq!(memory.snapshot().used, 96, "exported [hash][shard] remains charged");
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let mut reader = with_optional_resource_unit(
|
||||
Some(resources.with_memory_budget(memory.clone())),
|
||||
create_bitrot_reader_from_bytes(
|
||||
Some(bytes.clone()),
|
||||
None,
|
||||
"bucket",
|
||||
"inline",
|
||||
0,
|
||||
64,
|
||||
64,
|
||||
HashAlgorithm::HighwayHash256S,
|
||||
false,
|
||||
false,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("reader")
|
||||
.expect("inline reader");
|
||||
let mut output = [0; 64];
|
||||
assert_eq!(reader.read(&mut output).await.expect("bitrot remains verified"), 64);
|
||||
assert_eq!(output, [19; 64]);
|
||||
assert_eq!(memory.snapshot().used, 192, "input owner and verification scratch are both live");
|
||||
drop(reader);
|
||||
assert_eq!(memory.snapshot().used, 96);
|
||||
let slice = bytes.slice(32..36);
|
||||
drop(bytes);
|
||||
assert_eq!(memory.snapshot().used, 96);
|
||||
drop(slice);
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn inline_heal_writer_memory_denial_is_retryable_after_read_and_verify_fit() {
|
||||
use crate::erasure::coding::Erasure;
|
||||
let erasure = Erasure::new(2, 1, 16);
|
||||
let encoded = erasure.encode_data(&[37; 16]).expect("foreground test fixture");
|
||||
let readers = encoded
|
||||
.into_iter()
|
||||
.map(|shard| Some(BitrotReader::new(Cursor::new(shard), 8, HashAlgorithm::None, false)))
|
||||
.collect();
|
||||
let memory = BackgroundMemoryBudget::new();
|
||||
// Read shards: 24 bytes; source parity: 8; verify scratch: 8.
|
||||
// After verification, input shards still use 24. The target needs a
|
||||
// new 40-byte [hash][shard] allocation and must fail before copying.
|
||||
memory.set_sampled_limit(40, Duration::from_secs(30));
|
||||
let (resources, _) = test_resource_units(1);
|
||||
let result = with_optional_resource_unit(Some(resources.with_memory_budget(memory.clone())), async {
|
||||
let writer = create_bitrot_writer(true, None, "bucket", "inline", 8, 8, HashAlgorithm::HighwayHash256S)
|
||||
.await
|
||||
.expect("writer setup does not allocate payload");
|
||||
let mut writers = vec![Some(writer), None, None];
|
||||
erasure.heal(&mut writers, readers, 16, &[]).await
|
||||
})
|
||||
.await;
|
||||
assert!(matches!(result, Err(DiskError::Io(error)) if error.kind() == io::ErrorKind::WouldBlock));
|
||||
assert_eq!(memory.snapshot().used, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,7 +55,9 @@ mod set_disk;
|
||||
mod storage_api_contracts;
|
||||
mod store;
|
||||
|
||||
pub use disk::{with_background_disk_io, with_background_resource_unit};
|
||||
pub use disk::{
|
||||
spawn_blocking_with_resource_ownership, with_background_disk_io, with_background_resource_set, with_background_resource_unit,
|
||||
};
|
||||
pub use store::PoolMetaWriteGateStatus;
|
||||
|
||||
// pub mod checksum;
|
||||
|
||||
@@ -144,6 +144,8 @@ pub(crate) fn namespace_lock_signal_test_fence_is_lost(signal: &Arc<rustfs_lock:
|
||||
.is_some_and(|(_, fence)| fence.is_lock_lost())
|
||||
}
|
||||
|
||||
type FreshPutQuota = std::result::Result<(Option<crate::bucket::quota::BucketQuota>, Uuid, OffsetDateTime), String>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ObjectLockConfigSnapshot {
|
||||
store_id: Option<Uuid>,
|
||||
@@ -154,6 +156,7 @@ pub struct ObjectLockConfigSnapshot {
|
||||
lifecycle_fence: NamespaceLockFence,
|
||||
_lifecycle_guard: Option<rustfs_lock::NamespaceLockGuard>,
|
||||
metadata_transaction_guard: Option<Arc<rustfs_lock::NamespaceLockGuard>>,
|
||||
fresh_put_quota: std::sync::Mutex<Option<FreshPutQuota>>,
|
||||
}
|
||||
|
||||
impl ObjectLockConfigSnapshot {
|
||||
@@ -167,6 +170,7 @@ impl ObjectLockConfigSnapshot {
|
||||
lifecycle_fence: NamespaceLockFence::new(),
|
||||
_lifecycle_guard: None,
|
||||
metadata_transaction_guard: None,
|
||||
fresh_put_quota: std::sync::Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,6 +191,7 @@ impl ObjectLockConfigSnapshot {
|
||||
lifecycle_fence: NamespaceLockFence::new(),
|
||||
_lifecycle_guard: None,
|
||||
metadata_transaction_guard: None,
|
||||
fresh_put_quota: std::sync::Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -212,6 +217,7 @@ impl ObjectLockConfigSnapshot {
|
||||
lifecycle_fence,
|
||||
_lifecycle_guard: Some(lifecycle_guard),
|
||||
metadata_transaction_guard: Some(Arc::new(metadata_transaction_guard)),
|
||||
fresh_put_quota: std::sync::Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -233,9 +239,23 @@ impl ObjectLockConfigSnapshot {
|
||||
lifecycle_fence,
|
||||
_lifecycle_guard: None,
|
||||
metadata_transaction_guard: Some(metadata_transaction_guard),
|
||||
fresh_put_quota: std::sync::Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn set_fresh_put_quota(&self, quota: FreshPutQuota) {
|
||||
if self._lifecycle_guard.is_some() && self.metadata_transaction_guard.is_some() {
|
||||
*self.fresh_put_quota.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = Some(quota);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn take_fresh_put_quota(&self) -> Option<FreshPutQuota> {
|
||||
self.fresh_put_quota
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.take()
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "snapshot-scope predicate asserted by this file's tests (backlog#1823)")]
|
||||
pub(crate) fn is_for_store_bucket(
|
||||
&self,
|
||||
|
||||
@@ -941,6 +941,32 @@ impl InstanceContext {
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Remove the `BucketMetadataSys` when its backing `ECStore` is gone.
|
||||
///
|
||||
/// Called from `ECStore::drop`. When test fixtures share the bootstrap
|
||||
/// context, the `BucketMetadataSys` stores a `Weak<ECStore>` to prevent a
|
||||
/// reference cycle; once the store drops the weak is dangling and any
|
||||
/// `object_store()` call panics. Clearing the slot here lets the next
|
||||
/// fixture install a fresh `BucketMetadataSys` without hitting the stale one.
|
||||
pub fn clear_bucket_metadata_sys_if_stale(&self) {
|
||||
let mut guard = self.bucket_metadata_sys.lock().expect("bucket_metadata_sys lock poisoned");
|
||||
if let Some(sys) = guard.as_ref() {
|
||||
// `read()` may block briefly; the async RwLock is acceptable here
|
||||
// because Drop is sync and the lock is held only long enough to
|
||||
// check the weak reference.
|
||||
let is_stale = {
|
||||
let s = sys.try_read();
|
||||
match s {
|
||||
Ok(s) => s.is_store_stale(),
|
||||
Err(_) => true, // contended: treat as stale-safe to clear
|
||||
}
|
||||
};
|
||||
if is_stale {
|
||||
*guard = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Set this instance's background-services cancellation token (once).
|
||||
pub fn init_background_cancel_token(&self, token: CancellationToken) -> Result<(), CancellationToken> {
|
||||
self.background_cancel_token.set(token)
|
||||
|
||||
@@ -1288,7 +1288,7 @@ async fn try_create_bitrot_readers_via_batch_pread(
|
||||
// parity readers unopened; preading them here would discard their bytes
|
||||
// when fill_deferred_bitrot_readers installs stripe-aligned reserves.
|
||||
let data_first = strategy != BitrotReaderSetupStrategy::AllShards
|
||||
&& crate::erasure::coding::decode::get_lockstep_data_shards_only_enabled();
|
||||
&& crate::erasure::coding::decode::get_lockstep_data_shards_only_with_deferred_parity_enabled();
|
||||
let mut setup = BitrotReaderSetup::new(disks.len());
|
||||
for (idx, disk) in disks.iter().enumerate() {
|
||||
if disk.is_none() {
|
||||
@@ -2549,6 +2549,8 @@ pub(in crate::set_disk) struct RenameTailCleanup {
|
||||
pub(in crate::set_disk) struct RenameTailOutcome {
|
||||
pub(in crate::set_disk) convergence: RenameConvergence,
|
||||
pub(in crate::set_disk) cleanup: Vec<RenameTailCleanup>,
|
||||
pub(in crate::set_disk) old_current_source_all_successful_checked: bool,
|
||||
pub(in crate::set_disk) all_online_local_succeeded: bool,
|
||||
}
|
||||
|
||||
const RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP_ENV: &str = "RUSTFS_PUT_RENAME_TAIL_CLEANUP_COUNTERFACTUAL_SKIP";
|
||||
@@ -3572,7 +3574,7 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
|
||||
Guards: Send + 'static,
|
||||
Finalize: FnOnce() -> FinalizeFuture + Send,
|
||||
FinalizeFuture: Future<Output = ()> + Send,
|
||||
Cleanup: FnOnce(Guards, Vec<RenameTailCleanup>) -> CleanupFuture + Send,
|
||||
Cleanup: FnOnce(Guards, Vec<RenameTailCleanup>, bool, bool) -> CleanupFuture + Send,
|
||||
CleanupFuture: Future<Output = ()> + Send,
|
||||
Cleanup: 'static,
|
||||
CleanupFuture: 'static,
|
||||
@@ -3580,34 +3582,41 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
|
||||
SubmitFuture: Future<Output = ()> + Send,
|
||||
{
|
||||
let tail_async_started = rustfs_io_metrics::put_stage_timer();
|
||||
let (needs_heal, tail_cleanup, tail_complete) = match tail_drain.await {
|
||||
Ok(Some(outcome)) => (outcome.convergence.needs_heal(), outcome.cleanup, true),
|
||||
Ok(None) => {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
state = "missing_convergence",
|
||||
bucket = %request.bucket,
|
||||
object = request.object_prefix.as_deref().unwrap_or_default(),
|
||||
"committed rename tail drain completed without convergence"
|
||||
);
|
||||
(true, Vec::new(), false)
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
state = "failed",
|
||||
bucket = %request.bucket,
|
||||
object = request.object_prefix.as_deref().unwrap_or_default(),
|
||||
error = %err,
|
||||
"rename tail drain failed"
|
||||
);
|
||||
(true, Vec::new(), false)
|
||||
}
|
||||
};
|
||||
let (needs_heal, tail_cleanup, tail_complete, old_current_source_all_successful_checked, all_online_local_succeeded) =
|
||||
match tail_drain.await {
|
||||
Ok(Some(outcome)) => (
|
||||
outcome.convergence.needs_heal(),
|
||||
outcome.cleanup,
|
||||
true,
|
||||
outcome.old_current_source_all_successful_checked,
|
||||
outcome.all_online_local_succeeded,
|
||||
),
|
||||
Ok(None) => {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
state = "missing_convergence",
|
||||
bucket = %request.bucket,
|
||||
object = request.object_prefix.as_deref().unwrap_or_default(),
|
||||
"committed rename tail drain completed without convergence"
|
||||
);
|
||||
(true, Vec::new(), false, false, false)
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
state = "failed",
|
||||
bucket = %request.bucket,
|
||||
object = request.object_prefix.as_deref().unwrap_or_default(),
|
||||
error = %err,
|
||||
"rename tail drain failed"
|
||||
);
|
||||
(true, Vec::new(), false, false, false)
|
||||
}
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_ASYNC_WAIT,
|
||||
tail_async_started,
|
||||
@@ -3651,7 +3660,12 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
|
||||
let job = RenameTailCleanupJob {
|
||||
enqueued_at: Instant::now(),
|
||||
targets,
|
||||
future: Box::pin(cleanup(guards, tail_cleanup)),
|
||||
future: Box::pin(cleanup(
|
||||
guards,
|
||||
tail_cleanup,
|
||||
old_current_source_all_successful_checked,
|
||||
all_online_local_succeeded,
|
||||
)),
|
||||
};
|
||||
if let Err(job) = rename_tail_cleanup_queue().try_enqueue(job).await {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -3676,7 +3690,13 @@ pub(in crate::set_disk) async fn finish_rename_tail_heal<
|
||||
cleanup_submit_started,
|
||||
);
|
||||
let cleanup_started = rustfs_io_metrics::put_stage_timer();
|
||||
cleanup(guards, tail_cleanup).await;
|
||||
cleanup(
|
||||
guards,
|
||||
tail_cleanup,
|
||||
old_current_source_all_successful_checked,
|
||||
all_online_local_succeeded,
|
||||
)
|
||||
.await;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP,
|
||||
cleanup_started,
|
||||
@@ -4155,7 +4175,14 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if !sent_commit && success_count >= write_quorum {
|
||||
let pending_tasks = tasks.len();
|
||||
let checked_count = old_current_source_checked.iter().filter(|checked| **checked).count();
|
||||
let max_source_votes = Self::max_common_old_current_source_votes(&old_current_sources);
|
||||
let can_change_old_current_source = (checked_count < write_quorum
|
||||
&& checked_count + pending_tasks >= write_quorum)
|
||||
|| (max_source_votes < write_quorum && max_source_votes + pending_tasks >= write_quorum);
|
||||
|
||||
if !sent_commit && success_count >= write_quorum && !can_change_old_current_source {
|
||||
let snapshot_commit = Self::rename_data_commit_from_observations(
|
||||
RenameDataObservations {
|
||||
disks: &coordinator_disks,
|
||||
@@ -4174,6 +4201,8 @@ impl SetDisks {
|
||||
cleanup_at_snapshot = commit.cleanup_disks.iter().map(Option::is_some).collect();
|
||||
}
|
||||
if let Some(commit_tx) = commit_tx.take() {
|
||||
#[cfg(test)]
|
||||
rename_fanout_barrier::wait_for_armed_rename_before_early_ack(&fanout_dst_object).await;
|
||||
let _ = commit_tx.send(snapshot_commit);
|
||||
}
|
||||
sent_commit = true;
|
||||
@@ -4296,9 +4325,21 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
let old_current_source_all_successful_checked = errs
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, err)| err.is_none())
|
||||
.all(|(idx, _)| old_current_source_checked.get(idx).copied().unwrap_or(false));
|
||||
let all_online_local_succeeded = coordinator_disks.iter().zip(errs.iter()).all(|(disk, err)| match disk {
|
||||
Some(d) => d.is_local() && err.is_none(),
|
||||
None => true,
|
||||
});
|
||||
|
||||
Some(RenameTailOutcome {
|
||||
convergence: final_convergence,
|
||||
cleanup: tail_cleanup,
|
||||
old_current_source_all_successful_checked,
|
||||
all_online_local_succeeded,
|
||||
})
|
||||
}
|
||||
});
|
||||
@@ -4863,6 +4904,14 @@ impl SetDisks {
|
||||
})
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn max_common_old_current_source_votes(old_current_sources: &[Option<FileInfo>]) -> usize {
|
||||
let mut counts: HashMap<[u8; 32], usize> = HashMap::new();
|
||||
for source in old_current_sources.iter().flatten() {
|
||||
*counts.entry(Self::file_info_quorum_hash(source)).or_insert(0) += 1;
|
||||
}
|
||||
counts.into_values().max().unwrap_or(0)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn reduce_common_old_current_source(
|
||||
old_current_sources: &[Option<FileInfo>],
|
||||
write_quorum: usize,
|
||||
@@ -6688,9 +6737,7 @@ impl SetDisks {
|
||||
object,
|
||||
);
|
||||
}
|
||||
let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
|
||||
let if_match = http_preconditions.if_match_value().map(str::to_owned);
|
||||
if should_prevent_write(&oi, if_none_match, if_match)
|
||||
if should_prevent_write(&oi, http_preconditions.if_none_match_value(), http_preconditions.if_match_value())
|
||||
&& !crate::data_movement::can_replace_stale_data_movement_target(&oi, opts)
|
||||
{
|
||||
return Some(StorageError::PreconditionFailed);
|
||||
@@ -7422,6 +7469,34 @@ pub(crate) mod rename_fanout_barrier {
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) async fn wait_for_armed_rename_before_early_ack(object: &str) {
|
||||
loop {
|
||||
let (any_armed, any_unpaused) = {
|
||||
let reg = lock();
|
||||
let mut any_armed = false;
|
||||
let mut any_unpaused = false;
|
||||
for ((obj, _, phase), armed) in ®.armed {
|
||||
if obj == object && *phase == Phase::Rename {
|
||||
any_armed = true;
|
||||
if !armed.paused.load(Ordering::SeqCst) {
|
||||
any_unpaused = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
(any_armed, any_unpaused)
|
||||
};
|
||||
if !any_armed {
|
||||
return;
|
||||
}
|
||||
if !any_unpaused {
|
||||
tokio::task::yield_now().await;
|
||||
return;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
/// RAII scope that observes in-flight fan-out task counts for a single
|
||||
/// `object`. Counts accrue only while the scope is alive; it clears its own
|
||||
/// entry on drop.
|
||||
@@ -8488,9 +8563,9 @@ mod tests {
|
||||
}
|
||||
assert_eq!(
|
||||
recorder.counter_value("rustfs_io_get_object_local_batch_shards_total", &[("outcome", "success")]),
|
||||
4
|
||||
2
|
||||
);
|
||||
assert_eq!(recorder.counter_value("rustfs_io_get_object_local_batch_read_bytes_total", &[]), 14);
|
||||
assert_eq!(recorder.counter_value("rustfs_io_get_object_local_batch_read_bytes_total", &[]), 7);
|
||||
assert_eq!(recorder.histogram_sample_count("rustfs_io_get_object_local_batch_prepare_seconds"), 8);
|
||||
}
|
||||
|
||||
@@ -9546,11 +9621,11 @@ mod tests {
|
||||
writer.shutdown().await.expect("inline writer should shutdown");
|
||||
let mut fi = base.clone();
|
||||
fi.erasure.index = index + 1;
|
||||
fi.data = Some(Bytes::from(
|
||||
fi.data = Some(
|
||||
writer
|
||||
.into_inline_data()
|
||||
.expect("inline bitrot writer should retain encoded data"),
|
||||
));
|
||||
);
|
||||
files.push(fi);
|
||||
}
|
||||
files
|
||||
@@ -10915,7 +10990,7 @@ mod tests {
|
||||
move || async move {
|
||||
*finalize_captured.lock().expect("finalize recorder should not poison") = true;
|
||||
},
|
||||
move |(), _| async move {
|
||||
move |(), _, _, _| async move {
|
||||
*cleanup_captured.lock().expect("cleanup recorder should not poison") = true;
|
||||
},
|
||||
|request| async move {
|
||||
|
||||
@@ -80,7 +80,7 @@ use crate::erasure::codec::bridge::{
|
||||
use crate::erasure::coding;
|
||||
use crate::error::{Error, Result, is_err_version_not_found};
|
||||
use crate::error::{GenericError, ObjectApiError, is_err_object_not_found};
|
||||
use crate::io_support::bitrot::{create_bitrot_reader, create_bitrot_reader_from_bytes, create_bitrot_writer};
|
||||
use crate::io_support::bitrot::{create_bitrot_reader_from_bytes, create_bitrot_writer};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::object_api::get_object_body_cache_hook;
|
||||
use crate::object_api::object_api_utils::get_raw_etag;
|
||||
@@ -120,7 +120,6 @@ use bytesize::ByteSize;
|
||||
use futures::future::join_all;
|
||||
use http::HeaderMap;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use regex::Regex;
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_filemeta::metadata_keys;
|
||||
use rustfs_filemeta::{
|
||||
@@ -4537,6 +4536,7 @@ fn resolve_tiered_decommission_write_quorum_result(
|
||||
pub struct SetDisks {
|
||||
pub locker_owner: String,
|
||||
pub disks: Arc<RwLock<Vec<Option<DiskStore>>>>,
|
||||
pub(crate) physical_heal_admission: Arc<crate::disk::PhysicalHealBinding>,
|
||||
pub set_endpoints: Vec<Endpoint>,
|
||||
pub set_drive_count: usize,
|
||||
pub default_parity_count: usize,
|
||||
@@ -5154,6 +5154,12 @@ impl SetDisks {
|
||||
if request.expected_bucket_incarnation_id.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if let Some(object) = request.object_prefix.as_deref()
|
||||
&& skip_e2e_mrf_fallback(&request.bucket, object).await
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
request.source = rustfs_heal_contracts::heal_channel::HealRequestSource::Mrf;
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -5473,6 +5479,7 @@ impl SetDisks {
|
||||
Arc::new(SetDisks {
|
||||
locker_owner,
|
||||
disks,
|
||||
physical_heal_admission: Arc::default(),
|
||||
set_drive_count,
|
||||
default_parity_count,
|
||||
set_index,
|
||||
@@ -5861,6 +5868,25 @@ impl SetDisks {
|
||||
// }
|
||||
}
|
||||
|
||||
// The isolated root-heal test suppresses only the direct best-effort fallback.
|
||||
// Persisted MRF responsibilities and their replay keep their normal behavior.
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
pub(in crate::set_disk) async fn skip_e2e_mrf_fallback(bucket: &str, object: &str) -> bool {
|
||||
let Some(path) = std::env::var_os("RUSTFS_E2E_SKIP_MRF_FALLBACK_PATH") else {
|
||||
return false;
|
||||
};
|
||||
let path = std::path::PathBuf::from(path);
|
||||
let Ok(prefix) = tokio::fs::read_to_string(&path).await else {
|
||||
return false;
|
||||
};
|
||||
let key = format!("{bucket}/{object}");
|
||||
if prefix.is_empty() || !key.starts_with(&prefix) {
|
||||
return false;
|
||||
}
|
||||
let _ = tokio::fs::write(path.with_extension("observed"), key).await;
|
||||
true
|
||||
}
|
||||
|
||||
fn is_explicit_null_version(version_id: Option<Uuid>) -> bool {
|
||||
version_id == Some(Uuid::nil())
|
||||
}
|
||||
@@ -8009,16 +8035,29 @@ fn parts_after_marker(part_numbers: &[usize], part_number_marker: usize) -> Opti
|
||||
Some(&part_numbers[index..])
|
||||
}
|
||||
|
||||
pub fn canonicalize_etag(etag: &str) -> String {
|
||||
let re = Regex::new("\"*?([^\"]*?)\"*?$").unwrap();
|
||||
re.replace_all(etag, "$1").to_string()
|
||||
fn canonicalized_etag(etag: &str) -> Cow<'_, str> {
|
||||
// Preserve the legacy suffix-matching behavior for embedded quotes.
|
||||
let trimmed = etag.trim_end_matches('"');
|
||||
let Some(last_quote) = trimmed.rfind('"') else {
|
||||
return Cow::Borrowed(trimmed);
|
||||
};
|
||||
let prefix = trimmed[..last_quote].trim_end_matches('"');
|
||||
let suffix = &trimmed[last_quote + 1..];
|
||||
if prefix.is_empty() {
|
||||
Cow::Borrowed(suffix)
|
||||
} else {
|
||||
let mut result = String::with_capacity(prefix.len() + suffix.len());
|
||||
result.push_str(prefix);
|
||||
result.push_str(suffix);
|
||||
Cow::Owned(result)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn e_tag_matches(etag: &str, condition: &str) -> bool {
|
||||
if condition.trim() == "*" {
|
||||
return true;
|
||||
}
|
||||
canonicalize_etag(etag) == canonicalize_etag(condition)
|
||||
canonicalized_etag(etag) == canonicalized_etag(condition)
|
||||
}
|
||||
|
||||
fn weak_etag_matches(etag: &str, condition: &str) -> bool {
|
||||
@@ -8033,15 +8072,12 @@ fn weak_etag_matches(etag: &str, condition: &str) -> bool {
|
||||
.or_else(|| trimmed.strip_prefix("w/"))
|
||||
.unwrap_or(trimmed)
|
||||
}
|
||||
canonicalize_etag(strip_weak(etag)) == canonicalize_etag(strip_weak(condition))
|
||||
canonicalized_etag(strip_weak(etag)) == canonicalized_etag(strip_weak(condition))
|
||||
}
|
||||
|
||||
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_match: Option<String>) -> bool {
|
||||
let if_none_match = if_none_match
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|condition| !condition.is_empty());
|
||||
let if_match = if_match.as_deref().map(str::trim).filter(|condition| !condition.is_empty());
|
||||
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<&str>, if_match: Option<&str>) -> bool {
|
||||
let if_none_match = if_none_match.map(str::trim).filter(|condition| !condition.is_empty());
|
||||
let if_match = if_match.map(str::trim).filter(|condition| !condition.is_empty());
|
||||
|
||||
match &oi.etag {
|
||||
Some(etag) => {
|
||||
@@ -13745,6 +13781,22 @@ mod tests {
|
||||
assert!(result2.iter().all(|d| d.is_none()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalized_etag_preserves_legacy_regex_semantics() {
|
||||
let regex = regex::Regex::new("\"*?([^\"]*?)\"*?$").expect("valid legacy ETag pattern");
|
||||
let alphabet = ['a', '"', '\n', 'é', '\0'];
|
||||
for length in 0..=6_u32 {
|
||||
for mut number in 0..alphabet.len().pow(length) {
|
||||
let mut input = String::new();
|
||||
for _ in 0..length {
|
||||
input.push(alphabet[number % alphabet.len()]);
|
||||
number /= alphabet.len();
|
||||
}
|
||||
assert_eq!(canonicalized_etag(&input), regex.replace_all(&input, "$1"), "input: {input:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_etag_matches() {
|
||||
assert!(e_tag_matches("abc", "abc"));
|
||||
@@ -14320,61 +14372,61 @@ mod tests {
|
||||
etag: Some("abc".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let if_none_match = Some("abc".to_string());
|
||||
let if_none_match = Some("abc");
|
||||
let if_match = None;
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_none_match = Some("*");
|
||||
let if_match = None;
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = Some("def".to_string());
|
||||
let if_match = Some("def");
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = Some("*".to_string());
|
||||
let if_match = Some("*");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_none_match = Some("def");
|
||||
let if_match = None;
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_match = Some("*".to_string());
|
||||
let if_none_match = Some("def");
|
||||
let if_match = Some("*");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
let if_none_match = Some("def");
|
||||
let if_match = Some("\"abc\"");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
let if_none_match = Some("*");
|
||||
let if_match = Some("\"abc\"");
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let oi = ObjectInfo {
|
||||
etag: None,
|
||||
..Default::default()
|
||||
};
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
let if_none_match = Some("*");
|
||||
let if_match = Some("\"abc\"");
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = None;
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some(String::new());
|
||||
let if_match = Some(" ".to_string());
|
||||
let if_none_match = Some("");
|
||||
let if_match = Some(" ");
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let oi = ObjectInfo {
|
||||
etag: Some("abc".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(should_prevent_write(&oi, Some("W/\"abc\"".to_string()), None));
|
||||
assert!(!should_prevent_write(&oi, Some("W/\"def\"".to_string()), None));
|
||||
assert!(should_prevent_write(&oi, None, Some("W/\"abc\"".to_string())));
|
||||
assert!(should_prevent_write(&oi, Some("W/\"abc\""), None));
|
||||
assert!(!should_prevent_write(&oi, Some("W/\"def\""), None));
|
||||
assert!(should_prevent_write(&oi, None, Some("W/\"abc\"")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -14782,7 +14834,7 @@ mod tests {
|
||||
file.erasure.index = files.len() + 1;
|
||||
file.erasure.block_size = erasure.block_size;
|
||||
file.uses_legacy_checksum = uses_legacy;
|
||||
file.data = Some(Bytes::from(data));
|
||||
file.data = Some(data);
|
||||
files.push(file);
|
||||
}
|
||||
|
||||
@@ -15243,7 +15295,7 @@ mod tests {
|
||||
}
|
||||
|
||||
let encoded = bitrot_writer.into_inline_data().expect("bitrot encoded data should exist");
|
||||
disk.write_all(bucket, &shard_path, Bytes::from(encoded))
|
||||
disk.write_all(bucket, &shard_path, encoded)
|
||||
.await
|
||||
.expect("encoded shard should be stored");
|
||||
|
||||
@@ -15327,7 +15379,7 @@ mod tests {
|
||||
}
|
||||
|
||||
let encoded = bitrot_writer.into_inline_data().expect("bitrot encoded data should exist");
|
||||
disk.write_all(bucket, &shard_path, Bytes::from(encoded))
|
||||
disk.write_all(bucket, &shard_path, encoded)
|
||||
.await
|
||||
.expect("encoded shard should be stored");
|
||||
|
||||
@@ -15440,7 +15492,7 @@ mod tests {
|
||||
}
|
||||
|
||||
let encoded = bitrot_writer.into_inline_data().expect("bitrot encoded data should exist");
|
||||
disk.write_all(bucket, &shard_path, Bytes::from(encoded))
|
||||
disk.write_all(bucket, &shard_path, encoded)
|
||||
.await
|
||||
.expect("encoded shard should be stored");
|
||||
}
|
||||
|
||||
@@ -97,6 +97,14 @@ pub(crate) mod test_hooks {
|
||||
.push((bucket.to_owned(), object.to_owned(), hook));
|
||||
}
|
||||
|
||||
pub(super) fn has_before_gc_finalize(bucket: &str, object: &str) -> bool {
|
||||
BEFORE_GC_FINALIZE
|
||||
.lock()
|
||||
.expect("GC finalize hooks")
|
||||
.iter()
|
||||
.any(|(hook_bucket, hook_object, _)| hook_bucket == bucket && hook_object == object)
|
||||
}
|
||||
|
||||
pub(super) async fn run_before_gc_finalize(bucket: &str, object: &str) {
|
||||
let hook = {
|
||||
let mut hooks = BEFORE_GC_FINALIZE.lock().expect("GC finalize hooks");
|
||||
@@ -418,6 +426,59 @@ impl SetDisks {
|
||||
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
|
||||
context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
|
||||
let can_commit = context.can_commit;
|
||||
let can_try_local_fast_path = disks.len() == self.set_drive_count
|
||||
&& disks.iter().all(|d| d.as_ref().is_some_and(|d| d.is_local()))
|
||||
&& !crate::disk::local::has_delete_version_test_failpoints()
|
||||
&& {
|
||||
#[cfg(test)]
|
||||
{
|
||||
!test_hooks::has_before_gc_finalize(bucket, object)
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
true
|
||||
}
|
||||
};
|
||||
if can_try_local_fast_path && can_commit() {
|
||||
let fast_published = join_all(disks.iter().map(|disk| async {
|
||||
if !can_commit() {
|
||||
return Err(DiskError::other("delete recovery namespace fence was lost"));
|
||||
}
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
disk.try_publish_prepared_delete_local(bucket, object, transaction, context.namespace_owner.clone())
|
||||
.await
|
||||
}))
|
||||
.await;
|
||||
let all_fast_published = fast_published.len() == self.set_drive_count
|
||||
&& fast_published.iter().all(|res| matches!(res, Ok(Some((_, true, true)))));
|
||||
if all_fast_published && can_commit() {
|
||||
#[cfg(test)]
|
||||
test_hooks::record_cleanup_attempt(bucket, object);
|
||||
let finalized = join_all(disks.iter().zip(fast_published.iter()).map(|(disk, pub_res)| {
|
||||
let ns_owner = context.namespace_owner.clone();
|
||||
async move {
|
||||
if !can_commit() {
|
||||
return Ok(false);
|
||||
}
|
||||
let Some(disk) = disk.as_ref() else {
|
||||
return Ok(false);
|
||||
};
|
||||
let Ok(Some((journal, _, _))) = pub_res else {
|
||||
return Ok(false);
|
||||
};
|
||||
disk.try_finish_and_finalize_prepared_delete_local(bucket, object, transaction, journal, ns_owner)
|
||||
.await
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
if finalized.iter().all(|res| matches!(res, Ok(true))) {
|
||||
if let Some(ticket) = context.recovery_ticket.as_ref() {
|
||||
self.ctx.delete_recovery_hints().ack(ticket);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
let states = inspect(disks, bucket, object, transaction).await;
|
||||
let untouched_absence = |state: &Result<DeleteRecoveryInspection>| matches!(state, Ok(state) if state.state == DeleteRecoveryState::Absent && state.current_digest.is_none());
|
||||
if states.iter().filter(|state| untouched_absence(state)).count() >= write_quorum
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -158,13 +158,15 @@ fn rename_tail_cleanup_zero_target_tmp_delete_skip_enabled() -> bool {
|
||||
}
|
||||
|
||||
fn tier_free_version_source_reuse_rename_data_enabled() -> bool {
|
||||
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
|
||||
*ENABLED.get_or_init(|| {
|
||||
matches!(
|
||||
std::env::var(TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ENV).ok().as_deref(),
|
||||
Some("1" | "true" | "TRUE" | "yes" | "YES" | "on" | "ON")
|
||||
)
|
||||
})
|
||||
#[cfg(test)]
|
||||
{
|
||||
rustfs_utils::get_env_bool(TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ENV, true)
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
|
||||
*ENABLED.get_or_init(|| rustfs_utils::get_env_bool(TIER_FREE_VERSION_SOURCE_REUSE_RENAME_DATA_ENV, true))
|
||||
}
|
||||
}
|
||||
|
||||
fn tier_free_version_source_reuse_rename_data_assume_homogeneous_enabled() -> bool {
|
||||
@@ -4741,7 +4743,7 @@ impl SetDisks {
|
||||
.ok_or_else(|| Error::other(format!("inline encoder omitted disk shard {i}")))?,
|
||||
);
|
||||
} else if let Some(writer) = writers[i].take() {
|
||||
pfi.data = Some(writer.into_inline_data().map(Bytes::from).unwrap_or_default());
|
||||
pfi.data = Some(writer.into_inline_data().unwrap_or_default());
|
||||
}
|
||||
|
||||
pfi.set_inline_data();
|
||||
@@ -5523,8 +5525,10 @@ impl SetDisks {
|
||||
);
|
||||
|
||||
let tier_free_version_source_lookup_started = rustfs_io_metrics::put_stage_timer();
|
||||
let all_commit_disks_local = commit_disks.iter().flatten().all(|disk| disk.is_local());
|
||||
let reuse_rename_data_source_for_tier_free_version = tier_free_version_source_reuse_rename_data_enabled()
|
||||
&& (commit_set.tier_free_version_rename_data_source_capable()
|
||||
&& (all_commit_disks_local
|
||||
|| commit_set.tier_free_version_rename_data_source_capable()
|
||||
|| tier_free_version_source_reuse_rename_data_assume_homogeneous_enabled());
|
||||
let mut put_tier_free_version_source = if commit_put_tier_free_version_id.is_some()
|
||||
&& commit_tier_free_version_receipt_sink.is_some()
|
||||
@@ -5630,9 +5634,11 @@ impl SetDisks {
|
||||
);
|
||||
let lock_held_tail_handoff_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Ok(rename_commit) = rename_result.as_mut() {
|
||||
commit_set.observe_tier_free_version_rename_data_source_capability(
|
||||
rename_commit.tail_drain.is_none() && rename_commit.old_current_source_all_successful_checked,
|
||||
);
|
||||
if rename_commit.tail_drain.is_none() || !rename_commit.old_current_source_all_successful_checked {
|
||||
commit_set.observe_tier_free_version_rename_data_source_capability(
|
||||
rename_commit.tail_drain.is_none() && rename_commit.old_current_source_all_successful_checked,
|
||||
);
|
||||
}
|
||||
commit_set.record_capacity_scope_if_needed(commit_capacity_scope_token, &rename_commit.capacity_disks);
|
||||
// Install the tail watcher before any post-commit await. The
|
||||
// latch keeps namespace guards through their prior handoff point.
|
||||
@@ -5700,7 +5706,12 @@ impl SetDisks {
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
targets| {
|
||||
targets,
|
||||
old_current_source_all_successful_checked,
|
||||
all_online_local_succeeded| {
|
||||
cleanup_set.observe_tier_free_version_rename_data_source_capability(
|
||||
old_current_source_all_successful_checked,
|
||||
);
|
||||
drop(object_lock_guard);
|
||||
drop(write_context);
|
||||
drop(publication_guard);
|
||||
@@ -5727,7 +5738,10 @@ impl SetDisks {
|
||||
)
|
||||
.await;
|
||||
let tmp_delete_started = rustfs_io_metrics::put_stage_timer();
|
||||
if zero_target_cleanup && rename_tail_cleanup_zero_target_tmp_delete_skip_enabled() {
|
||||
if zero_target_cleanup
|
||||
&& (all_online_local_succeeded
|
||||
|| rename_tail_cleanup_zero_target_tmp_delete_skip_enabled())
|
||||
{
|
||||
rustfs_io_metrics::record_put_object_stage_duration(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_TAIL_CLEANUP_TMP_DELETE_COUNTERFACTUAL_SKIP,
|
||||
1.0,
|
||||
@@ -9262,6 +9276,10 @@ impl SetDisks {
|
||||
// cannot safely target the object name after bucket recreation.
|
||||
return Ok(());
|
||||
};
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if crate::set_disk::skip_e2e_mrf_fallback(bucket, object).await {
|
||||
return Ok(());
|
||||
}
|
||||
let mut request = rustfs_heal_contracts::heal_channel::create_heal_request_with_options(
|
||||
bucket.to_string(),
|
||||
Some(object.to_string()),
|
||||
|
||||
@@ -599,6 +599,33 @@ impl ECStore {
|
||||
Ok((write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta))
|
||||
}
|
||||
|
||||
/// Snapshot only the object sets participating in this request. The sampler
|
||||
/// resolves their physical leaves before Heal enters namespace/cache locks.
|
||||
pub fn physical_heal_plan(&self, object: &str, opts: &HealOpts) -> Result<Option<crate::disk::PhysicalResourcePlan>> {
|
||||
if let Some(index) = opts.pool
|
||||
&& index >= self.pools.len()
|
||||
{
|
||||
return Err(invalid_heal_pool_index(index, self.pools.len()));
|
||||
}
|
||||
let object = rustfs_utils::path::encode_dir_object_ref(object);
|
||||
let mut selected: Option<crate::disk::PhysicalResourcePlan> = None;
|
||||
for (index, pool) in self.pools.iter().enumerate() {
|
||||
if opts.pool.is_some_and(|requested| requested != index) {
|
||||
continue;
|
||||
}
|
||||
let set = pool.get_disks_for_heal_object(&object, opts)?;
|
||||
let Some(plan) = set.physical_heal_admission.plan() else { return Ok(None) };
|
||||
if let Some(selected) = &mut selected {
|
||||
if selected.union_with(&plan).is_err() {
|
||||
return Ok(None);
|
||||
}
|
||||
} else {
|
||||
selected = Some(plan);
|
||||
}
|
||||
}
|
||||
Ok(selected)
|
||||
}
|
||||
|
||||
fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
|
||||
match opts.pool {
|
||||
Some(pool_idx) => Ok(vec![
|
||||
@@ -1609,21 +1636,25 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
|
||||
let format = FormatV3::new(1, 1);
|
||||
let endpoint_url = format!("http://127.0.0.1:{}/data", 19000 + pool_idx);
|
||||
let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(pool_idx);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(0);
|
||||
|
||||
async fn minimal_heal_pool(pool_idx: usize, set_count: usize) -> Arc<Sets> {
|
||||
let format = FormatV3::new(set_count, 1);
|
||||
let endpoints = (0..set_count)
|
||||
.map(|set_idx| {
|
||||
let endpoint_url = format!("http://127.0.0.1:{}/data", 19000 + pool_idx + set_idx * 2);
|
||||
let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(pool_idx);
|
||||
endpoint.set_set_index(set_idx);
|
||||
endpoint.set_disk_index(0);
|
||||
endpoint
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
Sets::new(
|
||||
vec![None],
|
||||
vec![None; set_count],
|
||||
&PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
set_count,
|
||||
drives_per_set: 1,
|
||||
endpoints: Endpoints::from(vec![endpoint]),
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: String::new(),
|
||||
platform: String::new(),
|
||||
},
|
||||
@@ -2451,7 +2482,7 @@ mod tests {
|
||||
ECStore {
|
||||
id: Uuid::new_v4(),
|
||||
disk_map: HashMap::new(),
|
||||
pools: vec![minimal_heal_pool(0).await, minimal_heal_pool(1).await],
|
||||
pools: vec![minimal_heal_pool(0, 1).await, minimal_heal_pool(1, 1).await],
|
||||
peer_sys: S3PeerSys {
|
||||
clients: Vec::new(),
|
||||
pools_count: 2,
|
||||
@@ -2466,6 +2497,88 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn physical_heal_plan_uses_the_real_pool_set_and_directory_placement() {
|
||||
use crate::disk::PhysicalDiskBinding;
|
||||
use rustfs_concurrency::adaptive::{
|
||||
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
|
||||
};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
let mut store = minimal_heal_store().await;
|
||||
store.pools[0] = minimal_heal_pool(0, 2).await;
|
||||
let pool = &store.pools[0];
|
||||
let a = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let b = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
for (set, gate) in pool.disk_set.iter().zip([&a, &b]) {
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("physical fixture");
|
||||
set.physical_heal_admission
|
||||
.set_test_plan(binding.plan().expect("live").resource_plan());
|
||||
}
|
||||
let node = NodeAdmission::new(NonZeroU16::MIN);
|
||||
node.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
let directory = (0..100)
|
||||
.map(|n| format!("dir{n}/"))
|
||||
.find(|name| {
|
||||
store.pools[0].get_disks_by_key(name).set_index
|
||||
!= store.pools[0].get_disks_by_key(&encode_dir_object(name)).set_index
|
||||
})
|
||||
.expect("different directory placement");
|
||||
for object in ["plain-object", directory.as_str()] {
|
||||
for set in [None, Some(0), Some(1)] {
|
||||
let opts = HealOpts {
|
||||
pool: Some(0),
|
||||
set,
|
||||
..Default::default()
|
||||
};
|
||||
let expected = store.pools[0]
|
||||
.get_disks_for_heal_object(&encode_dir_object(object), &opts)
|
||||
.expect("placement")
|
||||
.set_index;
|
||||
let permit = store
|
||||
.physical_heal_plan(object, &opts)
|
||||
.expect("valid selection")
|
||||
.expect("known set")
|
||||
.try_acquire(&node)
|
||||
.expect("physical admission");
|
||||
assert_eq!((a.snapshot().active, b.snapshot().active), if expected == 0 { (1, 0) } else { (0, 1) });
|
||||
drop(permit);
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
store
|
||||
.physical_heal_plan("object", &HealOpts::default())
|
||||
.expect("valid all pools")
|
||||
.is_none(),
|
||||
"an undiscovered participating pool cannot be skipped"
|
||||
);
|
||||
for opts in [
|
||||
HealOpts {
|
||||
pool: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(2),
|
||||
..Default::default()
|
||||
},
|
||||
] {
|
||||
assert!(matches!(
|
||||
store.physical_heal_plan("object", &opts),
|
||||
Err(StorageError::InvalidArgument(..))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove_pool_meta_shard(store: &ECStore, pool_idx: usize) -> DiskStore {
|
||||
let target_set = store.pools[pool_idx].get_disks_by_key(POOL_META_NAME);
|
||||
let missing_disk = target_set.disks.read().await[0]
|
||||
@@ -2845,9 +2958,18 @@ mod tests {
|
||||
let (configuration, _) = read_config_no_lock_preserve_empty_with_metadata(store.clone(), &object)
|
||||
.await
|
||||
.expect("read the current bucket configuration");
|
||||
save_config(store.clone(), &object, configuration)
|
||||
.await
|
||||
.expect("publish a newer config revision in the same bucket incarnation");
|
||||
crate::config::com::save_config_with_opts(
|
||||
store.clone(),
|
||||
&object,
|
||||
configuration,
|
||||
&ObjectOptions {
|
||||
max_parity: true,
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("publish a newer config revision on the target disk");
|
||||
let committed =
|
||||
std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("new target configuration");
|
||||
assert!(
|
||||
|
||||
@@ -5340,6 +5340,203 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn conditional_multipart_must_not_commit_against_an_older_readable_pool() {
|
||||
use crate::storage_api_contracts::multipart::CompletePart;
|
||||
use crate::storage_api_contracts::object::HTTPPreconditions;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create conditional multipart test directory");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "conditional-mpu-unreadable", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("conditional-mpu-{}", uuid::Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create conditional multipart test bucket");
|
||||
let mut old_body = PutObjReader::from_vec(b"older-pool".to_vec());
|
||||
let old = store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut old_body,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write old object to staging pool");
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload in pool 0");
|
||||
let mut part_body = PutObjReader::from_vec(b"stale-writer".to_vec());
|
||||
let part = store.pools[0]
|
||||
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut part_body, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage multipart part in pool 0");
|
||||
let mut new_body = PutObjReader::from_vec(b"newer-pool".to_vec());
|
||||
let new = store.pools[1]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut new_body,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write newer object to pool 1");
|
||||
let old_etag = old.etag.clone().expect("old object must have an ETag");
|
||||
let before = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
|
||||
|
||||
let mut metadata = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let path = temp_dir
|
||||
.path()
|
||||
.join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/xl.meta"));
|
||||
let original = tokio::fs::read(&path)
|
||||
.await
|
||||
.expect("save original object metadata before fault injection");
|
||||
metadata.push((path, original));
|
||||
}
|
||||
let mut fault_error = None;
|
||||
for (path, _) in &metadata {
|
||||
if let Err(err) = tokio::fs::write(path, b"invalid object metadata").await {
|
||||
fault_error = Some(err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut fault_contents = Vec::new();
|
||||
if fault_error.is_none() {
|
||||
for (path, _) in &metadata {
|
||||
fault_contents.push(tokio::fs::read(path).await);
|
||||
}
|
||||
store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.invalidate_get_object_metadata_cache(&bucket, object)
|
||||
.await;
|
||||
}
|
||||
let unreadable = if fault_error.is_none() {
|
||||
Some(
|
||||
store.pools[1]
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let completed = if fault_error.is_none() {
|
||||
let completion_store = Arc::clone(&store);
|
||||
let completion_bucket = bucket.clone();
|
||||
let upload_id = upload.upload_id.clone();
|
||||
let part_etag = part.etag.clone();
|
||||
Some(
|
||||
tokio::spawn(async move {
|
||||
completion_store
|
||||
.complete_multipart_upload(
|
||||
&completion_bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: part_etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(old_etag),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut restore_error = None;
|
||||
for (path, original) in &metadata {
|
||||
if let Err(err) = tokio::fs::write(path, original).await {
|
||||
restore_error = Some(err);
|
||||
}
|
||||
}
|
||||
store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.invalidate_get_object_metadata_cache(&bucket, object)
|
||||
.await;
|
||||
let after = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
|
||||
let recovered_body = match store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(mut reader) => {
|
||||
let mut body = Vec::new();
|
||||
let result = reader
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.map(|_| body)
|
||||
.map_err(|err| err.to_string());
|
||||
drop(reader);
|
||||
result
|
||||
}
|
||||
Err(err) => Err(err.to_string()),
|
||||
};
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, object, &upload.upload_id, None, 100, &ObjectOptions::default())
|
||||
.await;
|
||||
shutdown.cancel();
|
||||
|
||||
assert!(fault_error.is_none(), "failed to inject metadata fault: {fault_error:?}");
|
||||
assert!(restore_error.is_none(), "failed to restore metadata: {restore_error:?}");
|
||||
assert_eq!(fault_contents.len(), 4, "all four pool 1 metadata files must be inspected");
|
||||
for observed in fault_contents {
|
||||
assert_eq!(
|
||||
observed.expect("read injected metadata"),
|
||||
b"invalid object metadata",
|
||||
"each pool 1 metadata file must contain the injected corruption"
|
||||
);
|
||||
}
|
||||
assert_eq!(before.expect("latest object before fault").etag, new.etag);
|
||||
assert_ne!(old.etag, new.etag, "fixture bodies must produce different ETags");
|
||||
let read_error = unreadable
|
||||
.expect("metadata fault should have been injected")
|
||||
.expect_err("the newer pool must fail its object metadata read during the fault");
|
||||
assert!(
|
||||
!crate::error::is_err_object_not_found(&read_error) && !crate::error::is_err_version_not_found(&read_error),
|
||||
"corrupt metadata must produce a non-not-found read error: {read_error:?}"
|
||||
);
|
||||
let completed = completed
|
||||
.expect("completion should have been attempted")
|
||||
.expect("completion task should not panic");
|
||||
let Err(completion_error) = completed else {
|
||||
panic!("stale If-Match must not commit while the newer object's metadata is unreadable");
|
||||
};
|
||||
assert_eq!(completion_error, read_error, "completion must propagate the unreadable pool's error");
|
||||
assert_eq!(after.expect("newer object after recovery").etag, new.etag);
|
||||
assert_eq!(recovered_body.expect("newer body after recovery"), b"newer-pool");
|
||||
let parts = parts.expect("failed completion must leave the upload retryable");
|
||||
assert_eq!(parts.parts.len(), 1);
|
||||
assert_eq!(parts.parts[0].etag, part.etag);
|
||||
}
|
||||
|
||||
async fn migrate_versioned_decommission_test_object(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
|
||||
@@ -458,8 +458,8 @@ pub(crate) mod utils;
|
||||
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, get_disk_infos, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
|
||||
get_disk_infos, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
|
||||
@@ -498,6 +498,17 @@ pub struct ECStore {
|
||||
pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
|
||||
}
|
||||
|
||||
impl Drop for ECStore {
|
||||
/// Clear the instance context's `BucketMetadataSys` when its backing store
|
||||
/// is dropped. Without this, the shared bootstrap context retains a
|
||||
/// `BucketMetadataSys` whose `Weak<ECStore>` is dangling after the store
|
||||
/// goes away (e.g. between sequential test fixtures that share the
|
||||
/// bootstrap), and the next `object_store()` call panics.
|
||||
fn drop(&mut self) {
|
||||
self.ctx.clear_bucket_metadata_sys_if_stale();
|
||||
}
|
||||
}
|
||||
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
|
||||
@@ -1388,6 +1399,18 @@ impl crate::storage_api_contracts::object::ObjectIO for ECStore {
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
pub(crate) fn all_disks_local_and_online(&self) -> bool {
|
||||
!self.disk_map.is_empty()
|
||||
&& self.disk_map.values().all(|disks| {
|
||||
!disks.is_empty()
|
||||
&& disks.iter().all(|disk| {
|
||||
disk.as_ref().is_some_and(|d| {
|
||||
d.is_local() && d.runtime_state() == crate::disk::health_state::RuntimeDriveHealthState::Online
|
||||
})
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// `put_object` plus the rename_data old-size backfill
|
||||
/// (rustfs/backlog#1009); see `SetDisks::put_object_with_old_current_size`.
|
||||
/// Post-write hooks (immediate ILM transition enqueue, list-cache
|
||||
@@ -1400,19 +1423,36 @@ impl ECStore {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
let result = match self.handle_put_object(bucket, object, data, opts).await {
|
||||
Ok((object_info, old_current_size)) => {
|
||||
enqueue_transition_after_write(self, Ok(object_info), LcEventSrc::S3PutObject, opts)
|
||||
.await
|
||||
.map(|object_info| (object_info, old_current_size))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
let result = self.handle_put_object(bucket, object, data, opts).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn copy_object_with_old_current_size(
|
||||
&self,
|
||||
src_bucket: &str,
|
||||
src_object: &str,
|
||||
dst_bucket: &str,
|
||||
dst_object: &str,
|
||||
src_info: &mut ObjectInfo,
|
||||
src_opts: &ObjectOptions,
|
||||
dst_opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<crate::disk::OldCurrentSize>)> {
|
||||
let (oi, old_current_size) = box_copy_object_handler(move || {
|
||||
self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
|
||||
})
|
||||
.await?;
|
||||
let result = enqueue_transition_after_write(self, Ok(oi), LcEventSrc::S3CopyObject, dst_opts)
|
||||
.await
|
||||
.map(|oi| (oi, old_current_size));
|
||||
if result.is_ok() {
|
||||
list_objects::observe_scanner_namespace_mutations(dst_bucket, 1);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
@@ -1490,20 +1530,9 @@ impl crate::storage_api_contracts::object::ObjectOperations for ECStore {
|
||||
src_opts: &ObjectOptions,
|
||||
dst_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let result = enqueue_transition_after_write(
|
||||
self,
|
||||
box_copy_object_handler(move || {
|
||||
self.handle_copy_object(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
|
||||
})
|
||||
.await,
|
||||
LcEventSrc::S3CopyObject,
|
||||
dst_opts,
|
||||
)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_scanner_namespace_mutations(dst_bucket, 1);
|
||||
}
|
||||
result
|
||||
self.copy_object_with_old_current_size(src_bucket, src_object, dst_bucket, dst_object, src_info, src_opts, dst_opts)
|
||||
.await
|
||||
.map(|(oi, _)| oi)
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -1763,15 +1792,10 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for ECStore {
|
||||
uploaded_parts: Vec<CompletePart>,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
let result = enqueue_transition_after_write(
|
||||
self.as_ref(),
|
||||
self.clone()
|
||||
.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
|
||||
.await,
|
||||
LcEventSrc::S3CompleteMultipartUpload,
|
||||
opts,
|
||||
)
|
||||
.await;
|
||||
let result = self
|
||||
.clone()
|
||||
.handle_complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
|
||||
.await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
use crate::storage_api_contracts::object::ObjectOperations as _;
|
||||
use futures::{StreamExt, stream};
|
||||
use rustfs_utils::path::encode_dir_object_ref;
|
||||
use std::collections::HashSet;
|
||||
|
||||
const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
|
||||
@@ -612,7 +613,7 @@ impl ECStore {
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& !opts.data_movement
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, &opts).await
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, &opts, false).await
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
@@ -640,6 +641,19 @@ impl ECStore {
|
||||
return Ok((res, idx, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
|
||||
if self.single_pool() && !self.is_suspended(0).await && !self.is_pool_rebalancing(0).await {
|
||||
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
|
||||
let lock_object = encode_dir_object(object);
|
||||
let res = self
|
||||
.run_external_decommission_capacity_object_mutation(0, bucket, &lock_object, object, opts, |opts| async move {
|
||||
self.pools[0].new_multipart_upload(bucket, object, &opts).await
|
||||
})
|
||||
.await?;
|
||||
return Ok((res, 0, expected_bucket_incarnation_id));
|
||||
}
|
||||
|
||||
for (idx, pool) in self.pools.iter().enumerate() {
|
||||
if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
|
||||
continue;
|
||||
@@ -1141,12 +1155,31 @@ impl ECStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn check_write_precondition(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Option<StorageError> {
|
||||
async fn check_write_precondition(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
require_all_pools: bool,
|
||||
) -> Option<StorageError> {
|
||||
let mut lookup_opts = opts.clone();
|
||||
let http_preconditions = lookup_opts.http_preconditions?;
|
||||
lookup_opts.http_preconditions = None;
|
||||
lookup_opts.no_lock = true;
|
||||
let oi = self.get_object_info_for_delete(bucket, object, &lookup_opts).await;
|
||||
let oi = if require_all_pools {
|
||||
lookup_opts.metadata_cache_safe = false;
|
||||
lookup_opts.suppress_read_repair = true;
|
||||
// Only completion uses this branch, after validating bucket and object.
|
||||
let encoded_object = encode_dir_object_ref(object);
|
||||
self.get_latest_object_info_for_conditional_write(bucket, encoded_object.as_ref(), &lookup_opts)
|
||||
.await
|
||||
.and_then(|(info, _)| {
|
||||
lookup_opts.precondition_check(&info)?;
|
||||
Ok(info)
|
||||
})
|
||||
} else {
|
||||
self.get_object_info_for_delete(bucket, object, &lookup_opts).await
|
||||
};
|
||||
crate::set_disk::SetDisks::evaluate_write_precondition(oi, &http_preconditions, bucket, object, opts)
|
||||
}
|
||||
|
||||
@@ -1167,19 +1200,22 @@ impl ECStore {
|
||||
let opts = &opts;
|
||||
|
||||
if self.single_pool() {
|
||||
return self.pools[0]
|
||||
let result = self.pools[0]
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
|
||||
.await;
|
||||
return enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, opts).await;
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& !opts.data_movement
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
|
||||
{
|
||||
return Err(err);
|
||||
if opts.http_preconditions.is_some() && !opts.data_movement {
|
||||
// Preserve explicitly scoped internal reads and borrowed-lock callers.
|
||||
let require_all_pools =
|
||||
!opts.replication_request && !opts.no_lock && !opts.skip_decommissioned && !opts.skip_rebalancing;
|
||||
if let Some(err) = self.check_write_precondition(bucket, object, opts, require_all_pools).await {
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
|
||||
match self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await {
|
||||
let result = match self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await {
|
||||
Ok(pool_idx) => {
|
||||
self.pools[pool_idx]
|
||||
.clone()
|
||||
@@ -1190,21 +1226,29 @@ impl ECStore {
|
||||
// the pool that published it; other pools answer InvalidUploadID.
|
||||
Err(err) if is_err_invalid_upload_id(&err) => {
|
||||
let mut missing = err;
|
||||
let mut found = None;
|
||||
for pool_idx in self.existing_multipart_pool_order().await {
|
||||
match self.pools[pool_idx]
|
||||
.clone()
|
||||
.complete_multipart_upload(bucket, object, upload_id, uploaded_parts.clone(), opts)
|
||||
.await
|
||||
{
|
||||
Ok(info) => return Ok(info),
|
||||
Ok(info) => {
|
||||
found = Some(Ok(info));
|
||||
break;
|
||||
}
|
||||
Err(err) if is_err_invalid_upload_id(&err) => missing = err,
|
||||
Err(err) => return Err(err),
|
||||
Err(err) => {
|
||||
found = Some(Err(err));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(missing)
|
||||
found.unwrap_or(Err(missing))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
};
|
||||
enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, opts).await
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
@@ -1326,6 +1370,7 @@ impl ECStore {
|
||||
// Data movement already owns the pool-meta write lease. Forward its
|
||||
// loss signal into SetDisks so commit fencing observes the same lease
|
||||
// without trying to reacquire the namespace.
|
||||
let mut transition_opts = opts.clone();
|
||||
let result = self
|
||||
.run_decommission_capacity_admitted_mutation_with_capacity_lease(
|
||||
target_pool_idx,
|
||||
@@ -1341,8 +1386,9 @@ impl ECStore {
|
||||
)
|
||||
.await;
|
||||
drop(publication_guard);
|
||||
transition_opts.no_lock = request_opts.no_lock;
|
||||
let result =
|
||||
enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, request_opts).await;
|
||||
enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, &transition_opts).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ use crate::bucket::lifecycle::{
|
||||
};
|
||||
use crate::bucket::metadata_sys::{
|
||||
acquire_bucket_metadata_transaction_read_lock_in, get_cached_bucket_incarnation_id_in, get_delete_config_from_disk_in,
|
||||
get_object_lock_config_and_incarnation_from_disk_in,
|
||||
get_object_lock_and_quota_config_from_disk_with_guard_in, get_object_lock_config_and_incarnation_from_disk_with_guard_in,
|
||||
};
|
||||
use crate::bucket::object_lock::objectlock_sys::{
|
||||
check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
|
||||
@@ -3162,7 +3162,7 @@ impl ECStore {
|
||||
let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
|
||||
let (state, bucket_incarnation_id, config_revision) =
|
||||
get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
|
||||
get_object_lock_config_and_incarnation_from_disk_with_guard_in(&self.ctx, bucket, Some(&lifecycle_guard)).await?;
|
||||
if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
|
||||
return Err(Error::other("bucket lifecycle lock was lost while loading the Object Lock snapshot"));
|
||||
}
|
||||
@@ -3177,6 +3177,30 @@ impl ECStore {
|
||||
)))
|
||||
}
|
||||
|
||||
async fn object_lock_config_snapshot_for_single_put(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
|
||||
check_valid_bucket_name(bucket)?;
|
||||
let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
|
||||
let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
|
||||
let (state, bucket_incarnation_id, config_revision, quota_res) =
|
||||
get_object_lock_and_quota_config_from_disk_with_guard_in(&self.ctx, bucket, Some(&lifecycle_guard)).await?;
|
||||
if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
|
||||
return Err(Error::other("bucket lifecycle lock was lost while loading the Object Lock snapshot"));
|
||||
}
|
||||
let snapshot = ObjectLockConfigSnapshot::for_guarded_store_bucket(
|
||||
self.id,
|
||||
bucket,
|
||||
bucket_incarnation_id,
|
||||
config_revision,
|
||||
state,
|
||||
lifecycle_guard,
|
||||
metadata_guard,
|
||||
);
|
||||
if !self.ctx.lock_manager().is_disabled() {
|
||||
snapshot.set_fresh_put_quota(quota_res);
|
||||
}
|
||||
Ok(Arc::new(snapshot))
|
||||
}
|
||||
|
||||
pub(super) async fn object_lock_config_snapshot_under_lifecycle_fence(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -3187,7 +3211,7 @@ impl ECStore {
|
||||
}
|
||||
let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
|
||||
let (state, bucket_incarnation_id, config_revision) =
|
||||
get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
|
||||
get_object_lock_config_and_incarnation_from_disk_with_guard_in(&self.ctx, bucket, Some(&metadata_guard)).await?;
|
||||
if lifecycle_fence.is_lock_lost() || metadata_guard.is_lock_lost() {
|
||||
return Err(Error::other("bucket lock was lost while loading the Object Lock snapshot"));
|
||||
}
|
||||
@@ -4652,7 +4676,7 @@ impl ECStore {
|
||||
.expected_bucket_incarnation_id
|
||||
.ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
|
||||
if opts.object_lock_config_snapshot.is_none() {
|
||||
opts.object_lock_config_snapshot = Some(self.object_lock_config_snapshot(bucket).await?);
|
||||
opts.object_lock_config_snapshot = Some(self.object_lock_config_snapshot_for_single_put(bucket).await?);
|
||||
}
|
||||
let snapshot = match opts.object_lock_config_snapshot.as_ref() {
|
||||
Some(snapshot) if snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) => {
|
||||
@@ -4741,6 +4765,7 @@ impl ECStore {
|
||||
self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence.or(fixed_read_anchor))
|
||||
.await;
|
||||
let expected_data_bytes = usize::try_from(data.size()).ok();
|
||||
let mut transition_opts = opts.clone();
|
||||
let result = self
|
||||
.run_decommission_capacity_admitted_mutation_with_capacity_lease(
|
||||
idx,
|
||||
@@ -4772,7 +4797,8 @@ impl ECStore {
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let result = enqueue_transition_after_write(self, result, LcEventSrc::S3PutObject, request_opts).await;
|
||||
transition_opts.no_lock = request_opts.no_lock;
|
||||
let result = enqueue_transition_after_write(self, result, LcEventSrc::S3PutObject, &transition_opts).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_scanner_namespace_mutations(bucket, 1);
|
||||
}
|
||||
@@ -4788,7 +4814,7 @@ impl ECStore {
|
||||
data: &mut PutObjReader,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
let (object, opts) = self.prepare_put_object(bucket, object, opts).await?;
|
||||
let (object, opts) = Box::pin(self.prepare_put_object(bucket, object, opts)).await?;
|
||||
let idx = self
|
||||
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
|
||||
.await?;
|
||||
@@ -4797,11 +4823,16 @@ impl ECStore {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
|
||||
let result = self.pools[idx]
|
||||
.put_object_with_old_current_size(bucket, object.as_str(), data, &opts)
|
||||
.await;
|
||||
let result = Box::pin(self.pools[idx].put_object_with_old_current_size(bucket, object.as_str(), data, &opts)).await;
|
||||
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
|
||||
result
|
||||
match result {
|
||||
Ok((object_info, old_current_size)) => {
|
||||
Box::pin(enqueue_transition_after_write(self, Ok(object_info), LcEventSrc::S3PutObject, &opts))
|
||||
.await
|
||||
.map(|object_info| (object_info, old_current_size))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(level = "trace", skip(self))]
|
||||
@@ -4859,7 +4890,7 @@ impl ECStore {
|
||||
src_info: &mut ObjectInfo,
|
||||
src_opts: &ObjectOptions,
|
||||
dst_opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
|
||||
check_copy_obj_args(src_bucket, src_object)?;
|
||||
if src_bucket == dst_bucket && src_object == dst_object {
|
||||
// In-place copies (metadata replace, restore) rewrite a key that
|
||||
@@ -4938,9 +4969,17 @@ impl ECStore {
|
||||
};
|
||||
|
||||
if cp_src_dst_same {
|
||||
let (_, pool_idx) = self
|
||||
.get_latest_accessible_object_info_with_idx(src_bucket, &src_object, &version_aware_lookup_opts(src_opts, true))
|
||||
.await?;
|
||||
let pool_idx = if self.single_pool() {
|
||||
0
|
||||
} else {
|
||||
self.get_latest_accessible_object_info_with_idx(
|
||||
src_bucket,
|
||||
&src_object,
|
||||
&version_aware_lookup_opts(src_opts, true),
|
||||
)
|
||||
.await?
|
||||
.1
|
||||
};
|
||||
let source_pool_writable = !self.is_suspended(pool_idx).await && !self.is_pool_rebalancing(pool_idx).await;
|
||||
|
||||
if source_pool_writable
|
||||
@@ -4961,7 +5000,8 @@ impl ECStore {
|
||||
.await
|
||||
},
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
.map(|oi| (oi, None));
|
||||
}
|
||||
|
||||
if source_pool_writable && !dst_opts.versioned && src_opts.version_id.is_none() {
|
||||
@@ -4989,7 +5029,8 @@ impl ECStore {
|
||||
.await
|
||||
},
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
.map(|oi| (oi, None));
|
||||
}
|
||||
// Transitioned object self-copy: restore from tier into the same pool.
|
||||
let mut put_opts = ObjectOptions {
|
||||
@@ -5017,7 +5058,7 @@ impl ECStore {
|
||||
return if let Some(reader) = src_info.put_object_reader.as_mut() {
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
|
||||
let result = self.pools[pool_idx]
|
||||
.put_object(dst_bucket, &dst_object, reader, &put_opts)
|
||||
.put_object_with_old_current_size(dst_bucket, &dst_object, reader, &put_opts)
|
||||
.await;
|
||||
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
|
||||
result
|
||||
@@ -5061,7 +5102,7 @@ impl ECStore {
|
||||
}
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
|
||||
let result = self.pools[pool_idx]
|
||||
.put_object(dst_bucket, &dst_object, reader, &put_opts)
|
||||
.put_object_with_old_current_size(dst_bucket, &dst_object, reader, &put_opts)
|
||||
.await;
|
||||
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
|
||||
return result;
|
||||
@@ -5081,11 +5122,14 @@ impl ECStore {
|
||||
.await
|
||||
},
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
.map(|oi| (oi, None));
|
||||
}
|
||||
}
|
||||
|
||||
let pool_idx = if dst_opts.no_lock {
|
||||
let pool_idx = if self.single_pool() && !self.is_suspended(0).await && !self.is_pool_rebalancing(0).await {
|
||||
0
|
||||
} else if dst_opts.no_lock {
|
||||
self.get_pool_idx_no_lock(dst_bucket, &dst_object, src_info.size).await?
|
||||
} else {
|
||||
self.get_pool_idx(dst_bucket, &dst_object, src_info.size).await?
|
||||
@@ -5118,7 +5162,7 @@ impl ECStore {
|
||||
if let Some(put_object_reader) = src_info.put_object_reader.as_mut() {
|
||||
let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
|
||||
let result = self.pools[pool_idx]
|
||||
.put_object(dst_bucket, dst_object_name, put_object_reader, &put_opts)
|
||||
.put_object_with_old_current_size(dst_bucket, dst_object_name, put_object_reader, &put_opts)
|
||||
.await;
|
||||
enqueue_recorded_tier_free_versions(self, receipt_sink).await;
|
||||
return result;
|
||||
@@ -5350,8 +5394,9 @@ impl ECStore {
|
||||
opts.add_namespace_lock_guard(guard);
|
||||
}
|
||||
let current_bucket_incarnation_id = if let Some(metadata_guard) = _object_lock_metadata_guard.as_ref() {
|
||||
let guard_ref = _bucket_lifecycle_guard.as_deref().unwrap_or_else(|| metadata_guard.as_ref());
|
||||
let (state, incarnation_id, config_revision) =
|
||||
get_delete_config_from_disk_in(&self.ctx, bucket, None, opts.lifecycle_expiry_policy.as_ref()).await?;
|
||||
get_delete_config_from_disk_in(&self.ctx, bucket, Some(guard_ref), opts.lifecycle_expiry_policy.as_ref()).await?;
|
||||
opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
|
||||
self.id,
|
||||
bucket,
|
||||
@@ -5591,6 +5636,42 @@ impl ECStore {
|
||||
return Ok(obj);
|
||||
}
|
||||
|
||||
if self.single_pool()
|
||||
&& !creates_latest_marker
|
||||
&& !opts.data_movement
|
||||
&& opts.http_preconditions.is_none()
|
||||
&& !opts.replication_request
|
||||
&& !opts.versioned
|
||||
&& !opts.version_suspended
|
||||
&& opts.version_id.is_none()
|
||||
&& !self.is_suspended(0).await
|
||||
&& !self.is_pool_rebalancing(0).await
|
||||
{
|
||||
let pool = self.pools[0].clone();
|
||||
match self
|
||||
.run_external_decommission_capacity_object_delete(0, bucket, object, object, opts.clone(), |opts| async move {
|
||||
pool.delete_object(bucket, object, opts).await
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(mut res) => {
|
||||
res.name = decode_dir_object(res.name.as_str());
|
||||
return Ok(res);
|
||||
}
|
||||
Err(err) if is_err_read_quorum(&err) => return Err(StorageError::ErasureWriteQuorum),
|
||||
Err(err) => {
|
||||
if should_purge_orphan_dir_on_missing(&err, object) && self.purge_orphan_dir_object(bucket, object).await {
|
||||
return Ok(ObjectInfo {
|
||||
bucket: bucket.to_owned(),
|
||||
name: decode_dir_object(object),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Determine which pool contains it
|
||||
let existing_pool_info = if creates_latest_marker {
|
||||
self.get_pool_info_for_delete_marker(bucket, object, &gopts).await
|
||||
@@ -9903,6 +9984,7 @@ mod tests {
|
||||
copy_store
|
||||
.handle_copy_object(bucket, source, bucket, target, &mut source_info, &ObjectOptions::default(), &options)
|
||||
.await
|
||||
.map(|(oi, _)| oi)
|
||||
});
|
||||
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
|
||||
tokio::time::timeout_at(deadline, barrier.wait_until_paused())
|
||||
|
||||
@@ -99,6 +99,9 @@ impl ECStore {
|
||||
fi: FileInfo,
|
||||
expected: Uuid,
|
||||
) -> DiskResult<()> {
|
||||
if crate::disk::remote_io_admission::active() {
|
||||
local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
}
|
||||
let disk_ref = disk_ref.to_owned();
|
||||
let path = target.1.to_owned();
|
||||
self.run_bucket_heal_at_incarnation(target.0, expected, &Default::default(), move |store, volume, _| async move {
|
||||
@@ -117,6 +120,9 @@ impl ECStore {
|
||||
opts: DeleteOptions,
|
||||
expected: Uuid,
|
||||
) -> DiskResult<()> {
|
||||
if crate::disk::remote_io_admission::active() {
|
||||
local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
}
|
||||
let disk_ref = disk_ref.to_owned();
|
||||
let path = target.1.to_owned();
|
||||
self.run_bucket_heal_at_incarnation(target.0, expected, &Default::default(), move |store, volume, _| async move {
|
||||
@@ -144,6 +150,9 @@ impl ECStore {
|
||||
if expected.is_nil() || opts.undo_write || self.ctx.lock_manager().is_disabled() {
|
||||
return Err(DiskError::other("invalid incarnation-bound heal deletion"));
|
||||
}
|
||||
if crate::disk::remote_io_admission::active() {
|
||||
local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
}
|
||||
let fence = Arc::new(
|
||||
self.acquire_bucket_incarnation_fence(target.0, expected)
|
||||
.await
|
||||
@@ -300,7 +309,9 @@ impl ECStore {
|
||||
// The optional ID is a cold lookup to cache only after final admission.
|
||||
async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||
let map = ctx.local_disk_map();
|
||||
if let Some(disk) = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned() {
|
||||
let path_disk = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned();
|
||||
if let Some(disk) = path_disk {
|
||||
admit_rpc_disk(&disk)?;
|
||||
return Ok((disk, None));
|
||||
}
|
||||
let disk_id = Uuid::parse_str(disk_ref).map_err(|_| DiskError::DiskNotFound)?;
|
||||
@@ -309,11 +320,16 @@ async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> Dis
|
||||
let cached_disk = map.read().await.get(&path).and_then(Option::as_ref).cloned();
|
||||
if let Some(disk) = cached_disk
|
||||
&& matches!(disk.as_ref(), Disk::Local(_))
|
||||
&& disk.get_disk_id().await? == Some(disk_id)
|
||||
{
|
||||
return Ok((disk, None));
|
||||
admit_rpc_disk(&disk)?;
|
||||
if disk.get_disk_id().await? == Some(disk_id) {
|
||||
return Ok((disk, None));
|
||||
}
|
||||
}
|
||||
}
|
||||
if crate::disk::remote_io_admission::active() {
|
||||
return Err(DiskError::Io(std::io::Error::from(std::io::ErrorKind::WouldBlock)));
|
||||
}
|
||||
let disks: Vec<_> = map.read().await.values().filter_map(Clone::clone).collect();
|
||||
// Disk identity may perform format I/O. No registry guard spans this await.
|
||||
for disk in disks {
|
||||
@@ -324,6 +340,17 @@ async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> Dis
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
|
||||
fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
|
||||
use crate::disk::remote_io_admission;
|
||||
if remote_io_admission::active() {
|
||||
let binding = disk
|
||||
.physical_admission()
|
||||
.ok_or_else(|| DiskError::Io(std::io::Error::from(std::io::ErrorKind::WouldBlock)))?;
|
||||
remote_io_admission::admit(&binding).map_err(DiskError::Io)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn admit_local_disk(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk: &DiskStore,
|
||||
@@ -483,6 +510,18 @@ pub async fn find_local_disk_by_ref(disk_ref: &str) -> Option<DiskStore> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Resolve and admit an authenticated RPC's target before its storage locks.
|
||||
/// Unknown UUID mappings defer background work instead of probing unrelated disks.
|
||||
pub async fn find_local_disk_by_ref_for_rpc(disk_ref: &str) -> DiskResult<Option<DiskStore>> {
|
||||
use crate::disk::remote_io_admission;
|
||||
|
||||
if !remote_io_admission::active() {
|
||||
return Ok(find_local_disk_by_ref(disk_ref).await);
|
||||
}
|
||||
let (disk, _) = local_disk_candidate(&crate::runtime::global::current_ctx(), disk_ref).await?;
|
||||
Ok(Some(disk))
|
||||
}
|
||||
|
||||
pub async fn all_local_disk_path() -> Vec<String> {
|
||||
runtime_sources::local_disk_paths().await
|
||||
}
|
||||
@@ -843,6 +882,92 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_background_target_lookup_defers_before_io_and_reuses_multi_phase_admission() {
|
||||
use crate::disk::{RemoteBackgroundIo, with_remote_background_io};
|
||||
use rustfs_concurrency::adaptive::{
|
||||
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
|
||||
};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("target directory");
|
||||
let id = Uuid::new_v4();
|
||||
let disk = target_disk(&ctx, root.path(), id).await;
|
||||
disk.make_volume("bucket").await.expect("fixture volume");
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 0);
|
||||
disk.physical_admission()
|
||||
.expect("local binding")
|
||||
.set_test_capacity(vec![gate.clone()])
|
||||
.expect("physical device");
|
||||
let node = NodeAdmission::new(NonZeroU16::MIN);
|
||||
node.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
let path = disk.endpoint().to_string();
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
assert!(
|
||||
matches!(disk.read_all("bucket", "object").await,
|
||||
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock),
|
||||
"no implicit target admission"
|
||||
);
|
||||
assert!(
|
||||
matches!(local_disk_candidate(&ctx, &path).await,
|
||||
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock),
|
||||
"closed receiver capacity"
|
||||
);
|
||||
gate.set_limit(1);
|
||||
assert!(
|
||||
matches!(local_disk_candidate(&ctx, &id.to_string()).await,
|
||||
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock),
|
||||
"cold UUID cannot probe unrelated disks"
|
||||
);
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
let (selected, _) = local_disk_candidate(&ctx, &path).await.expect("select known local path");
|
||||
assert!(Arc::ptr_eq(&selected, &disk));
|
||||
for body in [b"first phase".as_slice(), b"second phase".as_slice()] {
|
||||
disk.write_all("bucket", "object", bytes::Bytes::copy_from_slice(body))
|
||||
.await
|
||||
.expect("write phase");
|
||||
let (again, _) = local_disk_candidate(&ctx, &path)
|
||||
.await
|
||||
.expect("repeat target resolution at capacity one");
|
||||
assert_eq!(again.read_all("bucket", "object").await.expect("read phase").as_ref(), body);
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [1, 1]);
|
||||
}
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(1), Some(1)]);
|
||||
assert!(record_local_disk_id_if_active(&ctx, &disk, id).await);
|
||||
let replacement = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
replacement
|
||||
.physical_admission()
|
||||
.expect("replacement binding")
|
||||
.set_test_capacity(vec![gate.clone()])
|
||||
.expect("replacement device");
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
assert!(
|
||||
matches!(local_disk_candidate(&ctx, &id.to_string()).await,
|
||||
Err(DiskError::Io(error)) if error.kind() == std::io::ErrorKind::WouldBlock),
|
||||
"stale UUID must not resolve replacement data"
|
||||
);
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!(
|
||||
[node.completed(), gate.completed()],
|
||||
[Some(1), Some(1)],
|
||||
"identity checks alone are not work completion"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_cache_cannot_admit_a_different_format_at_the_same_path() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
|
||||
@@ -837,7 +837,29 @@ impl ECStore {
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
let suspended_pools = if opts.skip_decommissioned {
|
||||
self.get_latest_object_info_with_pool_policy(bucket, object, opts, false)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Observe every pool for a conditional-write preflight. This does not
|
||||
/// hold the namespace locks needed to prove a condition at publication.
|
||||
pub(super) async fn get_latest_object_info_for_conditional_write(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
self.get_latest_object_info_with_pool_policy(bucket, object, opts, true).await
|
||||
}
|
||||
|
||||
async fn get_latest_object_info_with_pool_policy(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
require_all_pools: bool,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
let suspended_pools = if opts.skip_decommissioned && !require_all_pools {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
Some(
|
||||
(0..self.pools.len())
|
||||
@@ -853,7 +875,7 @@ impl ECStore {
|
||||
continue;
|
||||
}
|
||||
|
||||
if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
|
||||
if opts.skip_rebalancing && !require_all_pools && self.is_pool_rebalancing(idx).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -873,6 +895,9 @@ impl ECStore {
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
if require_all_pools && !is_err_object_not_found(&e) && !is_err_version_not_found(&e) {
|
||||
return Err(e);
|
||||
}
|
||||
candidates.push(LatestObjectInfoCandidate {
|
||||
info: None,
|
||||
idx,
|
||||
|
||||
@@ -88,6 +88,12 @@ pub enum HealObjectDisposition {
|
||||
pub struct HealObjectReceipt {
|
||||
pub identity: HealObjectIdentity,
|
||||
pub disposition: HealObjectDisposition,
|
||||
/// Data directory selected from quorum metadata during storage execution.
|
||||
/// This does not establish lock ownership or identity at completion.
|
||||
pub quorum_selected_data_dir: Option<Uuid>,
|
||||
/// Storage completed its Deep check path for the selected metadata.
|
||||
/// This does not certify all bytes, final identity, or scanner coverage.
|
||||
pub deep_check_observed: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
|
||||
@@ -84,7 +84,7 @@ rustfs-heal-contracts = { workspace = true }
|
||||
rustfs-madmin = { workspace = true }
|
||||
rustfs-utils = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "io-util", "time", "macros", "fs", "rt-multi-thread"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
|
||||
tonic = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
@@ -100,6 +100,7 @@ crc-fast = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
rustfs-ecstore = { workspace = true, features = ["test-util"] }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
rustfs-test-utils = { workspace = true }
|
||||
|
||||
@@ -186,7 +186,7 @@ pub(crate) enum HealErrorClass {
|
||||
/// Offline disk, missing volume, unmet quorum, transport failure,
|
||||
/// slow-down or cancelled RPC: the same request can succeed later.
|
||||
Transient,
|
||||
/// Lock contention or a lock/RPC timeout (non-fatal `LockError`).
|
||||
/// Nonblocking admission/lock contention or a lock/RPC timeout.
|
||||
LockContention,
|
||||
/// A storage grace window; `retry_not_before` is the absolute UNIX
|
||||
/// deadline when storage reported one.
|
||||
@@ -279,8 +279,7 @@ pub(crate) fn classify_storage_error(error: &EcstoreError) -> HealErrorClass {
|
||||
}
|
||||
}
|
||||
|
||||
/// Classify a disk error. Nonblocking local CAS and replacement leases report
|
||||
/// lock contention as `WouldBlock`.
|
||||
/// Classify a disk error without losing the underlying I/O deferral kind.
|
||||
pub(crate) fn classify_disk_error(error: &DiskError) -> HealErrorClass {
|
||||
if error.is_dangling_delete_grace() {
|
||||
return HealErrorClass::DanglingGrace {
|
||||
@@ -291,7 +290,6 @@ pub(crate) fn classify_disk_error(error: &DiskError) -> HealErrorClass {
|
||||
DiskError::FileNotFound | DiskError::FileVersionNotFound => HealErrorClass::Absent,
|
||||
DiskError::NoHealRequired => HealErrorClass::NoHealRequired,
|
||||
DiskError::Timeout => HealErrorClass::LockContention,
|
||||
DiskError::Io(error) if error.kind() == std::io::ErrorKind::WouldBlock => HealErrorClass::LockContention,
|
||||
DiskError::DiskNotFound
|
||||
| DiskError::VolumeNotFound
|
||||
| DiskError::ErasureReadQuorum
|
||||
@@ -306,6 +304,14 @@ pub(crate) fn classify_disk_error(error: &DiskError) -> HealErrorClass {
|
||||
}
|
||||
|
||||
fn classify_io_error(error: &std::io::Error) -> HealErrorClass {
|
||||
// Native memory admission and nonblocking CAS keep this kind when their
|
||||
// errors cross disk, storage and direct I/O wrappers.
|
||||
if error.kind() == std::io::ErrorKind::WouldBlock {
|
||||
return HealErrorClass::LockContention;
|
||||
}
|
||||
if error.kind() == std::io::ErrorKind::OutOfMemory {
|
||||
return HealErrorClass::Transient;
|
||||
}
|
||||
if DiskError::io_error_is_dangling_delete_grace(error) {
|
||||
return HealErrorClass::DanglingGrace {
|
||||
retry_not_before: dangling_deadline(DiskError::io_error_dangling_delete_retry_after(error)),
|
||||
@@ -864,6 +870,29 @@ mod tests {
|
||||
assert!(Error::Disk(DiskError::VolumeNotFound).is_recoverable_object_heal());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resource_pressure_stays_retryable_across_error_wrappers() {
|
||||
for (kind, expected) in [
|
||||
(std::io::ErrorKind::WouldBlock, super::HealErrorClass::LockContention),
|
||||
(std::io::ErrorKind::OutOfMemory, super::HealErrorClass::Transient),
|
||||
] {
|
||||
let pressure = || DiskError::Io(std::io::Error::new(kind, "background buffer memory budget exhausted"));
|
||||
for error in [
|
||||
Error::Disk(pressure()),
|
||||
Error::Storage(EcstoreError::from(pressure())),
|
||||
Error::Io(EcstoreError::from(pressure()).into()),
|
||||
] {
|
||||
assert_eq!(super::classify_heal_error(&error), expected);
|
||||
assert!(error.is_recoverable_heal(), "resource pressure must preserve retry responsibility");
|
||||
assert!(!error.is_object_absent());
|
||||
}
|
||||
}
|
||||
for kind in [std::io::ErrorKind::Other, std::io::ErrorKind::PermissionDenied] {
|
||||
let error = Error::Io(std::io::Error::new(kind, "background buffer memory budget exhausted"));
|
||||
assert!(!error.is_recoverable_heal(), "display text must not override the typed error kind");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classify_heal_error_table() {
|
||||
use super::{HealErrorClass as C, classify_heal_error};
|
||||
|
||||
@@ -28,6 +28,7 @@ use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use serde::Serialize;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
@@ -60,6 +61,8 @@ pub struct HealChannelProcessor {
|
||||
response_sender: mpsc::UnboundedSender<HealChannelResponse>,
|
||||
/// Response receiver
|
||||
response_receiver: mpsc::UnboundedReceiver<HealChannelResponse>,
|
||||
#[cfg(test)]
|
||||
command_pause: Option<(oneshot::Sender<()>, oneshot::Receiver<()>)>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -161,6 +164,8 @@ impl HealChannelProcessor {
|
||||
heal_manager,
|
||||
response_sender: response_tx,
|
||||
response_receiver: response_rx,
|
||||
#[cfg(test)]
|
||||
command_pause: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -229,9 +234,19 @@ impl HealChannelProcessor {
|
||||
|
||||
/// Start processing legacy and canonical-receipt heal channel requests.
|
||||
pub async fn start_with_receipts(
|
||||
&mut self,
|
||||
receiver: HealChannelReceiver,
|
||||
receipt_receiver: HealReceiptReceiver,
|
||||
) -> Result<()> {
|
||||
self.start_with_shutdown(receiver, receipt_receiver, CancellationToken::new())
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn start_with_shutdown(
|
||||
&mut self,
|
||||
mut receiver: HealChannelReceiver,
|
||||
mut receipt_receiver: HealReceiptReceiver,
|
||||
stop: CancellationToken,
|
||||
) -> Result<()> {
|
||||
info!(
|
||||
target: "rustfs::heal::channel",
|
||||
@@ -244,10 +259,30 @@ impl HealChannelProcessor {
|
||||
|
||||
let mut receipt_channel_open = true;
|
||||
loop {
|
||||
if stop.is_cancelled() {
|
||||
receiver.close();
|
||||
receipt_receiver.close();
|
||||
while let Some(command) = receiver.recv().await {
|
||||
Self::reject_stopped_command(command);
|
||||
}
|
||||
while let Some(HealReceiptCommand { response_tx, .. }) = receipt_receiver.recv().await {
|
||||
let _ = response_tx.send(Err(Error::other("Heal channel is stopping").to_string()));
|
||||
}
|
||||
self.response_receiver.close();
|
||||
while self.response_receiver.try_recv().is_ok() {}
|
||||
break;
|
||||
}
|
||||
tokio::select! {
|
||||
_ = stop.cancelled() => {}
|
||||
command = receiver.recv() => {
|
||||
match command {
|
||||
Some(command) => {
|
||||
if stop.is_cancelled() {
|
||||
Self::reject_stopped_command(command);
|
||||
continue;
|
||||
}
|
||||
#[cfg(test)]
|
||||
self.pause_selected_command().await;
|
||||
if let Err(e) = self.process_command(command).await {
|
||||
error!(
|
||||
target: "rustfs::heal::channel",
|
||||
@@ -278,6 +313,12 @@ impl HealChannelProcessor {
|
||||
receipt_channel_open = false;
|
||||
continue;
|
||||
};
|
||||
if stop.is_cancelled() {
|
||||
let _ = response_tx.send(Err(Error::other("Heal channel is stopping").to_string()));
|
||||
continue;
|
||||
}
|
||||
#[cfg(test)]
|
||||
self.pause_selected_command().await;
|
||||
if let Err(e) = self.process_start_request(request, false, true, response_tx).await {
|
||||
error!(
|
||||
target: "rustfs::heal::channel",
|
||||
@@ -319,6 +360,26 @@ impl HealChannelProcessor {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reject_stopped_command(command: HealChannelCommand) {
|
||||
let error = Error::other("Heal channel is stopping").to_string();
|
||||
match command {
|
||||
HealChannelCommand::Start { response_tx, .. } => {
|
||||
let _ = response_tx.send(Err(error));
|
||||
}
|
||||
HealChannelCommand::Query { response_tx, .. } | HealChannelCommand::Cancel { response_tx, .. } => {
|
||||
let _ = response_tx.send(Err(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_selected_command(&mut self) {
|
||||
if let Some((started, release)) = self.command_pause.take() {
|
||||
let _ = started.send(());
|
||||
let _ = release.await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Process heal command
|
||||
async fn process_command(&self, command: HealChannelCommand) -> Result<()> {
|
||||
match command {
|
||||
@@ -1000,6 +1061,8 @@ mod tests {
|
||||
HealObjectDisposition::Failed(HealFailureClass::RetryExhausted)
|
||||
};
|
||||
outcome.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".into(),
|
||||
@@ -1084,6 +1147,8 @@ mod tests {
|
||||
outcome.start();
|
||||
for index in 0..256 {
|
||||
outcome.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".into(),
|
||||
@@ -2469,4 +2534,126 @@ mod tests {
|
||||
assert_eq!(response.request_id, "missing-token");
|
||||
assert!(response.error.unwrap_or_default().contains("Heal task not found"));
|
||||
}
|
||||
async fn stop_after_selected_start(canonical: bool) {
|
||||
let manager = create_test_heal_manager();
|
||||
let mut processor = HealChannelProcessor::new(manager.clone());
|
||||
let (selected_tx, selected_rx) = oneshot::channel();
|
||||
let (release_tx, release_rx) = oneshot::channel();
|
||||
processor.command_pause = Some((selected_tx, release_rx));
|
||||
let (command_tx, command_rx) = mpsc::unbounded_channel();
|
||||
let (receipt_tx, receipt_rx) = mpsc::unbounded_channel();
|
||||
let stop = CancellationToken::new();
|
||||
let worker_stop = stop.clone();
|
||||
let worker = tokio::spawn(async move { processor.start_with_shutdown(command_rx, receipt_rx, worker_stop).await });
|
||||
let request = HealChannelRequest {
|
||||
id: "active-shutdown-command".to_string(),
|
||||
bucket: "bucket".to_string(),
|
||||
object_prefix: Some("active".to_string()),
|
||||
source: HealRequestSource::Admin,
|
||||
..Default::default()
|
||||
};
|
||||
let mut legacy_response = None;
|
||||
let mut canonical_response = None;
|
||||
if canonical {
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
receipt_tx.send(HealReceiptCommand { request, response_tx }).unwrap();
|
||||
canonical_response = Some(response_rx);
|
||||
} else {
|
||||
let (response_tx, response_rx) = oneshot::channel();
|
||||
command_tx.send(HealChannelCommand::Start { request, response_tx }).unwrap();
|
||||
legacy_response = Some(response_rx);
|
||||
}
|
||||
tokio::time::timeout(Duration::from_secs(1), selected_rx)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let pending = HealChannelRequest {
|
||||
id: "queued-shutdown-command".to_string(),
|
||||
bucket: "bucket".to_string(),
|
||||
object_prefix: Some("pending".to_string()),
|
||||
source: HealRequestSource::Admin,
|
||||
..Default::default()
|
||||
};
|
||||
let (start_tx, start_rx) = oneshot::channel();
|
||||
command_tx
|
||||
.send(HealChannelCommand::Start {
|
||||
request: pending.clone(),
|
||||
response_tx: start_tx,
|
||||
})
|
||||
.unwrap();
|
||||
let (query_tx, query_rx) = oneshot::channel();
|
||||
command_tx
|
||||
.send(HealChannelCommand::Query {
|
||||
heal_path: "bucket/active".to_string(),
|
||||
client_token: "active-shutdown-command".to_string(),
|
||||
since_seq: None,
|
||||
response_tx: query_tx,
|
||||
})
|
||||
.unwrap();
|
||||
let (cancel_tx, cancel_rx) = oneshot::channel();
|
||||
command_tx
|
||||
.send(HealChannelCommand::Cancel {
|
||||
heal_path: "bucket/active".to_string(),
|
||||
client_token: "active-shutdown-command".to_string(),
|
||||
response_tx: cancel_tx,
|
||||
})
|
||||
.unwrap();
|
||||
let (receipt_response_tx, receipt_response_rx) = oneshot::channel();
|
||||
receipt_tx
|
||||
.send(HealReceiptCommand {
|
||||
request: pending,
|
||||
response_tx: receipt_response_tx,
|
||||
})
|
||||
.unwrap();
|
||||
stop.cancel();
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!worker.is_finished(), "stop cannot cancel the selected command");
|
||||
release_tx.send(()).unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(1), worker)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
if let Some(response) = legacy_response {
|
||||
assert_eq!(response.await.unwrap().unwrap(), HealAdmissionResult::Accepted);
|
||||
}
|
||||
if let Some(response) = canonical_response {
|
||||
let receipt = response.await.unwrap().unwrap();
|
||||
assert_eq!(receipt.result, HealAdmissionResult::Accepted);
|
||||
assert_eq!(receipt.task_id, "active-shutdown-command");
|
||||
}
|
||||
let expected = Error::other("Heal channel is stopping").to_string();
|
||||
assert_eq!(start_rx.await.unwrap().unwrap_err(), expected);
|
||||
assert_eq!(query_rx.await.unwrap().unwrap_err(), expected);
|
||||
assert_eq!(cancel_rx.await.unwrap().unwrap_err(), expected);
|
||||
assert_eq!(receipt_response_rx.await.unwrap().unwrap_err(), expected);
|
||||
assert!(command_tx.is_closed());
|
||||
assert!(receipt_tx.is_closed());
|
||||
assert_eq!(
|
||||
manager
|
||||
.get_task_status_for_path("bucket/active", "active-shutdown-command")
|
||||
.await
|
||||
.unwrap(),
|
||||
HealTaskStatus::Pending,
|
||||
"queued Cancel must not execute"
|
||||
);
|
||||
assert!(
|
||||
manager
|
||||
.get_task_status_for_path("bucket/pending", "queued-shutdown-command")
|
||||
.await
|
||||
.is_err(),
|
||||
"queued starts must not be admitted"
|
||||
);
|
||||
manager.stop().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_channel_stop_finishes_legacy_command_and_rejects_all_queued_variants() {
|
||||
stop_after_selected_start(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_channel_stop_finishes_receipt_command_and_rejects_all_queued_variants() {
|
||||
stop_after_selected_start(true).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -127,6 +127,7 @@ pub struct ErasureSetHealer {
|
||||
replacement_task_id: Option<String>,
|
||||
replacement_target_identities: Option<Arc<[ReplacementTargetIdentity]>>,
|
||||
replacement_execution: Option<Arc<super::storage::ReplacementExecution>>,
|
||||
work_parent: Option<super::manager::HealWork>,
|
||||
mainline_pacer: Option<Arc<super::pacing::MainlinePacer>>,
|
||||
}
|
||||
|
||||
@@ -367,6 +368,7 @@ impl ErasureSetHealer {
|
||||
replacement_task_id: None,
|
||||
replacement_target_identities: None,
|
||||
replacement_execution: None,
|
||||
work_parent: None,
|
||||
mainline_pacer: None,
|
||||
}
|
||||
}
|
||||
@@ -381,6 +383,11 @@ impl ErasureSetHealer {
|
||||
self
|
||||
}
|
||||
|
||||
pub(super) fn with_work_parent(mut self, parent: Option<super::manager::HealWork>) -> Self {
|
||||
self.work_parent = parent;
|
||||
self
|
||||
}
|
||||
|
||||
pub(crate) fn with_replacement_execution(mut self, execution: Option<Arc<super::storage::ReplacementExecution>>) -> Self {
|
||||
self.replacement_execution = execution;
|
||||
self
|
||||
@@ -1349,6 +1356,8 @@ impl ErasureSetHealer {
|
||||
};
|
||||
(
|
||||
HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Metadata,
|
||||
bucket: RUSTFS_META_BUCKET.to_string(),
|
||||
@@ -1720,6 +1729,8 @@ impl ErasureSetHealer {
|
||||
let admin_identity = self.admin_identity(bucket, &object_name, version_id.as_deref(), incarnation);
|
||||
let admin_task_id = self.admin_task.as_ref().map(|task| task.id.clone());
|
||||
let execution = self.replacement_execution.clone();
|
||||
let replacement_object = execution.is_some();
|
||||
let work_parent = execution.as_ref().and(self.work_parent.as_ref()).cloned();
|
||||
let failure_identity = execution
|
||||
.as_ref()
|
||||
.map(|_| (dedup_key.clone(), object_name.clone(), version_id.clone()));
|
||||
@@ -1751,10 +1762,22 @@ impl ErasureSetHealer {
|
||||
let result = if cancel_token.is_cancelled() {
|
||||
(0, Err(Error::TaskCancelled))
|
||||
} else {
|
||||
match storage
|
||||
.heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts)
|
||||
.await
|
||||
{
|
||||
let result = if replacement_object {
|
||||
storage
|
||||
.heal_object_with_task_cancel(
|
||||
&bucket_name,
|
||||
&object_name,
|
||||
version_id.as_deref(),
|
||||
&heal_opts,
|
||||
&cancel_token,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
storage
|
||||
.heal_object(&bucket_name, &object_name, version_id.as_deref(), &heal_opts)
|
||||
.await
|
||||
};
|
||||
match result {
|
||||
Ok((result, None)) if target_outcomes_complete(&result, &target_endpoints) => {
|
||||
let object_size = result_object_size_u64(&result);
|
||||
if !replacement_commit_evidence_required {
|
||||
@@ -1846,7 +1869,11 @@ impl ErasureSetHealer {
|
||||
};
|
||||
page_tasks.push(async move {
|
||||
if let (Some(execution), Some((key, object, version))) = (execution, failure_identity) {
|
||||
match execution.run(work).await {
|
||||
let result = match work_parent {
|
||||
Some(parent) => execution.run_owned(&parent, work).await,
|
||||
None => execution.run(work).await,
|
||||
};
|
||||
match result {
|
||||
Ok(result) => result,
|
||||
Err(error) => (key, object, version, (0, Err(error)), None),
|
||||
}
|
||||
@@ -2758,6 +2785,8 @@ mod resume_loop_tests {
|
||||
.unwrap_or(Some(HealObjectDisposition::Repaired));
|
||||
let receipt = if error.is_none() {
|
||||
disposition.map(|disposition| HealObjectReceipt {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: bucket.to_owned(),
|
||||
|
||||
@@ -145,9 +145,11 @@ pub(super) fn skipped(identity: Option<HealObjectIdentity>, reason: &str) -> Opt
|
||||
identity.map(|identity| {
|
||||
(
|
||||
CanonicalObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
identity,
|
||||
disposition: HealObjectDisposition::Unknown,
|
||||
detail: Some(reason.to_owned()),
|
||||
quorum_selected_data_dir: None,
|
||||
},
|
||||
0,
|
||||
)
|
||||
@@ -249,9 +251,11 @@ pub(super) async fn heal_object(
|
||||
(size, legacy),
|
||||
Some((
|
||||
CanonicalObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
identity,
|
||||
disposition,
|
||||
detail,
|
||||
quorum_selected_data_dir: None,
|
||||
},
|
||||
failures,
|
||||
)),
|
||||
|
||||
@@ -420,6 +420,7 @@ async fn real_storage_grace_round_trip() {
|
||||
&HealOpts {
|
||||
scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
|
||||
recreate: true,
|
||||
remove: true,
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
|
||||
+291
-38
@@ -13,16 +13,20 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::heal::{
|
||||
outcome::HealTaskOutcome,
|
||||
outcome::{HealExecutionOutcome, HealTaskOutcome},
|
||||
progress::{HealProgress, HealStatistics},
|
||||
resume::{
|
||||
ReplacementPhase, ReplacementRecoveryBlock, ReplacementRecoveryCandidate, ResumeGc, ResumeManager, ResumeState,
|
||||
ResumeUtils, merge_replacement_recovery_candidate, replacement_retry_is_exhausted_active,
|
||||
},
|
||||
storage::HealStorageAPI,
|
||||
storage::{HealStorageAPI, run_local_metadata},
|
||||
task::{HealOptions, HealPriority, HealRequest, HealTask, HealTaskStatus, HealType, demote_to_debug_when},
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use futures::{
|
||||
FutureExt,
|
||||
future::{BoxFuture, Shared},
|
||||
};
|
||||
use metrics::{counter, gauge, histogram};
|
||||
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
|
||||
use rustfs_concurrency::workload::{ForegroundPressure, foreground_pressure};
|
||||
@@ -32,8 +36,10 @@ use rustfs_heal_contracts::heal_channel::{
|
||||
HealAdmissionDropReason, HealAdmissionReceipt, HealAdmissionResult, HealRequestSource,
|
||||
};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use std::future::Future;
|
||||
#[cfg(test)]
|
||||
use std::sync::LazyLock;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::{
|
||||
collections::{BinaryHeap, HashMap, HashSet},
|
||||
sync::{Arc, Mutex as StdMutex, MutexGuard as StdMutexGuard},
|
||||
@@ -44,11 +50,206 @@ use tokio::{
|
||||
time::{interval, sleep},
|
||||
};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tokio_util::task::{TaskTracker, task_tracker::TaskTrackerToken};
|
||||
use tracing::{debug, error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::{DiskError, Endpoint, HealDiskExt as _, local_disk_map_read};
|
||||
|
||||
// Four long-lived loops have fixed receipts; finished dynamic jobs leave the
|
||||
// tracker immediately. No manager/cache lock is needed by a drain waiter.
|
||||
type HealWorkerJoin = Shared<BoxFuture<'static, std::result::Result<(), Arc<tokio::task::JoinError>>>>;
|
||||
|
||||
struct HealWorkerRegistration {
|
||||
open: bool,
|
||||
stopping: usize,
|
||||
fixed: [Option<HealWorkerJoin>; 4],
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct HealWorkers {
|
||||
registration: Arc<StdMutex<HealWorkerRegistration>>,
|
||||
tasks: TaskTracker,
|
||||
startup: TaskTracker,
|
||||
failed: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct HealWork {
|
||||
owner: HealWorkers,
|
||||
_token: TaskTrackerToken,
|
||||
}
|
||||
|
||||
// Each in-flight stop owns its own intent. Dropping one waiter cannot reopen
|
||||
// startup while another stop still owns the handoff boundary.
|
||||
struct HealStopIntent(HealWorkers);
|
||||
|
||||
impl Drop for HealStopIntent {
|
||||
fn drop(&mut self) {
|
||||
if let Ok(mut registration) = self.0.registration.lock() {
|
||||
registration.stopping -= 1;
|
||||
if registration.stopping == 0 && registration.open {
|
||||
self.0.startup.reopen();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct HealWorkGuard {
|
||||
work: HealWork,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
impl Drop for HealWorkGuard {
|
||||
fn drop(&mut self) {
|
||||
if !self.completed {
|
||||
self.work.owner.failed.store(true, Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl HealWork {
|
||||
fn spawn<F, Fut, T>(&self, work: F) -> tokio::task::JoinHandle<T>
|
||||
where
|
||||
F: FnOnce(HealWork) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
// Clone before spawn, while the live parent prevents an empty tracker.
|
||||
let guard = HealWorkGuard {
|
||||
work: self.clone(),
|
||||
completed: false,
|
||||
};
|
||||
tokio::spawn(async move {
|
||||
let mut guard = guard;
|
||||
let output = work(guard.work.clone()).await;
|
||||
guard.completed = true;
|
||||
output
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl HealWorkers {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
registration: Arc::new(StdMutex::new(HealWorkerRegistration {
|
||||
open: true,
|
||||
stopping: 0,
|
||||
fixed: std::array::from_fn(|_| None),
|
||||
})),
|
||||
tasks: TaskTracker::new(),
|
||||
startup: TaskTracker::new(),
|
||||
failed: Arc::new(AtomicBool::new(false)),
|
||||
}
|
||||
}
|
||||
|
||||
fn enter_start(&self) -> Result<(HealWorkGuard, TaskTrackerToken)> {
|
||||
let registration = self
|
||||
.registration
|
||||
.lock()
|
||||
.map_err(|_| Error::other("heal worker registration poisoned"))?;
|
||||
if !registration.open || registration.stopping != 0 {
|
||||
return Err(Error::other("heal manager workers are stopping or terminally closed"));
|
||||
}
|
||||
Ok((
|
||||
HealWorkGuard {
|
||||
work: HealWork {
|
||||
owner: self.clone(),
|
||||
_token: self.tasks.token(),
|
||||
},
|
||||
completed: false,
|
||||
},
|
||||
self.startup.token(),
|
||||
))
|
||||
}
|
||||
|
||||
fn begin_stop(&self) -> Result<HealStopIntent> {
|
||||
let mut registration = self
|
||||
.registration
|
||||
.lock()
|
||||
.map_err(|_| Error::other("heal worker registration poisoned"))?;
|
||||
registration.stopping += 1;
|
||||
self.startup.close();
|
||||
Ok(HealStopIntent(self.clone()))
|
||||
}
|
||||
|
||||
fn spawn_fixed<F, Fut>(&self, slot: usize, work: F) -> Result<()>
|
||||
where
|
||||
F: FnOnce(HealWork) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = ()> + Send + 'static,
|
||||
{
|
||||
let mut registration = self
|
||||
.registration
|
||||
.lock()
|
||||
.map_err(|_| Error::other("heal worker registration poisoned"))?;
|
||||
if !registration.open || registration.fixed[slot].is_some() {
|
||||
return Err(Error::other("heal manager loop is closed or already registered"));
|
||||
}
|
||||
let parent = HealWork {
|
||||
owner: self.clone(),
|
||||
_token: self.tasks.token(),
|
||||
};
|
||||
let task = parent.spawn(work);
|
||||
registration.fixed[slot] = Some(async move { task.await.map_err(Arc::new) }.boxed().shared());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn close(&self) -> Result<()> {
|
||||
let mut registration = self
|
||||
.registration
|
||||
.lock()
|
||||
.map_err(|_| Error::other("heal worker registration poisoned"))?;
|
||||
registration.open = false;
|
||||
self.tasks.close();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait(&self) -> Result<()> {
|
||||
let fixed = {
|
||||
let registration = self
|
||||
.registration
|
||||
.lock()
|
||||
.map_err(|_| Error::other("heal worker registration poisoned"))?;
|
||||
if registration.open {
|
||||
return Err(Error::other("heal worker admission is not closed"));
|
||||
}
|
||||
registration.fixed.clone()
|
||||
};
|
||||
let mut join_failed = false;
|
||||
for task in fixed.into_iter().flatten() {
|
||||
join_failed |= task.await.is_err();
|
||||
}
|
||||
self.tasks.wait().await;
|
||||
if join_failed || self.failed.load(Ordering::Acquire) {
|
||||
return Err(Error::other("heal owned worker panicked or was unexpectedly cancelled"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn spawn_heal_work<F, Fut, T>(parent: Option<&HealWork>, work: F) -> tokio::task::JoinHandle<T>
|
||||
where
|
||||
F: FnOnce(Option<HealWork>) -> Fut + Send + 'static,
|
||||
Fut: Future<Output = T> + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
match parent {
|
||||
Some(parent) => parent.spawn(move |child| work(Some(child))),
|
||||
None => tokio::spawn(work(None)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn inspect_resume_disk(
|
||||
gc: &mut ResumeGc,
|
||||
disk: &super::DiskStore,
|
||||
admission: Option<&Arc<rustfs_concurrency::adaptive::NodeAdmission>>,
|
||||
cancellation: &CancellationToken,
|
||||
) -> Result<()> {
|
||||
run_local_metadata(disk, admission, cancellation, gc.inspect_disk(disk))
|
||||
.await?
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
const KEEP_HEAL_TASK_STATUS_DURATION: Duration = Duration::from_secs(10 * 60);
|
||||
// Each cache includes alias tokens in its count and byte budget. Eviction
|
||||
// removes every token sharing a snapshot; neither cache retains repair state.
|
||||
@@ -326,9 +527,18 @@ async fn publish_completed_heal(
|
||||
let object = outcome
|
||||
.filter(|outcome| completed.heal_type.is_per_object() && !outcome.objects_truncated && outcome.objects.len() == 1)
|
||||
.and_then(|outcome| outcome.objects.front())
|
||||
.map(|object| crate::heal::outcome::HealObjectReceipt {
|
||||
identity: object.identity.clone(),
|
||||
disposition: object.disposition.clone(),
|
||||
.map(|object| {
|
||||
let quorum_selected_data_dir = object.quorum_selected_data_dir.filter(|generation| !generation.is_nil());
|
||||
crate::heal::outcome::HealObjectReceipt {
|
||||
identity: object.identity.clone(),
|
||||
disposition: object.disposition.clone(),
|
||||
quorum_selected_data_dir,
|
||||
deep_check_observed: object.deep_check_observed
|
||||
&& quorum_selected_data_dir.is_some()
|
||||
&& !completed.options.dry_run
|
||||
&& matches!(completed.status, HealTaskStatus::Completed)
|
||||
&& outcome.is_some_and(|outcome| outcome.execution == HealExecutionOutcome::Completed),
|
||||
}
|
||||
});
|
||||
rustfs_heal_contracts::completion::publish_heal_completion(rustfs_heal_contracts::completion::HealCompletion {
|
||||
task_id: task_id.to_owned(),
|
||||
@@ -1014,6 +1224,7 @@ pub struct HealState {
|
||||
|
||||
/// Heal manager
|
||||
pub struct HealManager {
|
||||
workers: HealWorkers,
|
||||
/// Heal config
|
||||
config: Arc<RwLock<HealConfig>>,
|
||||
/// Heal state
|
||||
@@ -1188,15 +1399,22 @@ impl HealManager {
|
||||
)
|
||||
}
|
||||
|
||||
fn schedule_mainline_throttle_recheck(notify: Arc<Notify>, delay: Duration) {
|
||||
fn schedule_mainline_throttle_recheck(
|
||||
notify: Arc<Notify>,
|
||||
delay: Duration,
|
||||
parent: Option<&HealWork>,
|
||||
cancel: CancellationToken,
|
||||
) {
|
||||
if delay.is_zero() {
|
||||
notify.notify_one();
|
||||
return;
|
||||
}
|
||||
|
||||
tokio::spawn(async move {
|
||||
sleep(delay).await;
|
||||
notify.notify_one();
|
||||
spawn_heal_work(parent, move |_parent| async move {
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => {},
|
||||
_ = sleep(delay) => notify.notify_one(),
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1612,9 +1830,10 @@ impl HealManager {
|
||||
/// Start the bounded resume-state inspector. Destructive GC remains
|
||||
/// disabled until the durable owner/CAS contract from backlog#1927 is
|
||||
/// available; this task therefore cannot remove an active or stale file.
|
||||
async fn start_resume_gc(&self) {
|
||||
async fn start_resume_gc(&self) -> Result<()> {
|
||||
let cancel = self.cancel_token.clone();
|
||||
tokio::spawn(async move {
|
||||
let admission = self.storage.background_node_admission();
|
||||
self.workers.spawn_fixed(2, move |_parent| async move {
|
||||
let mut gc_by_disk = HashMap::<String, ResumeGc>::new();
|
||||
let mut ticker = interval(RESUME_GC_INTERVAL);
|
||||
loop {
|
||||
@@ -1630,7 +1849,7 @@ impl HealManager {
|
||||
let gc = gc_by_disk.entry(disk_key).or_default();
|
||||
tokio::select! {
|
||||
_ = cancel.cancelled() => return,
|
||||
result = gc.inspect_disk(&disk) => {
|
||||
result = inspect_resume_disk(gc, &disk, admission.as_ref(), &cancel) => {
|
||||
if let Err(error) = result {
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
@@ -1649,13 +1868,14 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn start_root_recovery_terminal_gc(&self) {
|
||||
async fn start_root_recovery_terminal_gc(&self) -> Result<()> {
|
||||
let cancel = self.cancel_token.clone();
|
||||
let root_recovery = self.root_recovery.clone();
|
||||
tokio::spawn(async move {
|
||||
self.workers.spawn_fixed(3, move |_parent| async move {
|
||||
let mut ticker = interval(RESUME_GC_INTERVAL);
|
||||
loop {
|
||||
tokio::select! {
|
||||
@@ -1694,7 +1914,8 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create new HealManager
|
||||
@@ -1719,6 +1940,7 @@ impl HealManager {
|
||||
) -> Self {
|
||||
let config = config.unwrap_or_default();
|
||||
Self {
|
||||
workers: HealWorkers::new(),
|
||||
config: Arc::new(RwLock::new(config)),
|
||||
state: Arc::new(RwLock::new(HealState::default())),
|
||||
active_heals: Arc::new(Mutex::new(HashMap::new())),
|
||||
@@ -1765,8 +1987,10 @@ impl HealManager {
|
||||
|
||||
/// Start HealManager
|
||||
pub async fn start(&self) -> Result<()> {
|
||||
let (mut starting, _startup_token) = self.workers.enter_start()?;
|
||||
let mut state = self.state.write().await;
|
||||
if state.is_running {
|
||||
starting.completed = true;
|
||||
warn!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_MANAGER_STATE,
|
||||
@@ -1819,8 +2043,8 @@ impl HealManager {
|
||||
self.process_unclean_shutdown().await;
|
||||
|
||||
// Inspect resume artifacts in a bounded, fail-closed background task.
|
||||
self.start_resume_gc().await;
|
||||
self.start_root_recovery_terminal_gc().await;
|
||||
self.start_resume_gc().await?;
|
||||
self.start_root_recovery_terminal_gc().await?;
|
||||
|
||||
// start auto disk scanner to heal unformatted disks
|
||||
if self.config.read().await.enable_auto_heal {
|
||||
@@ -1844,11 +2068,31 @@ impl HealManager {
|
||||
state = "running",
|
||||
"Heal manager started"
|
||||
);
|
||||
starting.completed = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn test_work_parent(&self) -> HealWork {
|
||||
assert!(self.workers.registration.lock().unwrap().open, "test root parent precedes stop");
|
||||
HealWork {
|
||||
owner: self.workers.clone(),
|
||||
_token: self.workers.tasks.token(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Durable stop handoff completed; owned workers may still be joining.
|
||||
pub(crate) fn stop_handoff_completed(&self) -> bool {
|
||||
self.cancel_token.is_cancelled()
|
||||
}
|
||||
|
||||
/// Stop HealManager
|
||||
pub async fn stop(&self) -> Result<()> {
|
||||
let _stop_intent = self.workers.begin_stop()?;
|
||||
// TaskTracker registers its notification before checking closed/empty.
|
||||
// Stop admission first; wait outside admin/active/cache locks so startup
|
||||
// can finish its existing recovery and register all four loop receipts.
|
||||
self.workers.startup.wait().await;
|
||||
let _admin_start_guard = self.admin_start_shutdown.lock().await;
|
||||
info!(
|
||||
target: "rustfs::heal::manager",
|
||||
@@ -1865,29 +2109,32 @@ impl HealManager {
|
||||
let mut active_heals = self.active_heals.lock().await;
|
||||
let queue = self.heal_queue.lock().await;
|
||||
let retrying = self.retrying_heals.lock().await;
|
||||
for task in active_heals.values() {
|
||||
if root_recovery::is_admin_heal_recovery(&task.heal_type, task.source) {
|
||||
if task.get_status().await == HealTaskStatus::Completed {
|
||||
self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?;
|
||||
} else {
|
||||
let mut request = match task.retry_request_with_remaining_timeout().await {
|
||||
Ok(request) => request,
|
||||
Err(Error::TaskTimeout) => {
|
||||
let mut request = task.retry_request();
|
||||
request.options.timeout = Some(Duration::ZERO);
|
||||
request
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
request.retry_attempts = task.retry_attempts;
|
||||
self.root_recovery.persist(&request).await?;
|
||||
if !self.cancel_token.is_cancelled() {
|
||||
for task in active_heals.values() {
|
||||
if root_recovery::is_admin_heal_recovery(&task.heal_type, task.source) {
|
||||
if task.get_status().await == HealTaskStatus::Completed {
|
||||
self.root_recovery.remove(&task.id, &task.heal_type, task.source).await?;
|
||||
} else {
|
||||
let mut request = match task.retry_request_with_remaining_timeout().await {
|
||||
Ok(request) => request,
|
||||
Err(Error::TaskTimeout) => {
|
||||
let mut request = task.retry_request();
|
||||
request.options.timeout = Some(Duration::ZERO);
|
||||
request
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
request.retry_attempts = task.retry_attempts;
|
||||
self.root_recovery.persist(&request).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
for request in queue.requests().chain(retrying.values().map(|retrying| &retrying.request)) {
|
||||
self.root_recovery.persist(request).await?;
|
||||
}
|
||||
self.workers.close()?;
|
||||
self.cancel_token.cancel();
|
||||
}
|
||||
for request in queue.requests().chain(retrying.values().map(|retrying| &retrying.request)) {
|
||||
self.root_recovery.persist(request).await?;
|
||||
}
|
||||
self.cancel_token.cancel();
|
||||
drop(retrying);
|
||||
drop(queue);
|
||||
|
||||
@@ -1931,6 +2178,12 @@ impl HealManager {
|
||||
}
|
||||
crate::set_heal_queue_length(0);
|
||||
|
||||
// Completion paths reacquire active/cache locks. Never wait while
|
||||
// holding admin, active, queue, retry, cache or state guards.
|
||||
drop(active_heals);
|
||||
drop(_admin_start_guard);
|
||||
self.workers.wait().await?;
|
||||
|
||||
// update state
|
||||
let mut state = self.state.write().await;
|
||||
state.is_running = false;
|
||||
|
||||
@@ -36,6 +36,7 @@ impl HealManager {
|
||||
let displaced_terminals = self.displaced_terminals.clone();
|
||||
let mrf_repair_notice_targets = self.mrf_repair_notice_targets.clone();
|
||||
let storage = self.storage.clone();
|
||||
let admission = storage.background_node_admission();
|
||||
let replacement_recovery_anchors = self.replacement_recovery_anchors.clone();
|
||||
let blocks = self.replacement_recovery_blocks.clone();
|
||||
let cancel_token = self.cancel_token.clone();
|
||||
@@ -57,7 +58,7 @@ impl HealManager {
|
||||
"Heal auto disk scanner started"
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
self.workers.spawn_fixed(1, move |_parent| async move {
|
||||
let mut interval = interval(duration);
|
||||
|
||||
loop {
|
||||
@@ -88,6 +89,10 @@ impl HealManager {
|
||||
let mut conflicted_recovery_tasks = HashSet::<String>::new();
|
||||
let mut conflicted_recovery_sets = HashSet::<String>::new();
|
||||
let mut deferred_recovery_sets = HashSet::<String>::new();
|
||||
// This tick lacks local evidence for these sets/endpoints;
|
||||
// later inventory must not turn the deferral into absence.
|
||||
let mut metadata_deferred_sets = HashSet::<String>::new();
|
||||
let mut metadata_deferred_endpoints = HashSet::<String>::new();
|
||||
let mut deferred_replacement_endpoints = HashSet::<String>::new();
|
||||
let local_disks = {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
@@ -133,8 +138,32 @@ impl HealManager {
|
||||
continue;
|
||||
}
|
||||
|
||||
// detect unformatted disk via get_disk_id()
|
||||
match disk.get_disk_id().await {
|
||||
// Preserve backend format results separately from admission deferral.
|
||||
let probe = run_local_metadata(
|
||||
disk, admission.as_ref(), &cancel_token, disk.get_disk_id(),
|
||||
).await;
|
||||
// Native capability projection can defer after the outer plan was acquired.
|
||||
let resource_deferred = admission.is_some()
|
||||
&& matches!(&probe, Ok(Err(DiskError::Io(error))) if error.kind() == std::io::ErrorKind::WouldBlock);
|
||||
let disk_id = match probe {
|
||||
Ok(result) if !resource_deferred => result,
|
||||
_ => {
|
||||
if cancel_token.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
metadata_deferred_endpoints.insert(endpoint.to_string());
|
||||
if let Some(set) = &set_disk_id {
|
||||
metadata_deferred_sets.insert(set.clone());
|
||||
deferred_recovery_sets.insert(set.clone());
|
||||
}
|
||||
skipped_invalid_count += 1;
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if admission.is_some() && cancel_token.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
match disk_id {
|
||||
Err(DiskError::UnformattedDisk) => {
|
||||
if !super::super::replacement_readiness::auto_replacement_target_ready(disk, &local_disks)
|
||||
.await
|
||||
@@ -203,8 +232,18 @@ impl HealManager {
|
||||
let mut recovery_disks = BTreeMap::new();
|
||||
extend_replacement_recovery_disks(&mut recovery_disks, local_disks.iter().cloned());
|
||||
for set_disk_id in &recovery_sets {
|
||||
match storage.replacement_intent_disks(set_disk_id).await {
|
||||
if metadata_deferred_sets.contains(set_disk_id) {
|
||||
continue;
|
||||
}
|
||||
match storage.replacement_intent_disks(set_disk_id, Some(&cancel_token)).await {
|
||||
Ok(disks) => extend_replacement_recovery_disks(&mut recovery_disks, disks),
|
||||
Err(Error::TransientSkip { .. }) => {
|
||||
if cancel_token.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
metadata_deferred_sets.insert(set_disk_id.clone());
|
||||
deferred_recovery_sets.insert(set_disk_id.clone());
|
||||
}
|
||||
Err(error) => {
|
||||
conflicted_recovery_sets.insert(set_disk_id.clone());
|
||||
warn!(
|
||||
@@ -230,6 +269,11 @@ impl HealManager {
|
||||
let endpoint = disk.endpoint();
|
||||
let disk_set_disk_id =
|
||||
crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx);
|
||||
if metadata_deferred_endpoints.contains(anchor)
|
||||
|| disk_set_disk_id.as_ref().is_some_and(|set| metadata_deferred_sets.contains(set))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let replacement_task_ids = match ResumeUtils::get_replacement_intent_tasks(disk).await {
|
||||
Ok(task_ids) => task_ids,
|
||||
Err(error) => {
|
||||
@@ -795,7 +839,7 @@ impl HealManager {
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,7 +74,7 @@ impl HealManager {
|
||||
let workload_provider = self.workload_provider.clone();
|
||||
let disposition_cache = self.disposition_cache.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
self.workers.spawn_fixed(0, move |parent| async move {
|
||||
let mut interval = interval(config.read().await.heal_interval);
|
||||
|
||||
loop {
|
||||
@@ -92,7 +92,7 @@ impl HealManager {
|
||||
break;
|
||||
}
|
||||
_ = notify.notified(), if event_driven_scheduler_enable => {
|
||||
Self::process_heal_queue(HealQueueContext {
|
||||
Self::process_owned_heal_queue(HealQueueContext {
|
||||
heal_queue: &heal_queue,
|
||||
active_heals: &active_heals,
|
||||
completed_heals: &completed_heals,
|
||||
@@ -109,11 +109,11 @@ impl HealManager {
|
||||
cancel_token: &cancel_token,
|
||||
workload_provider: &workload_provider,
|
||||
disposition_cache: &disposition_cache,
|
||||
})
|
||||
}, &parent)
|
||||
.await;
|
||||
}
|
||||
_ = interval.tick() => {
|
||||
Self::process_heal_queue(HealQueueContext {
|
||||
Self::process_owned_heal_queue(HealQueueContext {
|
||||
heal_queue: &heal_queue,
|
||||
active_heals: &active_heals,
|
||||
completed_heals: &completed_heals,
|
||||
@@ -130,19 +130,28 @@ impl HealManager {
|
||||
cancel_token: &cancel_token,
|
||||
workload_provider: &workload_provider,
|
||||
disposition_cache: &disposition_cache,
|
||||
})
|
||||
}, &parent)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Process heal queue
|
||||
/// Processes multiple tasks per cycle when capacity allows and queue has high-priority items
|
||||
#[cfg(test)]
|
||||
pub(super) async fn process_heal_queue(context: HealQueueContext<'_>) {
|
||||
Self::process_heal_queue_inner(context, None).await;
|
||||
}
|
||||
|
||||
async fn process_owned_heal_queue(context: HealQueueContext<'_>, parent: &HealWork) {
|
||||
Self::process_heal_queue_inner(context, Some(parent)).await;
|
||||
}
|
||||
|
||||
async fn process_heal_queue_inner(context: HealQueueContext<'_>, parent: Option<&HealWork>) {
|
||||
let HealQueueContext {
|
||||
heal_queue,
|
||||
active_heals,
|
||||
@@ -254,7 +263,7 @@ impl HealManager {
|
||||
let config_for_spawn = config.clone();
|
||||
|
||||
// start heal task
|
||||
tokio::spawn(async move {
|
||||
spawn_heal_work(parent, move |parent| async move {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_SCHEDULER_STATE,
|
||||
@@ -267,7 +276,7 @@ impl HealManager {
|
||||
state = "task_started",
|
||||
"Heal scheduler task started"
|
||||
);
|
||||
let result = task.execute().await;
|
||||
let result = task.execute_owned(parent.clone()).await;
|
||||
let retry_request = retry_request_for_result_with_budget(task.as_ref(), &result).await;
|
||||
record_task_disposition(&disposition_cache_clone, task.as_ref(), &result, retry_request.is_some()).await;
|
||||
match &result {
|
||||
@@ -506,7 +515,7 @@ impl HealManager {
|
||||
let retry_notify = notify_clone.clone();
|
||||
let retry_manager_cancel_token = manager_cancel_token.clone();
|
||||
let retry_config = config_for_spawn.clone();
|
||||
tokio::spawn(async move {
|
||||
spawn_heal_work(parent.as_ref(), move |_parent| async move {
|
||||
let mut grace_admission_backoff = Duration::ZERO;
|
||||
loop {
|
||||
let now = SystemTime::now()
|
||||
@@ -735,7 +744,7 @@ impl HealManager {
|
||||
|
||||
if delayed_by_mainline_throttle && let Some(pressure) = mainline_pressure {
|
||||
Self::record_mainline_throttle_delay(pressure, &config);
|
||||
Self::schedule_mainline_throttle_recheck(notify.clone(), config.mainline_max_sleep);
|
||||
Self::schedule_mainline_throttle_recheck(notify.clone(), config.mainline_max_sleep, parent, cancel_token.clone());
|
||||
}
|
||||
|
||||
// Log queue status if items remain
|
||||
|
||||
@@ -28,6 +28,7 @@ use tempfile::TempDir;
|
||||
|
||||
pub(super) mod admin_overlap;
|
||||
mod admission_timing;
|
||||
mod owned_workers;
|
||||
mod root_recovery;
|
||||
use uuid::Uuid;
|
||||
mod auto_scan;
|
||||
@@ -1386,6 +1387,8 @@ fn mrf_verified_repair_event_requires_positive_exact_identity() {
|
||||
durable_anchor: None,
|
||||
};
|
||||
let matching = HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".to_string(),
|
||||
@@ -1537,6 +1540,8 @@ fn unverified_legacy_fixture(bucket: &str) -> (MrfRepairNoticeTarget, crate::hea
|
||||
durable_anchor: Some(anchor.clone()),
|
||||
};
|
||||
let outcome = HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: bucket.to_string(),
|
||||
@@ -1731,6 +1736,8 @@ fn completed_mrf_notice_publishes_only_verified_positive_events() {
|
||||
outcome.execution = HealExecutionOutcome::Completed;
|
||||
outcome.coverage = crate::heal::outcome::HealTraversalCoverage::Complete;
|
||||
outcome.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: bucket.to_string(),
|
||||
@@ -1744,6 +1751,8 @@ fn completed_mrf_notice_publishes_only_verified_positive_events() {
|
||||
detail: None,
|
||||
});
|
||||
outcome.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: bucket.to_string(),
|
||||
@@ -3113,6 +3122,8 @@ async fn local_automatic_query_preserves_source_path_token_and_state() {
|
||||
},
|
||||
disposition: HealObjectDisposition::VerifiedHealthy,
|
||||
detail: None,
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
});
|
||||
snapshot.outcome = Some(Arc::new(outcome));
|
||||
snapshot.seqed_items = vec![(
|
||||
@@ -6608,6 +6619,7 @@ fn admin_bucket_discovery_retries_keep_the_existing_budget() {
|
||||
async fn check_node_admission_task_retry(source: HealRequestSource) {
|
||||
use crate::heal::outcome::HealFailureClass;
|
||||
use crate::heal::storage::ECStoreHealStorage;
|
||||
use crate::heal::storage_api::owner::{PhysicalDiskSample, PhysicalDiskSampler};
|
||||
use rustfs_concurrency::adaptive::{FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure};
|
||||
use rustfs_heal_contracts::heal_channel::HealScanMode;
|
||||
use std::num::NonZeroU16;
|
||||
@@ -6620,6 +6632,18 @@ async fn check_node_admission_task_retry(source: HealRequestSource) {
|
||||
env.make_bucket("admission-retry", false).await;
|
||||
env.put_object_bytes("admission-retry", "existing", b"repair me".to_vec())
|
||||
.await;
|
||||
let mut physical = PhysicalDiskSampler::default();
|
||||
physical.publish(
|
||||
PhysicalDiskSample::collect(&env.ecstore).with_test_single_device(),
|
||||
ResourcePressure::Unknown,
|
||||
);
|
||||
assert!(
|
||||
env.ecstore
|
||||
.physical_heal_plan("existing", &Default::default())
|
||||
.expect("fixture placement")
|
||||
.is_some(),
|
||||
"physical discovery must be ready before testing node capacity"
|
||||
);
|
||||
let gate = NodeAdmission::new(NonZeroU16::new(1).expect("one node unit"));
|
||||
let storage: Arc<dyn HealStorageAPI> =
|
||||
Arc::new(ECStoreHealStorage::new(env.ecstore.clone()).with_admission(Some(gate.clone())));
|
||||
@@ -6660,6 +6684,10 @@ async fn check_node_admission_task_retry(source: HealRequestSource) {
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
physical.publish(
|
||||
PhysicalDiskSample::collect(&env.ecstore).with_test_single_device(),
|
||||
ResourcePressure::Unknown,
|
||||
);
|
||||
let retried = HealTask::from_request(retry, storage);
|
||||
retried
|
||||
.execute()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,5 @@
|
||||
use super::*;
|
||||
use crate::heal::outcome::{HealObjectIdentity, HealObjectKind, HealObjectOutcome};
|
||||
use crate::heal::outcome::{HealAbortReason, HealObjectIdentity, HealObjectKind, HealObjectOutcome};
|
||||
use rustfs_heal_contracts::{completion::subscribe_heal_completions, heal_channel::HealScanMode};
|
||||
|
||||
#[tokio::test]
|
||||
@@ -22,6 +22,8 @@ async fn canonical_completion_publishes_typed_objects_only_for_terminal_untrunca
|
||||
completed.options.scan_mode = HealScanMode::Normal;
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".into(),
|
||||
@@ -86,3 +88,129 @@ async fn canonical_completion_publishes_typed_objects_only_for_terminal_untrunca
|
||||
.expect("truncated terminal must still be observable");
|
||||
assert!(truncated.object.is_none(), "a retained tail cannot certify a whole result");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_carries_local_generation_but_wire_roundtrip_does_not() {
|
||||
let mut receiver = subscribe_heal_completions();
|
||||
let completed_heals = Mutex::new(HashMap::new());
|
||||
let aliases = Mutex::new(HashMap::new());
|
||||
let generation = Uuid::new_v4();
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.record(HealObjectOutcome {
|
||||
deep_check_observed: true,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: Some(Uuid::new_v4()),
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
},
|
||||
disposition: HealObjectDisposition::VerifiedHealthy,
|
||||
detail: None,
|
||||
quorum_selected_data_dir: Some(generation),
|
||||
});
|
||||
outcome.finish(None);
|
||||
let wire = serde_json::to_value(&outcome).expect("serialize outcome");
|
||||
assert!(wire["objects"][0].get("quorumSelectedDataDir").is_none());
|
||||
assert!(wire["objects"][0].get("deepCheckObserved").is_none());
|
||||
let restored: HealTaskOutcome = serde_json::from_value(wire).expect("deserialize outcome");
|
||||
|
||||
for (observed_outcome, expected_generation, expected_proof) in [(outcome, Some(generation), true), (restored, None, false)] {
|
||||
let id = Uuid::new_v4().to_string();
|
||||
let mut completed = completed_retention_fixture(SystemTime::now());
|
||||
completed.heal_type = HealType::Object {
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
version_id: None,
|
||||
};
|
||||
completed.status = HealTaskStatus::Completed;
|
||||
completed.outcome = Some(Arc::new(observed_outcome));
|
||||
publish_completed_heal(&completed_heals, &aliases, &id, completed, true).await;
|
||||
let observed = tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let event = receiver.recv().await.expect("terminal broadcaster stays open");
|
||||
if event.task_id == id {
|
||||
break event;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("terminal should be published");
|
||||
let receipt = observed.object.as_ref().expect("single typed outcome");
|
||||
assert_eq!(receipt.quorum_selected_data_dir, expected_generation);
|
||||
assert_eq!(receipt.deep_check_observed, expected_proof);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn completion_requires_success_and_a_non_nil_selection_for_deep_observation() {
|
||||
let mut receiver = subscribe_heal_completions();
|
||||
let completed_heals = Mutex::new(HashMap::new());
|
||||
let aliases = Mutex::new(HashMap::new());
|
||||
let generation = Uuid::new_v4();
|
||||
for (status, execution, dry_run, observed_generation) in [
|
||||
(
|
||||
HealTaskStatus::Cancelled,
|
||||
HealExecutionOutcome::Aborted(HealAbortReason::Cancelled),
|
||||
false,
|
||||
Some(generation),
|
||||
),
|
||||
(
|
||||
HealTaskStatus::Failed { error: "failure".into() },
|
||||
HealExecutionOutcome::Completed,
|
||||
false,
|
||||
Some(generation),
|
||||
),
|
||||
(
|
||||
HealTaskStatus::Completed,
|
||||
HealExecutionOutcome::Aborted(HealAbortReason::Deadline),
|
||||
false,
|
||||
Some(generation),
|
||||
),
|
||||
(HealTaskStatus::Completed, HealExecutionOutcome::Completed, true, Some(generation)),
|
||||
(HealTaskStatus::Completed, HealExecutionOutcome::Completed, false, None),
|
||||
(HealTaskStatus::Completed, HealExecutionOutcome::Completed, false, Some(Uuid::nil())),
|
||||
] {
|
||||
let id = Uuid::new_v4().to_string();
|
||||
let mut outcome = HealTaskOutcome::default();
|
||||
outcome.record(HealObjectOutcome {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
version_id: None,
|
||||
bucket_incarnation_id: Some(Uuid::new_v4()),
|
||||
pool_index: Some(0),
|
||||
set_index: Some(0),
|
||||
},
|
||||
disposition: HealObjectDisposition::VerifiedHealthy,
|
||||
detail: None,
|
||||
quorum_selected_data_dir: observed_generation,
|
||||
deep_check_observed: true,
|
||||
});
|
||||
outcome.execution = execution;
|
||||
let mut completed = completed_retention_fixture(SystemTime::now());
|
||||
completed.heal_type = HealType::Object {
|
||||
bucket: "bucket".into(),
|
||||
object: "object".into(),
|
||||
version_id: None,
|
||||
};
|
||||
completed.options.dry_run = dry_run;
|
||||
completed.status = status;
|
||||
completed.outcome = Some(Arc::new(outcome));
|
||||
publish_completed_heal(&completed_heals, &aliases, &id, completed, true).await;
|
||||
let observed = tokio::time::timeout(Duration::from_secs(1), async {
|
||||
loop {
|
||||
let event = receiver.recv().await.expect("terminal broadcaster stays open");
|
||||
if event.task_id == id {
|
||||
break event;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("terminal should be published");
|
||||
assert!(!observed.object.as_ref().expect("single typed outcome").deep_check_observed);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,366 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn owned_config() -> HealConfig {
|
||||
HealConfig {
|
||||
enable_auto_heal: false,
|
||||
event_driven_scheduler_enable: true,
|
||||
heal_interval: Duration::from_secs(60),
|
||||
mainline_throttle_enable: false,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
async fn until(mut ready: impl AsyncFnMut() -> bool) {
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while !ready().await {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("real producer reaches expected boundary");
|
||||
}
|
||||
|
||||
async fn completion_fixture() -> (Arc<HealManager>, Arc<CompletedRetentionHook>, String, String) {
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(owned_config()),
|
||||
));
|
||||
let bucket = format!("owned-worker-{}", Uuid::new_v4());
|
||||
let request = HealRequest::object(bucket.clone(), "object".to_owned(), None);
|
||||
let id = request.id.clone();
|
||||
let hook = Arc::new(CompletedRetentionHook {
|
||||
pause_before_publish: true,
|
||||
..Default::default()
|
||||
});
|
||||
{
|
||||
let mut hooks = COMPLETED_RETENTION_HOOKS.lock().await;
|
||||
hooks.insert(bucket.clone(), hook.clone());
|
||||
hooks.insert(id.clone(), hook.clone());
|
||||
}
|
||||
manager.submit_heal_request(request).await.expect("queue real object heal");
|
||||
manager.start().await.expect("start actual scheduler and both GCs");
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
|
||||
.await
|
||||
.expect("actual worker reaches storage");
|
||||
hook.execute.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.before_publish.notified())
|
||||
.await
|
||||
.expect("actual worker reaches existing publication boundary");
|
||||
(manager, hook, bucket, id)
|
||||
}
|
||||
|
||||
async fn remove_hook(bucket: &str, id: &str) {
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.retain(|key, _| key != bucket && key != id);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_manager_waits_actual_worker_without_holding_manager_locks() {
|
||||
let (manager, hook, bucket, id) = completion_fixture().await;
|
||||
let stopping = manager.clone();
|
||||
let mut stop = tokio::spawn(async move { stopping.stop().await });
|
||||
manager.cancel_token.cancelled().await;
|
||||
assert!(tokio::time::timeout(Duration::from_millis(30), &mut stop).await.is_err());
|
||||
// The real worker needs these locks after the existing test boundary.
|
||||
let admin = manager.admin_start_shutdown.try_lock().expect("admin released before join");
|
||||
let active = manager.active_heals.try_lock().expect("active released before join");
|
||||
let cache = manager.completed_heals.try_lock().expect("cache released before join");
|
||||
assert!(!manager.workers.tasks.is_empty(), "actual publication worker is still owned");
|
||||
drop((admin, active, cache));
|
||||
hook.publish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
|
||||
.await
|
||||
.expect("real worker reaches existing final handoff boundary");
|
||||
hook.finish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), stop)
|
||||
.await
|
||||
.expect("worker drain deadline")
|
||||
.expect("stop waiter")
|
||||
.expect("actual worker joined");
|
||||
assert!(manager.workers.tasks.is_empty());
|
||||
assert!(
|
||||
manager
|
||||
.workers
|
||||
.registration
|
||||
.lock()
|
||||
.unwrap()
|
||||
.fixed
|
||||
.iter()
|
||||
.flatten()
|
||||
.all(|join| join.peek().is_some())
|
||||
);
|
||||
assert!(manager.start().await.is_err(), "terminal owner cannot restart");
|
||||
manager.stop().await.expect("successful receipt is repeatable");
|
||||
remove_hook(&bucket, &id).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_manager_cancelled_stop_waiter_keeps_actual_worker_receipt() {
|
||||
let (manager, hook, bucket, id) = completion_fixture().await;
|
||||
let stopping = manager.clone();
|
||||
let stop = tokio::spawn(async move { stopping.stop().await });
|
||||
manager.cancel_token.cancelled().await;
|
||||
until(async || manager.admin_start_shutdown.try_lock().is_ok()).await;
|
||||
stop.abort();
|
||||
assert!(stop.await.unwrap_err().is_cancelled());
|
||||
let stopping = manager.clone();
|
||||
let mut retry = tokio::spawn(async move { stopping.stop().await });
|
||||
assert!(tokio::time::timeout(Duration::from_millis(30), &mut retry).await.is_err());
|
||||
hook.publish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.handoff.notified())
|
||||
.await
|
||||
.expect("real worker reaches existing final handoff boundary");
|
||||
hook.finish.notify_one();
|
||||
tokio::time::timeout(Duration::from_secs(5), retry)
|
||||
.await
|
||||
.expect("repeat drain deadline")
|
||||
.expect("retry waiter")
|
||||
.expect("same actual worker joined");
|
||||
assert!(manager.workers.tasks.is_empty());
|
||||
remove_hook(&bucket, &id).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_manager_start_stop_race_counts_independent_stop_intents() {
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(owned_config()),
|
||||
));
|
||||
let admin = manager.admin_start_shutdown.lock().await;
|
||||
let starting = manager.clone();
|
||||
let start = tokio::spawn(async move { starting.start().await });
|
||||
until(async || manager.state.read().await.is_running).await;
|
||||
let stopping = manager.clone();
|
||||
let first = tokio::spawn(async move { stopping.stop().await });
|
||||
let stopping = manager.clone();
|
||||
let second = tokio::spawn(async move { stopping.stop().await });
|
||||
until(async || manager.workers.registration.lock().unwrap().stopping == 2).await;
|
||||
first.abort();
|
||||
assert!(first.await.unwrap_err().is_cancelled());
|
||||
assert_eq!(manager.workers.registration.lock().unwrap().stopping, 1);
|
||||
assert!(manager.start().await.is_err(), "cancelled waiter must not reopen another waiter's gate");
|
||||
assert!(
|
||||
!manager.cancel_token.is_cancelled(),
|
||||
"handoff waits for startup before signalling cancellation"
|
||||
);
|
||||
assert!(!second.is_finished());
|
||||
drop(admin);
|
||||
start
|
||||
.await
|
||||
.expect("actual startup waiter")
|
||||
.expect("startup finishes before handoff");
|
||||
tokio::time::timeout(Duration::from_secs(5), second)
|
||||
.await
|
||||
.expect("stop drains all newly registered loops")
|
||||
.expect("stop waiter")
|
||||
.expect("joined");
|
||||
assert!(manager.workers.tasks.is_empty());
|
||||
assert_eq!(manager.workers.registration.lock().unwrap().fixed.iter().flatten().count(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_manager_cancelled_pre_handoff_stop_reopens_start_admission() {
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(owned_config()),
|
||||
));
|
||||
let active = manager.active_heals.lock().await;
|
||||
let stopping = manager.clone();
|
||||
let stop = tokio::spawn(async move { stopping.stop().await });
|
||||
until(async || manager.workers.registration.lock().unwrap().stopping == 1).await;
|
||||
until(async || manager.admin_start_shutdown.try_lock().is_err()).await;
|
||||
stop.abort();
|
||||
assert!(stop.await.unwrap_err().is_cancelled());
|
||||
assert_eq!(manager.workers.registration.lock().unwrap().stopping, 0);
|
||||
assert!(!manager.cancel_token.is_cancelled());
|
||||
drop(active);
|
||||
manager
|
||||
.start()
|
||||
.await
|
||||
.expect("pre-handoff cancellation did not terminally close startup");
|
||||
manager.stop().await.expect("later real startup and stop succeed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_manager_failed_durable_handoff_remains_open_and_retryable() {
|
||||
use super::super::super::RUSTFS_META_BUCKET;
|
||||
use super::super::root_recovery::RootHealRecovery;
|
||||
let temp = TempDir::new().expect("owned recovery disk");
|
||||
let endpoint = Endpoint::try_from(temp.path().to_string_lossy().as_ref()).expect("endpoint");
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("real local disk");
|
||||
match disk.make_volume(RUSTFS_META_BUCKET).await {
|
||||
Ok(()) | Err(DiskError::VolumeExists) => {}
|
||||
Err(error) => panic!("metadata volume: {error}"),
|
||||
}
|
||||
let mut manager = HealManager::new(Arc::new(MockStorage), Some(owned_config()));
|
||||
manager.root_recovery = Arc::new(RootHealRecovery::with_disks(vec![disk.clone()]));
|
||||
let mut request = HealRequest::new(HealType::Cluster, HealOptions::default(), HealPriority::High);
|
||||
request.source = HealRequestSource::Admin;
|
||||
let id = request.id.clone();
|
||||
manager.submit_heal_request(request).await.expect("real durable admission");
|
||||
std::fs::remove_dir_all(temp.path().join(RUSTFS_META_BUCKET)).expect("remove accepted owner volume");
|
||||
assert!(manager.stop().await.is_err(), "real durable handoff fails");
|
||||
assert!(!manager.stop_handoff_completed());
|
||||
assert!(!manager.cancel_token.is_cancelled());
|
||||
assert!(manager.workers.registration.lock().unwrap().open);
|
||||
assert_eq!(manager.workers.registration.lock().unwrap().stopping, 0);
|
||||
assert!(manager.heal_queue.lock().await.requests().any(|request| request.id == id));
|
||||
disk.make_volume(RUSTFS_META_BUCKET).await.expect("restore original owner");
|
||||
manager.stop().await.expect("handoff retries on same disk and drains");
|
||||
assert!(manager.stop_handoff_completed());
|
||||
assert!(!manager.workers.registration.lock().unwrap().open);
|
||||
assert_eq!(
|
||||
manager
|
||||
.root_recovery
|
||||
.pending()
|
||||
.await
|
||||
.expect("durable successor remains")
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_manager_real_retry_and_throttle_wake_drain() {
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(owned_config()),
|
||||
));
|
||||
let hook = Arc::new(CompletedRetentionHook::default());
|
||||
COMPLETED_RETENTION_HOOKS
|
||||
.lock()
|
||||
.await
|
||||
.insert("retry-transition".to_owned(), hook.clone());
|
||||
manager
|
||||
.submit_heal_request(HealRequest::object("retry-transition".to_owned(), "object".to_owned(), None))
|
||||
.await
|
||||
.expect("queue actual recoverable storage failure");
|
||||
manager.start().await.expect("actual scheduler");
|
||||
tokio::time::timeout(Duration::from_secs(5), hook.started.notified())
|
||||
.await
|
||||
.expect("real object operation");
|
||||
hook.execute.notify_one();
|
||||
until(async || !manager.retrying_heals.lock().await.is_empty()).await;
|
||||
manager.stop().await.expect("actual delayed retry drained");
|
||||
assert!(manager.workers.tasks.is_empty());
|
||||
assert!(manager.retrying_heals.lock().await.is_empty());
|
||||
COMPLETED_RETENTION_HOOKS.lock().await.remove("retry-transition");
|
||||
|
||||
let provider: WorkloadSnapshotProviderRef = Arc::new(FixedWorkloadProvider {
|
||||
class: WorkloadClass::ForegroundRead,
|
||||
active: 8,
|
||||
limit: 10,
|
||||
state: AdmissionState::Open,
|
||||
});
|
||||
let manager = HealManager::new_with_workload_provider_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
mainline_throttle_enable: true,
|
||||
mainline_read_utilization_high_percent: 80,
|
||||
mainline_write_utilization_high_percent: 80,
|
||||
mainline_max_sleep: Duration::from_secs(60),
|
||||
..owned_config()
|
||||
}),
|
||||
Some(provider),
|
||||
);
|
||||
manager.start().await.expect("actual throttled scheduler");
|
||||
until(async || manager.workers.tasks.len() == 4).await;
|
||||
manager
|
||||
.submit_heal_request(bucket_request("owned-throttle", HealPriority::Normal, HealRequestSource::ReadRepair))
|
||||
.await
|
||||
.expect("real pressure-delayed request");
|
||||
until(async || manager.workers.tasks.len() > 4).await;
|
||||
assert_eq!(manager.get_queue_length().await, 1);
|
||||
tokio::time::timeout(Duration::from_secs(5), manager.stop())
|
||||
.await
|
||||
.expect("wake cancels without waiting 60s")
|
||||
.expect("wake drained");
|
||||
assert!(manager.workers.tasks.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_manager_real_scheduler_panic_is_sticky_failure() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
heal_interval: Duration::ZERO,
|
||||
..owned_config()
|
||||
}),
|
||||
);
|
||||
manager.start().await.expect("startup registered actual scheduler");
|
||||
until(async || manager.workers.failed.load(Ordering::Acquire)).await;
|
||||
assert!(manager.stop().await.is_err(), "actual interval panic cannot acknowledge joined success");
|
||||
assert!(manager.stop_handoff_completed());
|
||||
assert!(manager.stop().await.is_err(), "repeat wait cannot erase failure");
|
||||
assert!(manager.workers.tasks.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_manager_completed_dynamic_workers_do_not_accumulate_handles() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), Some(owned_config()));
|
||||
manager.start().await.expect("actual producer loops");
|
||||
for index in 0..64 {
|
||||
let request = HealRequest::object(format!("owned-completed-{index}"), "object".to_owned(), None);
|
||||
let id = request.id.clone();
|
||||
manager
|
||||
.submit_heal_request(request)
|
||||
.await
|
||||
.expect("actual scheduler admission");
|
||||
until(async || manager.completed_heals.lock().await.contains_key(&id)).await;
|
||||
}
|
||||
// Scheduler retains a live parent plus its guard; each GC retains one guard.
|
||||
// These four fixed-loop tokens remain, irrespective of the 64 finished jobs.
|
||||
until(async || manager.workers.tasks.len() == 4).await;
|
||||
assert_eq!(manager.workers.registration.lock().unwrap().fixed.iter().flatten().count(), 3);
|
||||
manager.stop().await.expect("bounded dynamic bookkeeping drains");
|
||||
assert!(manager.workers.tasks.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_manager_enabled_auto_scan_has_four_actual_loop_receipts() {
|
||||
let manager = HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(HealConfig {
|
||||
enable_auto_heal: true,
|
||||
..owned_config()
|
||||
}),
|
||||
);
|
||||
manager.start().await.expect("four actual producer loops");
|
||||
assert_eq!(manager.workers.registration.lock().unwrap().fixed.iter().flatten().count(), 4);
|
||||
manager.stop().await.expect("four loops joined");
|
||||
assert!(manager.workers.tasks.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_manager_cancelled_actual_start_is_sticky_failure() {
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(MockStorage),
|
||||
Some(owned_config()),
|
||||
));
|
||||
let admin = manager.admin_start_shutdown.lock().await;
|
||||
let starting = manager.clone();
|
||||
let start = tokio::spawn(async move { starting.start().await });
|
||||
until(async || manager.state.read().await.is_running).await;
|
||||
start.abort();
|
||||
assert!(start.await.unwrap_err().is_cancelled());
|
||||
drop(admin);
|
||||
assert!(
|
||||
manager.stop().await.is_err(),
|
||||
"cancelled partial startup cannot become successful receipt"
|
||||
);
|
||||
assert!(manager.stop().await.is_err(), "failure remains sticky for later waiter");
|
||||
assert!(manager.workers.tasks.is_empty());
|
||||
}
|
||||
@@ -192,6 +192,8 @@ fn terminal_with_outcome(heal_type: &HealType, status: HealTaskStatus) -> Comple
|
||||
HealObjectDisposition::VerifiedHealthy
|
||||
};
|
||||
outcome.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".to_string(),
|
||||
@@ -2049,6 +2051,8 @@ async fn root_recovery_cluster_replay_rejects_successor_and_preserves_completed_
|
||||
.insert(bucket.clone(), Some(expected));
|
||||
scope.next_bucket_index = 1;
|
||||
scope.outcome.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: bucket.clone(),
|
||||
@@ -3132,3 +3136,65 @@ async fn root_recovery_conflicting_record_does_not_block_other_replays() {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
#[tokio::test]
|
||||
async fn owned_terminal_gc_waits_for_actual_native_receipt_read() {
|
||||
use crate::heal::storage_api::{
|
||||
EcstoreLocalPublicationPause as LocalPublicationPause, EcstoreLocalPublicationStage as LocalPublicationStage,
|
||||
};
|
||||
|
||||
let (_temp, disk) = recovery_disk().await;
|
||||
let manager = recovery_manager(vec![disk.clone()]);
|
||||
let request = root_request();
|
||||
let completed_at = SystemTime::now()
|
||||
.checked_sub(KEEP_HEAL_TASK_STATUS_DURATION + Duration::from_secs(1))
|
||||
.expect("expired fixture time");
|
||||
let completed = completed_admin_status(&request.heal_type, completed_at);
|
||||
manager
|
||||
.publish_admin_terminal(&request.id, &request.heal_type, request.source, &completed)
|
||||
.await
|
||||
.expect("persist real expired terminal");
|
||||
let path = format!("terminal-root-heal-{}.json", request.id);
|
||||
assert!(
|
||||
!disk
|
||||
.read_all(RUSTFS_META_BUCKET, &path)
|
||||
.await
|
||||
.expect("nonempty real terminal")
|
||||
.is_empty()
|
||||
);
|
||||
let mut pause =
|
||||
LocalPublicationPause::install(disk.as_ref(), RUSTFS_META_BUCKET, &path, LocalPublicationStage::NativeReadOpen)
|
||||
.expect("native bounded terminal-read pause");
|
||||
manager
|
||||
.start_root_recovery_terminal_gc()
|
||||
.await
|
||||
.expect("actual slot-3 producer");
|
||||
tokio::time::timeout(Duration::from_secs(5), pause.entered())
|
||||
.await
|
||||
.expect("terminal GC native read entry deadline")
|
||||
.expect("real bounded terminal read entered");
|
||||
let mut stop = Box::pin(manager.stop());
|
||||
let first = tokio::time::timeout(Duration::from_millis(30), stop.as_mut()).await;
|
||||
let held = first.is_err();
|
||||
let unlocked = manager.admin_start_shutdown.try_lock().is_ok()
|
||||
&& manager.active_heals.try_lock().is_ok()
|
||||
&& manager.completed_heals.try_lock().is_ok();
|
||||
drop(pause);
|
||||
let result = match first {
|
||||
Ok(result) => Ok(result),
|
||||
Err(_) => tokio::time::timeout(Duration::from_secs(5), stop).await,
|
||||
};
|
||||
assert!(held, "terminal mutation branch must finish before its receipt");
|
||||
assert!(unlocked, "join may not hold manager locks");
|
||||
result.expect("released terminal GC joins").expect("terminal GC stop");
|
||||
assert!(matches!(disk.read_all(RUSTFS_META_BUCKET, &path).await, Err(DiskError::FileNotFound)));
|
||||
let registration = manager.workers.registration.lock().unwrap();
|
||||
assert!(
|
||||
registration.fixed[3]
|
||||
.as_ref()
|
||||
.expect("actual terminal GC receipt")
|
||||
.peek()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -240,7 +240,7 @@ impl HealManager {
|
||||
}
|
||||
|
||||
for set_disk_id in &set_disk_ids {
|
||||
match self.storage.replacement_intent_disks(set_disk_id).await {
|
||||
match self.storage.replacement_intent_disks(set_disk_id, None).await {
|
||||
Ok(disks) => recovery_disks.extend(disks),
|
||||
Err(error) => {
|
||||
conflicted_replacement_sets.insert(set_disk_id.clone());
|
||||
|
||||
@@ -1044,6 +1044,33 @@ struct MrfRuntime {
|
||||
}
|
||||
|
||||
impl MrfRuntime {
|
||||
fn admit_best_effort(&mut self, intent: MrfIntent) {
|
||||
if intent.kind.is_durable() {
|
||||
if self.admit_partial_write(intent.clone(), None).is_err() {
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if !self.queue.pending_keys.contains(&queue_key(&intent))
|
||||
&& (self.queue.depth() + self.partial_writes.depth() >= self.queue.capacity
|
||||
|| self.queue.bytes() + self.partial_writes.bytes() + intent.estimated_bytes() > self.queue.byte_budget)
|
||||
{
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1);
|
||||
return;
|
||||
}
|
||||
match self.queue.try_push_typed(intent.clone()) {
|
||||
MrfQueuePushResult::Enqueued => {
|
||||
self.new_since_flush += 1;
|
||||
self.dirty = true;
|
||||
}
|
||||
MrfQueuePushResult::Coalesced | MrfQueuePushResult::Rejected => {
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn adopt_replayed_partial_writes(&mut self, intents: Vec<(MrfIntent, Option<ResponsibilityCheckpoint>)>) {
|
||||
// The decoded checkpoint bounds replay; live admission subsequently
|
||||
// shares these limits with the ordinary pending queue.
|
||||
@@ -1836,6 +1863,20 @@ fn map_mrf_lifecycle_state_error(error: &'static str) -> MrfLifecycleControlErro
|
||||
/// ordinary MRF kill-switch.
|
||||
/// Public for integration tests that drive the real consumer loop.
|
||||
pub fn spawn_mrf_consumer(manager: Arc<HealManager>) {
|
||||
let _ = spawn_mrf_consumer_with_stop(manager, None);
|
||||
}
|
||||
|
||||
pub(crate) fn spawn_owned_mrf_consumer(
|
||||
manager: Arc<HealManager>,
|
||||
stop: tokio_util::sync::CancellationToken,
|
||||
) -> Option<tokio::task::JoinHandle<()>> {
|
||||
spawn_mrf_consumer_with_stop(manager, Some(stop))
|
||||
}
|
||||
|
||||
fn spawn_mrf_consumer_with_stop(
|
||||
manager: Arc<HealManager>,
|
||||
stop: Option<tokio_util::sync::CancellationToken>,
|
||||
) -> Option<tokio::task::JoinHandle<()>> {
|
||||
let enabled = rustfs_utils::get_env_bool(rustfs_config::ENV_HEAL_MRF_ENABLE, rustfs_config::DEFAULT_HEAL_MRF_ENABLE);
|
||||
rustfs_common::mrf_channel::set_mrf_delivery_enabled(enabled);
|
||||
if !enabled {
|
||||
@@ -1857,13 +1898,21 @@ pub fn spawn_mrf_consumer(manager: Arc<HealManager>) {
|
||||
error = err,
|
||||
"MRF channel initialization failed; intents will be dropped at producers"
|
||||
);
|
||||
return;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
tokio::spawn(async move {
|
||||
run_mrf_consumer(manager, receiver, durable_receiver, lifecycle_receiver).await;
|
||||
});
|
||||
let task = match stop {
|
||||
Some(stop) => tokio::spawn(run_mrf_consumer_with_stop(
|
||||
manager,
|
||||
receiver,
|
||||
durable_receiver,
|
||||
lifecycle_receiver,
|
||||
Some(stop),
|
||||
)),
|
||||
None => tokio::spawn(run_mrf_consumer(manager, receiver, durable_receiver, lifecycle_receiver)),
|
||||
};
|
||||
tracing::info!(target: "rustfs::heal::mrf", "MRF intent consumer started");
|
||||
Some(task)
|
||||
}
|
||||
|
||||
/// Replay the journal once for integration tests, publishing a durable
|
||||
@@ -2222,10 +2271,20 @@ async fn replay_into(
|
||||
/// Replay the journal, then keep draining the channel into the heal manager
|
||||
/// while persisting the pending snapshot.
|
||||
async fn run_mrf_consumer(
|
||||
manager: Arc<HealManager>,
|
||||
receiver: mpsc::Receiver<MrfIntent>,
|
||||
durable_receiver: mpsc::Receiver<MrfDurableSubmission>,
|
||||
lifecycle_receiver: mpsc::Receiver<MrfLifecycleCommand>,
|
||||
) {
|
||||
run_mrf_consumer_with_stop(manager, receiver, durable_receiver, lifecycle_receiver, None).await;
|
||||
}
|
||||
|
||||
async fn run_mrf_consumer_with_stop(
|
||||
manager: Arc<HealManager>,
|
||||
mut receiver: mpsc::Receiver<MrfIntent>,
|
||||
mut durable_receiver: mpsc::Receiver<MrfDurableSubmission>,
|
||||
mut lifecycle_receiver: mpsc::Receiver<MrfLifecycleCommand>,
|
||||
stop: Option<tokio_util::sync::CancellationToken>,
|
||||
) {
|
||||
let config = MrfConsumerConfig::default();
|
||||
let mut runtime = MrfRuntime {
|
||||
@@ -2277,10 +2336,26 @@ async fn run_mrf_consumer(
|
||||
let mut durable_batch = Vec::with_capacity(runtime.config.replay_batch);
|
||||
|
||||
loop {
|
||||
if stop.as_ref().is_some_and(tokio_util::sync::CancellationToken::is_cancelled) {
|
||||
finalize_mrf_consumer(&mut runtime, &mut receiver, &mut durable_receiver, &mut lifecycle_receiver).await;
|
||||
return;
|
||||
}
|
||||
if runtime.partial_writes.pending_lifecycle_transition.is_some() {
|
||||
tokio::select! {
|
||||
_ = async {
|
||||
match &stop {
|
||||
Some(stop) => stop.cancelled().await,
|
||||
None => std::future::pending::<()>().await,
|
||||
}
|
||||
} => {
|
||||
finalize_mrf_consumer(&mut runtime, &mut receiver, &mut durable_receiver, &mut lifecycle_receiver).await;
|
||||
return;
|
||||
}
|
||||
received = durable_receiver.recv_many(&mut durable_batch, runtime.config.replay_batch) => {
|
||||
if received == 0 {
|
||||
if stop.is_some() {
|
||||
finalize_mrf_consumer(&mut runtime, &mut receiver, &mut durable_receiver, &mut lifecycle_receiver).await;
|
||||
}
|
||||
return;
|
||||
}
|
||||
for submission in durable_batch.drain(..) {
|
||||
@@ -2289,6 +2364,9 @@ async fn run_mrf_consumer(
|
||||
}
|
||||
command = lifecycle_receiver.recv() => {
|
||||
let Some(command) = command else {
|
||||
if stop.is_some() {
|
||||
finalize_mrf_consumer(&mut runtime, &mut receiver, &mut durable_receiver, &mut lifecycle_receiver).await;
|
||||
}
|
||||
return;
|
||||
};
|
||||
runtime.handle_lifecycle_command(manager.as_ref(), command).await;
|
||||
@@ -2301,8 +2379,20 @@ async fn run_mrf_consumer(
|
||||
continue;
|
||||
}
|
||||
tokio::select! {
|
||||
_ = async {
|
||||
match &stop {
|
||||
Some(stop) => stop.cancelled().await,
|
||||
None => std::future::pending::<()>().await,
|
||||
}
|
||||
} => {
|
||||
finalize_mrf_consumer(&mut runtime, &mut receiver, &mut durable_receiver, &mut lifecycle_receiver).await;
|
||||
return;
|
||||
}
|
||||
received = durable_receiver.recv_many(&mut durable_batch, runtime.config.replay_batch) => {
|
||||
if received == 0 {
|
||||
if stop.is_some() {
|
||||
finalize_mrf_consumer(&mut runtime, &mut receiver, &mut durable_receiver, &mut lifecycle_receiver).await;
|
||||
}
|
||||
return;
|
||||
}
|
||||
let mut responses = Vec::with_capacity(received);
|
||||
@@ -2322,12 +2412,19 @@ async fn run_mrf_consumer(
|
||||
}
|
||||
command = lifecycle_receiver.recv() => {
|
||||
let Some(command) = command else {
|
||||
if stop.is_some() {
|
||||
finalize_mrf_consumer(&mut runtime, &mut receiver, &mut durable_receiver, &mut lifecycle_receiver).await;
|
||||
}
|
||||
return;
|
||||
};
|
||||
runtime.handle_lifecycle_command(manager.as_ref(), command).await;
|
||||
}
|
||||
received = receiver.recv_many(&mut batch, runtime.config.replay_batch) => {
|
||||
if received == 0 {
|
||||
if stop.is_some() {
|
||||
finalize_mrf_consumer(&mut runtime, &mut receiver, &mut durable_receiver, &mut lifecycle_receiver).await;
|
||||
return;
|
||||
}
|
||||
// Channel closed: flush once more unless the snapshot is
|
||||
// provably current AND idle (a dirty or pending state
|
||||
// gets one last persist attempt, matching the shutdown
|
||||
@@ -2342,30 +2439,7 @@ async fn run_mrf_consumer(
|
||||
return;
|
||||
}
|
||||
for intent in batch.drain(..) {
|
||||
if intent.kind.is_durable() {
|
||||
if runtime.admit_partial_write(intent.clone(), None).is_err() {
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if !runtime.queue.pending_keys.contains(&queue_key(&intent))
|
||||
&& (runtime.queue.depth() + runtime.partial_writes.depth() >= runtime.queue.capacity
|
||||
|| runtime.queue.bytes() + runtime.partial_writes.bytes() + intent.estimated_bytes() > runtime.queue.byte_budget)
|
||||
{
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
counter!("rustfs_heal_mrf_dropped_total", "reason" => "queue_overflow").increment(1);
|
||||
continue;
|
||||
}
|
||||
match runtime.queue.try_push_typed(intent.clone()) {
|
||||
MrfQueuePushResult::Enqueued => {
|
||||
runtime.new_since_flush += 1;
|
||||
runtime.dirty = true;
|
||||
}
|
||||
MrfQueuePushResult::Coalesced | MrfQueuePushResult::Rejected => {
|
||||
rustfs_common::mrf_channel::release_mrf_intent(&intent);
|
||||
}
|
||||
}
|
||||
runtime.admit_best_effort(intent);
|
||||
}
|
||||
if runtime.dirty && runtime.partial_writes.depth() > 0 {
|
||||
runtime.flush().await;
|
||||
@@ -2411,6 +2485,64 @@ async fn run_mrf_consumer(
|
||||
}
|
||||
}
|
||||
|
||||
// Closing the receivers fences new ingress while preserving all messages that
|
||||
// were already accepted by these channels. Existing admission limits and replies
|
||||
// remain unchanged; a failed checkpoint keeps this runtime alive for retry.
|
||||
async fn finalize_mrf_consumer(
|
||||
runtime: &mut MrfRuntime,
|
||||
receiver: &mut mpsc::Receiver<MrfIntent>,
|
||||
durable_receiver: &mut mpsc::Receiver<MrfDurableSubmission>,
|
||||
lifecycle_receiver: &mut mpsc::Receiver<MrfLifecycleCommand>,
|
||||
) {
|
||||
receiver.close();
|
||||
durable_receiver.close();
|
||||
lifecycle_receiver.close();
|
||||
while let Some(command) = lifecycle_receiver.recv().await {
|
||||
match command {
|
||||
MrfLifecycleCommand::List { response, .. } => {
|
||||
let _ = response.send(Err(MrfLifecycleControlError::Unavailable));
|
||||
}
|
||||
MrfLifecycleCommand::Recheck { response, .. }
|
||||
| MrfLifecycleCommand::Refresh { response, .. }
|
||||
| MrfLifecycleCommand::AcceptUnverifiedRisk { response, .. } => {
|
||||
let _ = response.send(Err(MrfLifecycleControlError::Unavailable));
|
||||
}
|
||||
}
|
||||
}
|
||||
while let Some(intent) = receiver.recv().await {
|
||||
runtime.admit_best_effort(intent);
|
||||
}
|
||||
let mut responses = Vec::new();
|
||||
while let Some(submission) = durable_receiver.recv().await {
|
||||
match runtime.admit_partial_write(submission.intent, submission.source_bucket_incarnation_id) {
|
||||
Ok(()) => responses.push(submission.response),
|
||||
Err(error) => {
|
||||
let _ = submission.response.send(Err(error));
|
||||
}
|
||||
}
|
||||
}
|
||||
let needs_flush = runtime.dirty || runtime.queue.depth() > 0 || runtime.partial_writes.depth() > 0;
|
||||
let persisted = !needs_flush || runtime.flush().await;
|
||||
for response in responses {
|
||||
let _ = response.send(if persisted {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(MrfDurableAdmissionError::Persistence)
|
||||
});
|
||||
}
|
||||
if !persisted {
|
||||
let mut retry = tokio::time::interval(runtime.config.flush_interval);
|
||||
retry.tick().await;
|
||||
loop {
|
||||
retry.tick().await;
|
||||
if runtime.flush().await {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
runtime.publish_metrics();
|
||||
}
|
||||
|
||||
/// What the periodic tick should do, as a pure function of the runtime state
|
||||
/// so the decision table is unit-testable.
|
||||
enum TickAction {
|
||||
@@ -2456,6 +2588,7 @@ mod tests {
|
||||
mod empty_events_tests;
|
||||
mod lifecycle_replay_tests;
|
||||
mod lifecycle_v2_reader_tests;
|
||||
mod owned_shutdown_tests;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn journal_disk_io_bounds_fanout_and_collects_every_result() {
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn empty_runtime(capacity: usize) -> MrfRuntime {
|
||||
let config = MrfConsumerConfig::default();
|
||||
MrfRuntime {
|
||||
queue: MrfQueue::new(capacity, config.journal_max_bytes),
|
||||
partial_writes: PartialWrites::default(),
|
||||
config,
|
||||
checkpoint_owner: Uuid::new_v4(),
|
||||
next_checkpoint_sequence: 1,
|
||||
lifecycle_format_version: MRF_LIFECYCLE_FORMAT_VERSION,
|
||||
new_since_flush: 0,
|
||||
dirty: false,
|
||||
journal_on_disk: false,
|
||||
retain_replay_journal: false,
|
||||
replay_readable: true,
|
||||
durable_replay_anchors: Vec::new(),
|
||||
replay_cleanup: None,
|
||||
runtime_checkpoint: None,
|
||||
backoff_until: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn partial(object: &str) -> MrfIntent {
|
||||
let mut item = intent("mrf-owned-shutdown", object, 0);
|
||||
item.kind = MrfKind::PartialWrite;
|
||||
item.version_id = None;
|
||||
item
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn owned_mrf_shutdown_drains_and_persists_before_ack_and_join() {
|
||||
let _env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("mrf_owned_shutdown")
|
||||
.build()
|
||||
.await;
|
||||
let mut runtime = empty_runtime(8);
|
||||
let (incoming, mut receiver) = mpsc::channel(4);
|
||||
let (durable, mut durable_receiver) = mpsc::channel(4);
|
||||
let (lifecycle, mut lifecycle_receiver) = mpsc::channel(4);
|
||||
incoming
|
||||
.send(intent("mrf-owned-shutdown", "ordinary", 0))
|
||||
.await
|
||||
.expect("queued ordinary intent");
|
||||
let (reply, ack) = tokio::sync::oneshot::channel();
|
||||
durable
|
||||
.send(MrfDurableSubmission {
|
||||
intent: partial("durable"),
|
||||
source_bucket_incarnation_id: Some(Uuid::new_v4()),
|
||||
response: reply,
|
||||
})
|
||||
.await
|
||||
.expect("queued durable submission");
|
||||
let (response, listed) = tokio::sync::oneshot::channel();
|
||||
lifecycle
|
||||
.send(MrfLifecycleCommand::List {
|
||||
after: None,
|
||||
limit: 1,
|
||||
response,
|
||||
})
|
||||
.await
|
||||
.expect("queued lifecycle query");
|
||||
tokio::time::timeout(
|
||||
StdDuration::from_secs(10),
|
||||
finalize_mrf_consumer(&mut runtime, &mut receiver, &mut durable_receiver, &mut lifecycle_receiver),
|
||||
)
|
||||
.await
|
||||
.expect("real final checkpoint completes with live sender handles");
|
||||
assert!(ack.await.expect("explicit durable acknowledgement").is_ok());
|
||||
assert!(matches!(
|
||||
listed.await.expect("explicit query rejection"),
|
||||
Err(MrfLifecycleControlError::Unavailable)
|
||||
));
|
||||
assert!(incoming.is_closed() && durable.is_closed() && lifecycle.is_closed());
|
||||
assert!(!runtime.dirty);
|
||||
let saved = snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes)
|
||||
.await
|
||||
.expect("inspect committed shutdown snapshot")
|
||||
.expect("shutdown successor exists");
|
||||
let (items, truncated) = decode_journal(saved.payload());
|
||||
assert_eq!(truncated, 0);
|
||||
assert_eq!(items.len(), 2, "both ingress kinds have durable replay storage before successful receipt");
|
||||
}
|
||||
|
||||
// The blocked paths belong to this fresh test environment. Restore them even
|
||||
// when an assertion fails so its ordinary disk cleanup can finish.
|
||||
struct BlockedMetadata(Vec<(PathBuf, PathBuf)>);
|
||||
impl Drop for BlockedMetadata {
|
||||
fn drop(&mut self) {
|
||||
for (original, held) in self.0.drain(..) {
|
||||
let _ = fs::remove_file(&original);
|
||||
let _ = fs::rename(held, original);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn owned_mrf_failed_flush_retains_runtime_until_disk_recovers() {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("mrf_owned_retry")
|
||||
.build()
|
||||
.await;
|
||||
let mut runtime = empty_runtime(8);
|
||||
runtime
|
||||
.admit_partial_write(partial("previously-committed"), Some(Uuid::new_v4()))
|
||||
.expect("accept resident responsibility");
|
||||
assert!(runtime.flush().await, "establish the prior committed responsibility before failure");
|
||||
let mut blocked = BlockedMetadata(Vec::new());
|
||||
for root in &env.disk_paths {
|
||||
let original = root.join(RUSTFS_META_BUCKET);
|
||||
let held = root.join("mrf-owned-held-metadata");
|
||||
assert!(!held.exists());
|
||||
fs::rename(&original, &held).expect("hold only this fixture metadata volume");
|
||||
blocked.0.push((original.clone(), held));
|
||||
fs::write(&original, b"block directory traversal").expect("force actual disk writes to fail");
|
||||
}
|
||||
let (incoming, mut receiver) = mpsc::channel(4);
|
||||
let (durable, mut durable_receiver) = mpsc::channel(4);
|
||||
let (_lifecycle, mut lifecycle_receiver) = mpsc::channel(4);
|
||||
let (reply, ack) = tokio::sync::oneshot::channel();
|
||||
durable
|
||||
.send(MrfDurableSubmission {
|
||||
intent: partial("retained-after-failed-flush"),
|
||||
source_bucket_incarnation_id: Some(Uuid::new_v4()),
|
||||
response: reply,
|
||||
})
|
||||
.await
|
||||
.expect("queue durable responsibility");
|
||||
let mut shutdown = tokio::spawn(async move {
|
||||
finalize_mrf_consumer(&mut runtime, &mut receiver, &mut durable_receiver, &mut lifecycle_receiver).await;
|
||||
runtime
|
||||
});
|
||||
assert!(matches!(
|
||||
tokio::time::timeout(StdDuration::from_secs(10), ack)
|
||||
.await
|
||||
.expect("failed checkpoint explicitly responds")
|
||||
.expect("durable reply is not dropped"),
|
||||
Err(MrfDurableAdmissionError::Persistence)
|
||||
));
|
||||
assert!(
|
||||
tokio::time::timeout(StdDuration::from_millis(100), &mut shutdown)
|
||||
.await
|
||||
.is_err(),
|
||||
"failed flush is not an exited/joined consumer"
|
||||
);
|
||||
assert!(incoming.is_closed() && durable.is_closed());
|
||||
drop(blocked);
|
||||
let runtime = tokio::time::timeout(StdDuration::from_secs(10), shutdown)
|
||||
.await
|
||||
.expect("same live worker retries after volume recovery")
|
||||
.expect("real worker joins");
|
||||
assert!(!runtime.dirty);
|
||||
assert_eq!(
|
||||
runtime.partial_writes.depth(),
|
||||
2,
|
||||
"prior committed and newly unacknowledged responsibilities both stay owned"
|
||||
);
|
||||
let saved = snapshot::inspect_local_committed_snapshot(runtime.config.journal_max_bytes)
|
||||
.await
|
||||
.expect("inspect recovered successor")
|
||||
.expect("recovered successor exists");
|
||||
assert_eq!(decode_journal(saved.payload()).0.len(), 2);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn owned_mrf_full_admission_replies_full_without_false_ack() {
|
||||
let _env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("mrf_owned_full")
|
||||
.build()
|
||||
.await;
|
||||
let mut runtime = empty_runtime(1);
|
||||
runtime.admit_best_effort(intent("mrf-owned-shutdown", "resident", 0));
|
||||
let (_incoming, mut receiver) = mpsc::channel(4);
|
||||
let (durable, mut durable_receiver) = mpsc::channel(4);
|
||||
let (_lifecycle, mut lifecycle_receiver) = mpsc::channel(4);
|
||||
let (reply, ack) = tokio::sync::oneshot::channel();
|
||||
durable
|
||||
.send(MrfDurableSubmission {
|
||||
intent: partial("beyond-resident-limit"),
|
||||
source_bucket_incarnation_id: Some(Uuid::new_v4()),
|
||||
response: reply,
|
||||
})
|
||||
.await
|
||||
.expect("queue unacknowledged submission");
|
||||
tokio::time::timeout(
|
||||
StdDuration::from_secs(10),
|
||||
finalize_mrf_consumer(&mut runtime, &mut receiver, &mut durable_receiver, &mut lifecycle_receiver),
|
||||
)
|
||||
.await
|
||||
.expect("resident checkpoint completes");
|
||||
assert!(matches!(
|
||||
ack.await.expect("Full explicitly returned"),
|
||||
Err(MrfDurableAdmissionError::Full)
|
||||
));
|
||||
assert_eq!(runtime.queue.depth(), 1);
|
||||
assert_eq!(runtime.partial_writes.depth(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
async fn owned_mrf_real_consumer_joins_with_live_static_senders() {
|
||||
const CHILD: &str = "RUSTFS_MRF_OWNED_CONSUMER_TEST_CHILD";
|
||||
const TEST: &str = "heal::mrf_queue::tests::owned_shutdown_tests::owned_mrf_real_consumer_joins_with_live_static_senders";
|
||||
if std::env::var_os(CHILD).is_none() {
|
||||
// Process-global senders cannot be reinitialized by another unit test.
|
||||
let output = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args(["--exact", TEST, "--nocapture", "--test-threads=1"])
|
||||
.env(CHILD, "1")
|
||||
.output()
|
||||
.expect("isolated MRF consumer test");
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
assert!(output.status.success(), "{stdout}{}", String::from_utf8_lossy(&output.stderr));
|
||||
assert!(stdout.contains("test result: ok. 1 passed; 0 failed; 0 ignored;"));
|
||||
assert!(stdout.contains("MRF_OWNED_CONSUMER_CHILD_OK"));
|
||||
return;
|
||||
}
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("mrf_owned_loop")
|
||||
.build()
|
||||
.await;
|
||||
let manager = Arc::new(HealManager::new_without_root_recovery_for_test(
|
||||
Arc::new(ECStoreHealStorage::new(env.ecstore.clone())),
|
||||
None,
|
||||
));
|
||||
manager.stop().await.expect("original manager handoff before MRF stop");
|
||||
let stop = tokio_util::sync::CancellationToken::new();
|
||||
let task = spawn_owned_mrf_consumer(manager, stop.clone()).expect("fresh process owns actual MRF startup");
|
||||
tokio::time::timeout(StdDuration::from_secs(10), list_legacy_responsibilities(None, 1))
|
||||
.await
|
||||
.expect("actual consumer must process a command before the stop probe")
|
||||
.expect("empty lifecycle query succeeds");
|
||||
assert!(!task.is_finished(), "the ready consumer is still running with static senders retained");
|
||||
stop.cancel();
|
||||
tokio::time::timeout(StdDuration::from_secs(10), task)
|
||||
.await
|
||||
.expect("explicit stop must terminate despite process-global sender lifetime")
|
||||
.expect("actual consumer joined");
|
||||
println!("MRF_OWNED_CONSUMER_CHILD_OK");
|
||||
}
|
||||
@@ -18,7 +18,6 @@
|
||||
use rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::VecDeque;
|
||||
#[cfg(test)]
|
||||
use uuid::Uuid;
|
||||
|
||||
const MAX_OUTCOME_ITEMS: usize = 128;
|
||||
@@ -64,6 +63,12 @@ pub struct HealObjectOutcome {
|
||||
pub identity: HealObjectIdentity,
|
||||
pub disposition: HealObjectDisposition,
|
||||
pub detail: Option<String>,
|
||||
/// Local quorum metadata selection; absent from wire snapshots.
|
||||
#[serde(skip)]
|
||||
pub quorum_selected_data_dir: Option<Uuid>,
|
||||
/// Local Deep check-path observation; absent from wire snapshots.
|
||||
#[serde(skip)]
|
||||
pub deep_check_observed: bool,
|
||||
}
|
||||
|
||||
impl HealObjectOutcome {
|
||||
@@ -442,6 +447,8 @@ mod canonical_outcome_tests {
|
||||
|
||||
fn item(disposition: HealObjectDisposition) -> HealObjectOutcome {
|
||||
HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket".to_string(),
|
||||
@@ -496,6 +503,8 @@ mod canonical_outcome_tests {
|
||||
..item(HealObjectDisposition::Unknown).identity
|
||||
};
|
||||
let mut receipt = HealObjectReceipt {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: expected.clone(),
|
||||
disposition: HealObjectDisposition::MetadataHealthy,
|
||||
};
|
||||
@@ -740,6 +749,8 @@ mod canonical_outcome_tests {
|
||||
..item(HealObjectDisposition::Unknown).identity
|
||||
};
|
||||
let mut receipt = HealObjectReceipt {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
identity: expected.clone(),
|
||||
disposition: HealObjectDisposition::Repaired,
|
||||
};
|
||||
|
||||
@@ -140,7 +140,23 @@ impl ReplacementExecution {
|
||||
/// A dropped page waiter must not release the lease of a mutation that is
|
||||
/// still executing. The owned worker keeps the lease through its I/O.
|
||||
pub(crate) async fn run<T: Send + 'static>(self: Arc<Self>, work: impl Future<Output = T> + Send + 'static) -> Result<T> {
|
||||
tokio::spawn(async move {
|
||||
self.run_with_owner(None, work).await
|
||||
}
|
||||
|
||||
pub(super) async fn run_owned<T: Send + 'static>(
|
||||
self: Arc<Self>,
|
||||
parent: &super::manager::HealWork,
|
||||
work: impl Future<Output = T> + Send + 'static,
|
||||
) -> Result<T> {
|
||||
self.run_with_owner(Some(parent), work).await
|
||||
}
|
||||
|
||||
async fn run_with_owner<T: Send + 'static>(
|
||||
self: Arc<Self>,
|
||||
parent: Option<&super::manager::HealWork>,
|
||||
work: impl Future<Output = T> + Send + 'static,
|
||||
) -> Result<T> {
|
||||
super::manager::spawn_heal_work(parent, move |_parent| async move {
|
||||
let _execution = self;
|
||||
work.await
|
||||
})
|
||||
|
||||
+845
-68
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user