Compare commits

...

11 Commits

Author SHA1 Message Date
overtrue de9009d882 ci: use the larger macOS release runner
Backport the reviewed runner change from 3a23122104 (#9260) after preview.1 reached the job timeout.
2026-11-16 02:02:29 +08:00
overtrue 94d54f8f55 fix(clippy): backport Rust 1.100 lint fixes
Backport the applicable changes from 0f904a170e (#9377). Preserve release multipart guards and existing scanner behavior, and omit tests for features absent from the release branch.
2026-11-15 21:27:43 +08:00
overtrue 01edeb0eab chore(release): prepare 1.0.6 2026-11-15 19:40:53 +08:00
cxymds dd2713687c fix(scanner): retire obsolete work without clearing responsibility 2026-11-14 16:14:54 +08:00
cxymds 4304e1b3ed test(scanner): preserve production drain for real checkpoints 2026-11-14 16:14:54 +08:00
overtrue 5b331fad73 fix(metrics): use iterator loops with Rust 1.100 2026-11-13 23:58:53 +08:00
overtrue 130a19de8f fix(admin): reuse S3 facade in MRF backport 2026-11-13 23:42:53 +08:00
overtrue 3e882ea0cf fix(heal): backport MRF management for 1.0.6 (#9399)
Backport PR #9399 at d57684d6e5 and adapt the scanner and admin interfaces to the 1.0.5 baseline.
2026-11-13 23:37:17 +08:00
overtrue 517ea37eb2 fix(scanner): backport usage convergence to 1.0.6 (#9397)
Backport PR #9397 at 222bdf83ee without main merge commits.
2026-11-13 23:09:12 +08:00
cxymds 6d0832b992 fix(heal): retain legacy MRF responsibility after physical repair (#9391)
(cherry picked from commit ce9d0b153e)
2026-11-13 23:08:37 +08:00
Chris An 34e6449ed9 revert(release): defer certification rollout for 1.0.5 (#9254)
Revert e5563e6004 only on release/1.0.5. Its trusted verifier workflow and immutable-candidate integration are not available yet. Restore the previous publication path under the approved release policy; retain the completed product regression and preview acceptance requirements.
2026-11-09 22:44:06 -05:00
289 changed files with 16514 additions and 7454 deletions
-77
View File
@@ -1,77 +0,0 @@
{
"schema_version": 1,
"plan_id": "rustfs-release-validation",
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
"release_types": ["preview", "prerelease", "stable"],
"targets": [
{"os": "linux", "arch": "x86_64", "abi": "musl"},
{"os": "linux", "arch": "aarch64", "abi": "musl"},
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
{"os": "macos", "arch": "aarch64", "abi": "none"},
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
],
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
"features": ["pyroscope"],
"trusted_workflows": [
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
],
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
"lanes": [
{
"lane": "identity",
"harness_repository": "rustfs/s3chaos",
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
"timeout_seconds": 1800,
"environment_capabilities": ["process-exec"],
"cases": ["artifact-identity", "startup"],
"applies_to": {
"release_types": ["preview", "prerelease", "stable"],
"platforms": "all"
},
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
},
{
"lane": "upgrade",
"harness_repository": "rustfs/s3chaos",
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
"timeout_seconds": 5400,
"environment_capabilities": ["process-exec", "previous-release-artifact"],
"cases": [
"direct-upgrade",
"mixed-version-upgrade",
"bucket-config-upgrade",
"bucket-config-rollback",
"multipart-layout-upgrade",
"multipart-layout-baseline"
],
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
"applies_to": {
"release_types": ["prerelease", "stable"],
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
},
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
}
],
"upgrade_sources": {
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"sources": [
{
"version": "1.0.1",
"role": "N-1",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
},
{
"version": "1.0.0-rc.5",
"role": "N-2",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
}
],
"rollback_supported": false,
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
}
}
+6 -312
View File
@@ -193,7 +193,7 @@ jobs:
{"target_id":"linux-aarch64-musl","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
{"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
{"target_id":"linux-aarch64-gnu","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
{"target_id":"macos-aarch64","os":"macos-latest-xlarge","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":"","glibc":""}
]}'
@@ -814,18 +814,13 @@ jobs:
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
# Determine upload path based on build type. Release and prerelease
# builds only stage their bytes here: the official
# artifacts/rustfs/release/ path is written solely by the
# certification-gated promote-release-artifacts job, from these same
# staged objects. Development builds keep the uncertified dev channel
# and never touch official or latest channels.
# Determine upload path based on build type
if [[ "$BUILD_TYPE" == "development" ]]; then
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
echo "📤 Uploading development build to R2 dev directory"
else
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
echo "📤 Staging release build for certification-gated promotion"
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
echo "📤 Uploading release build to R2 release directory"
fi
# Upload all rustfs zip files to R2 using S3-compatible API
@@ -874,221 +869,6 @@ jobs:
echo "This run does not trigger Docker publishing."
fi
# Aggregate the parent candidate batch. This job always collects results and
# retains its evidence, including on failure, and every publication job
# below requires its success. Missing, stale, substituted or non-PASS
# evidence fails closed here: only the trusted verification sources
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
# it, so a PR-uploaded JSON set never gains release authority.
release-certification:
name: Release Certification
needs: [ build-check, build-rustfs ]
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download all build artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./artifacts
pattern: rustfs-*
merge-multiple: true
- name: Write candidate and collect trusted validation results
env:
# Cross-repo read of the trusted verification run. Falls back to the
# workflow token, which cannot read rustfs/s3chaos; a missing token
# therefore fail-closes instead of certifying.
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
PRODUCT_COMMIT: ${{ github.sha }}
VERSION: ${{ needs.build-check.outputs.version }}
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
shell: bash
run: |
set -euo pipefail
mkdir -p results release-gate-output
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
import hashlib, json, os, sys, zipfile
plan_sha256 = sys.argv[1]
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
if profile is None:
raise SystemExit("release certification only covers preview, prerelease and stable builds")
def sha256(path):
digest = hashlib.sha256()
with open(path, "rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
deliverables = []
for name in sorted(os.listdir("artifacts")):
if not name.startswith("rustfs-") or not name.endswith(".zip"):
continue
path = os.path.join("artifacts", name)
members = zipfile.ZipFile(path).namelist()
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
with zipfile.ZipFile(path) as archive:
binary_bytes = archive.read(binary_name)
import tempfile
with tempfile.NamedTemporaryFile() as handle:
handle.write(binary_bytes)
handle.flush()
binary_sha = sha256(handle.name)
deliverables.append({
"kind": "zip",
"name": name,
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
"sha256": sha256(path),
"size": os.path.getsize(path),
"binary_sha256": binary_sha,
})
if not deliverables:
raise SystemExit("no built deliverables found to certify")
candidate = {
"schema_version": 1,
"product_commit": os.environ["PRODUCT_COMMIT"],
"version": os.environ["VERSION"],
"release_profile": profile,
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
"plan_sha256": plan_sha256,
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
"deliverables": deliverables,
}
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
print()
PY
# Trusted verification evidence only: the release-gate envelopes of
# rustfs/s3chaos for this exact product commit. Absence blocks below.
TRUSTED_REPO="rustfs/s3chaos"
TRUSTED_WORKFLOW="release-gate.yml"
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
if [[ -z "$TRUSTED_RUN" ]]; then
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
else
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
fi
python3 scripts/release/release_gate.py validate \
--plan .config/release-validation-plan.json \
--candidate candidate.json \
--results results \
--output release-gate-output
- name: Upload certification evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: release-certification
path: |
candidate.json
results/
release-gate-output/
if-no-files-found: warn
retention-days: 30
# The only writer of the official R2 release path. Promotion re-checks the
# exact bytes about to be published against the passed certification list
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
promote-release-artifacts:
name: Promote Release Artifacts
needs: [ build-check, build-rustfs, release-certification ]
# Target-channel lock for the official R2 release path.
concurrency:
group: release-channel-r2-release
cancel-in-progress: false
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
env:
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
AWS_EC2_METADATA_DISABLED: true
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download build and certification artifacts
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./artifacts
pattern: rustfs-*
merge-multiple: true
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify and promote staged release artifacts
shell: bash
run: |
set -euo pipefail
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
exit 1
fi
for zip_file in artifacts/rustfs-*.zip; do
[[ -f "$zip_file" ]] || continue
name="$(basename "$zip_file")"
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$zip_file" \
--artifact-name "$name"
done
if ! command -v aws >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y awscli
fi
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
for zip_file in artifacts/rustfs-*.zip; do
[[ -f "$zip_file" ]] || continue
name="$(basename "$zip_file")"
staged="$(mktemp)"
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
if [[ "$staged_sha" != "$built_sha" ]]; then
echo "❌ Staged object ${name} does not match the verified bytes" >&2
exit 1
fi
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
rm -f "$staged"
echo "✅ Promoted ${name} (${built_sha})"
done
# Create GitHub Release for every valid release tag, including previews
create-release:
name: Create GitHub Release
@@ -1248,10 +1028,6 @@ jobs:
update-latest-version:
name: Update Latest Version
needs: [ build-check, publish-release ]
# Target-channel lock for the latest.json version pointer.
concurrency:
group: release-channel-oss-latest
cancel-in-progress: false
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest
timeout-minutes: 30
@@ -1292,45 +1068,6 @@ jobs:
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
chmod +x "$OSSUTIL_BIN"
# Candidate/pointer ordering: an older candidate must never overwrite
# a newer pointer, even if its promotion job finishes later.
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
import json
import re
import sys
def key(version):
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
if not match:
raise SystemExit(3)
core = tuple(int(part) for part in match.groups()[:3])
prerelease = match.group(4)
if prerelease is None:
return core + (1,)
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
try:
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
proposed = sys.argv[2]
if current and key(current) > key(proposed):
raise SystemExit(1)
except SystemExit:
raise
except (OSError, ValueError, TypeError):
raise SystemExit(3)
PY
guard_status=$?
if [[ "$guard_status" -eq 1 ]]; then
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
exit 0
elif [[ "$guard_status" -ne 0 ]]; then
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
exit 1
fi
fi
# Create latest.json
cat > latest.json << EOF
{
@@ -1347,59 +1084,16 @@ jobs:
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
# Publish release (remove draft status). Requires the release-certification
# aggregate: without a complete legal success there is no public write.
# Publish release (remove draft status)
publish-release:
name: Publish Release
needs: [ build-check, create-release, upload-release-assets, release-certification ]
# Target-channel lock: concurrent releases serialize on this channel so an
# older candidate cannot interleave its publish with a newer one.
concurrency:
group: release-channel-github-release
cancel-in-progress: false
needs: [ build-check, create-release, upload-release-assets ]
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: write
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify release assets against certification
env:
GH_TOKEN: ${{ github.token }}
shell: bash
run: |
set -euo pipefail
TAG="${{ needs.build-check.outputs.version }}"
mkdir -p "$RUNNER_TEMP/promotion-assets"
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
found=0
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
[[ -f "$zip_file" ]] || continue
found=1
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$zip_file" \
--artifact-name "$(basename "$zip_file")"
done
if [[ "$found" -ne 1 ]]; then
echo "No release zip assets to verify against the certification" >&2
exit 1
fi
- name: Publish release
env:
GH_TOKEN: ${{ github.token }}
+2 -182
View File
@@ -329,75 +329,13 @@ jobs:
echo " - Create latest: $create_latest"
echo " - Source ref: $source_ref"
# Consume the trusted release certification for the exact version this
# workflow wraps and verify the certified input digests. A manual dispatch
# without that certification fails here and pushes no official tag; only
# results from the trusted verification sources carry release authority.
release-certification:
name: Release Certification
needs: build-check
if: needs.build-check.outputs.should_build == 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
ref: ${{ needs.build-check.outputs.source_ref }}
- name: Download and verify release certification
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
if [[ -z "$run_id" ]]; then
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
exit 1
fi
if ! gh run download "$run_id" -n release-certification -D certification; then
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
# Input digest binding: every zip this image wraps must be the
# certified byte-for-byte content, never a mutable latest pointer.
while IFS=$'\t' read -r name location expected; do
echo "Verifying certified input ${name}"
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
if [[ "$actual" != "$expected" ]]; then
echo "Certified input ${name} does not match its recorded digest" >&2
exit 1
fi
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
# Build multi-arch Docker images
# Strategy: Build images using pre-built binaries from dl.rustfs.com
# Supports both release and dev channel binaries based on build context
# Only runs when should_build is true (which includes workflow success check)
build-docker:
name: Build Docker Images
needs: [ build-check, release-certification ]
needs: build-check
if: needs.build-check.outputs.should_build == 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
@@ -505,14 +443,6 @@ jobs:
# Output tags
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
# Staging identity for this run's promotion: official tags are only
# created after the staged image verifies against the certification,
# and they point at this same digest (no rebuild-after-verify).
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
# Generate labels
LABELS="org.opencontainers.image.title=RustFS"
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
@@ -537,9 +467,7 @@ jobs:
file: ${{ matrix.file }}
platforms: ${{ env.DOCKER_PLATFORMS }}
push: ${{ needs.build-check.outputs.should_push == 'true' }}
# Push to staging tags only: official tags are created by the
# certification-gated promotion step below from this same digest.
tags: ${{ steps.meta.outputs.staging_tags }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
# No layer cache. This build compiles nothing — it downloads a
# release zip and runs apk/apt — so the cache could only save the
@@ -565,114 +493,6 @@ jobs:
no-cache: false
pull: true
# A later-built wrapper: the staged image must complete this platform's
# own binary identity verification before any official tag is written.
# It may not inherit an unverified package's conclusion.
- name: Verify staged image against certification
if: needs.build-check.outputs.should_push == 'true'
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
gh run download "$run_id" -n release-certification -D certification
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
case "${{ matrix.variant }}" in
musl) libc="musl" ;;
glibc) libc="gnu" ;;
esac
for platform in linux/amd64 linux/arm64; do
case "$platform" in
linux/amd64) arch_sub="x86_64" ;;
linux/arm64) arch_sub="aarch64" ;;
esac
docker pull --platform "$platform" "$STAGING_REF"
container="$(docker create --platform "$platform" "$STAGING_REF")"
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
docker rm "$container" >/dev/null
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
certification/candidate.json)"
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/staged-rustfs" \
--artifact-role binary
done
- name: Promote staged image to official tags
if: needs.build-check.outputs.should_push == 'true'
shell: bash
run: |
set -euo pipefail
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
for tag in "${official_tags[@]}"; do
docker buildx imagetools create -t "$tag" "$STAGING_REF"
echo "Promoted $tag -> $STAGING_REF"
done
# Downstream consumers (Helm) verify this record instead of inheriting
# an unverified wrapper's conclusion.
- name: Record image promotion
if: needs.build-check.outputs.should_push == 'true'
shell: bash
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
VERSION: ${{ needs.build-check.outputs.version }}
run: |
set -euo pipefail
mkdir -p certification
run_id="$TRIGGERING_RUN_ID"
if [[ -z "$run_id" ]]; then
run_id="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$VERSION" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
gh run download "$run_id" -n release-certification -D certification
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
jq -n \
--arg version "$VERSION" \
--arg digest "$digest" \
--arg staging_ref "$STAGING_REF" \
--arg candidate_sha256 "$candidate_sha" \
--arg tags "${{ steps.meta.outputs.tags }}" \
--slurpfile candidate certification/candidate.json \
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
> image-promotion-record.json
- name: Upload image promotion record
if: needs.build-check.outputs.should_push == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: image-promotion-record${{ matrix.suffix }}
path: image-promotion-record.json
if-no-files-found: error
retention-days: 30
# Note: Manifest creation is no longer needed as we only build one variant
# Multi-arch manifests are automatically created by docker/build-push-action
-93
View File
@@ -30,102 +30,9 @@ permissions:
contents: read
jobs:
# Consume the same trusted release certification as every other publication
# exit and verify the image promotion record of the Docker workflow. The
# chart wraps no binary of its own, so it may only reference an image whose
# platform verification already passed; it must not inherit an unverified
# package's conclusion. A run without that evidence fails closed here and
# never reaches the chart repository.
release-certification:
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
actions: read
outputs:
version: ${{ steps.version.outputs.raw_tag }}
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Normalize release version
id: version
env:
RAW_INPUT: ${{ github.event.inputs.version }}
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
run: |
set -euo pipefail
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
RAW="$RAW_INPUT"
else
RAW="$RAW_BRANCH"
fi
case "$RAW" in
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
*) RAW_TAG="$RAW" ;;
esac
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
- name: Download and verify release certification and image record
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification image-record
build_run="$TRIGGERING_RUN_ID"
if [[ -z "$build_run" ]]; then
build_run="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
fi
if [[ -z "$build_run" ]]; then
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
exit 1
fi
if ! gh run download "$build_run" -n release-certification -D certification; then
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
docker_run="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
--jq '.workflow_runs[0].id // empty' || true)"
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
exit 1
fi
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
if [[ "$record_sha" != "$candidate_sha" ]]; then
echo "Image promotion record does not belong to this certification" >&2
exit 1
fi
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
image-record/image-promotion-record.json)"
if [[ "$overlap" -ne 0 ]]; then
echo "Image promotion record carries binary digests outside the certification" >&2
exit 1
fi
build-helm-package:
runs-on: ubuntu-latest
timeout-minutes: 30
needs: [ release-certification ]
if: |
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
(
+14 -210
View File
@@ -225,57 +225,10 @@ jobs:
echo " Build run ID: $BUILD_RUN_ID"
echo " Build run number: $RUN_NUMBER"
# Consume the trusted release certification of the resolved build run and
# verify the certified input digests before any package is wrapped. A run
# without that certification fails closed here; packages built later must
# complete their own verification and cannot inherit an unverified
# conclusion.
release-certification:
name: Release Certification
needs: resolve
if: needs.resolve.result == 'success'
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download and verify release certification
env:
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
shell: bash
run: |
set -euo pipefail
mkdir -p certification
if [[ "$BUILD_TYPE" == "development" ]]; then
# Development artifacts are explicitly uncertified and never touch
# official or latest channels; only the staging/dev paths remain.
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
> certification/development-uncertified.json
exit 0
fi
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
exit 1
fi
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
# Build DEB and RPM packages for each architecture and libc variant
package:
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
needs: [ resolve, release-certification ]
needs: resolve
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
strategy:
@@ -541,55 +494,6 @@ jobs:
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
echo "✅ RPM built: $RPM_FILE"
- name: Download certification evidence
if: needs.resolve.outputs.build_type != 'development'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
# Later-built wrapper verification: the binary inside each package must
# match the certified binary digest for this platform before anything is
# uploaded, and the package itself stays on staging until promotion.
- name: Verify wrapped binary against certification
if: needs.resolve.outputs.build_type != 'development'
shell: bash
env:
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
ARTIFACT_NAME: ${{ matrix.artifact_name }}
run: |
set -euo pipefail
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
certification/candidate.json)"
if [[ -z "$expected" ]]; then
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
exit 1
fi
if ! command -v rpm2cpio >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y rpm2cpio
fi
if ! command -v cpio >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y cpio
fi
for package_file in "$DEB_FILE" "$RPM_FILE"; do
case "$package_file" in
*.deb)
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
*.rpm)
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
esac
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
--artifact-role binary
done
- name: Upload packages to artifacts
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
@@ -627,14 +531,10 @@ jobs:
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
# Build-phase writes stay on staging for release types: the official
# packages/release and packages/latest paths are written only by the
# certification-gated promote-packages job, from these same staged
# objects. Development packages keep the uncertified dev channel.
if [[ "$BUILD_TYPE" == "development" ]]; then
R2_PREFIX="artifacts/rustfs/packages/dev"
else
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
R2_PREFIX="artifacts/rustfs/packages/release"
fi
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
@@ -649,6 +549,18 @@ jobs:
echo "✅ Upload complete"
# Also upload as latest for release/prerelease
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
for f in "$DEB_FILE" "$RPM_FILE"; do
if [[ -n "$f" && -f "$f" ]]; then
echo "Uploading latest: $(basename "$f")"
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
fi
done
echo "✅ Latest packages updated"
fi
- name: Upload packages to GitHub Release
if: needs.resolve.outputs.tag != ''
env:
@@ -702,114 +614,6 @@ jobs:
echo "✅ GitHub release assets updated"
# The only writer of the official packages/release and packages/latest R2
# paths. Promotion re-checks each package's wrapped binary against the
# passed certification list and copies the staged objects byte-for-byte:
# no rebuild-after-verify, no mutable latest resolution downstream.
promote-packages:
name: Promote Packages
needs: [ resolve, package, release-certification ]
# Target-channel lock for the official package paths.
concurrency:
group: release-channel-r2-packages
cancel-in-progress: false
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
timeout-minutes: 30
permissions:
contents: read
actions: read
env:
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
R2_BUCKET: ${{ secrets.R2_BUCKET }}
AWS_EC2_METADATA_DISABLED: true
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download packages and certification
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ./packages
pattern: packages-*
merge-multiple: true
- name: Download certification evidence
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: release-certification
path: ./certification
- name: Verify and promote staged packages
shell: bash
run: |
set -euo pipefail
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
exit 1
fi
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
if ! command -v aws >/dev/null 2>&1; then
sudo apt-get update && sudo apt-get install -y awscli
fi
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
export AWS_DEFAULT_REGION="auto"
if ! command -v rpm2cpio >/dev/null 2>&1; then
sudo apt-get install -y rpm2cpio
fi
if ! command -v cpio >/dev/null 2>&1; then
sudo apt-get install -y cpio
fi
promoted=0
for package_file in packages/*.deb packages/*.rpm; do
[[ -f "$package_file" ]] || continue
name="$(basename "$package_file")"
case "$package_file" in
*.deb)
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
*.rpm)
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
;;
esac
python3 scripts/release/release_gate.py verify-promotion \
--plan .config/release-validation-plan.json \
--candidate certification/candidate.json \
--summary certification/release-gate-output \
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
--artifact-role binary
staged="$(mktemp)"
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
if [[ "$staged_sha" != "$built_sha" ]]; then
echo "❌ Staged package ${name} does not match the verified bytes" >&2
exit 1
fi
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
rm -f "$staged"
promoted=$((promoted + 1))
echo "✅ Promoted ${name} (${built_sha})"
done
if [[ "$promoted" -eq 0 ]]; then
echo "No staged packages found to promote" >&2
exit 1
fi
# Summary
summary:
name: Summary
Generated
+53 -52
View File
@@ -4136,7 +4136,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
[[package]]
name = "e2e_test"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"anyhow",
"async-compression",
@@ -9520,7 +9520,7 @@ dependencies = [
[[package]]
name = "rustfs"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"aes-gcm",
"anyhow",
@@ -9680,7 +9680,7 @@ dependencies = [
[[package]]
name = "rustfs-audit"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"const-str",
"futures",
@@ -9702,7 +9702,7 @@ dependencies = [
[[package]]
name = "rustfs-checksums"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"base64-simd",
"bytes",
@@ -9718,7 +9718,7 @@ dependencies = [
[[package]]
name = "rustfs-common"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"arc-swap",
"hotpath",
@@ -9733,7 +9733,7 @@ dependencies = [
[[package]]
name = "rustfs-concurrency"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"hotpath",
"insta",
@@ -9746,7 +9746,7 @@ dependencies = [
[[package]]
name = "rustfs-config"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"const-str",
"hotpath",
@@ -9756,7 +9756,7 @@ dependencies = [
[[package]]
name = "rustfs-connect-symbol-catalog"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"hex-simd",
"object 0.40.0",
@@ -9768,7 +9768,7 @@ dependencies = [
[[package]]
name = "rustfs-credentials"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"base64-simd",
"hmac 0.13.0",
@@ -9782,7 +9782,7 @@ dependencies = [
[[package]]
name = "rustfs-crypto"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"aes-gcm",
"argon2",
@@ -9803,17 +9803,18 @@ dependencies = [
[[package]]
name = "rustfs-data-usage"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"hotpath",
"rmp-serde",
"rustfs-filemeta",
"serde",
"serde_json",
]
[[package]]
name = "rustfs-ecstore"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"arc-swap",
"async-channel",
@@ -9951,7 +9952,7 @@ dependencies = [
[[package]]
name = "rustfs-extension-schema"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"hotpath",
"serde",
@@ -9961,7 +9962,7 @@ dependencies = [
[[package]]
name = "rustfs-filemeta"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"arc-swap",
"base64-simd",
@@ -9990,7 +9991,7 @@ dependencies = [
[[package]]
name = "rustfs-heal"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"async-trait",
"base64-simd",
@@ -10031,7 +10032,7 @@ dependencies = [
[[package]]
name = "rustfs-heal-contracts"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"serde",
"serde_json",
@@ -10041,7 +10042,7 @@ dependencies = [
[[package]]
name = "rustfs-iam"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"arc-swap",
"async-trait",
@@ -10091,7 +10092,7 @@ dependencies = [
[[package]]
name = "rustfs-io-core"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"bytes",
"hotpath",
@@ -10104,7 +10105,7 @@ dependencies = [
[[package]]
name = "rustfs-io-metrics"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"criterion",
"hotpath",
@@ -10168,7 +10169,7 @@ dependencies = [
[[package]]
name = "rustfs-keystone"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"bytes",
"futures",
@@ -10194,7 +10195,7 @@ dependencies = [
[[package]]
name = "rustfs-kms"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"aes-gcm",
"anyhow",
@@ -10244,14 +10245,14 @@ dependencies = [
[[package]]
name = "rustfs-license"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "rustfs-lifecycle"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"async-trait",
"hotpath",
@@ -10273,7 +10274,7 @@ dependencies = [
[[package]]
name = "rustfs-lock"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"async-trait",
"compact_str",
@@ -10297,7 +10298,7 @@ dependencies = [
[[package]]
name = "rustfs-log-analyzer"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"chrono",
"flate2",
@@ -10316,7 +10317,7 @@ dependencies = [
[[package]]
name = "rustfs-madmin"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"hotpath",
"http 1.5.0",
@@ -10354,7 +10355,7 @@ dependencies = [
[[package]]
name = "rustfs-notify"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"arc-swap",
"async-trait",
@@ -10389,7 +10390,7 @@ dependencies = [
[[package]]
name = "rustfs-object-capacity"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"criterion",
"futures",
@@ -10408,7 +10409,7 @@ dependencies = [
[[package]]
name = "rustfs-object-data-cache"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"bytes",
"criterion",
@@ -10425,7 +10426,7 @@ dependencies = [
[[package]]
name = "rustfs-obs"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"chrono",
"crossbeam-channel",
@@ -10488,7 +10489,7 @@ dependencies = [
[[package]]
name = "rustfs-policy"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"async-trait",
"base64-simd",
@@ -10519,7 +10520,7 @@ dependencies = [
[[package]]
name = "rustfs-protocols"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"async-compression",
"async-trait",
@@ -10582,7 +10583,7 @@ dependencies = [
[[package]]
name = "rustfs-protos"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"flatbuffers",
"hotpath",
@@ -10607,7 +10608,7 @@ dependencies = [
[[package]]
name = "rustfs-replication"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"byteorder",
"bytes",
@@ -10625,7 +10626,7 @@ dependencies = [
[[package]]
name = "rustfs-rio"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"aes-gcm",
"arc-swap",
@@ -10671,7 +10672,7 @@ dependencies = [
[[package]]
name = "rustfs-rio-v2"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"aes-gcm",
"bytes",
@@ -10694,7 +10695,7 @@ dependencies = [
[[package]]
name = "rustfs-s3-client"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"base64-simd",
"bytes",
@@ -10738,7 +10739,7 @@ dependencies = [
[[package]]
name = "rustfs-s3-ops"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"hotpath",
"rustfs-s3-types",
@@ -10746,7 +10747,7 @@ dependencies = [
[[package]]
name = "rustfs-s3-types"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"hotpath",
"serde",
@@ -10755,7 +10756,7 @@ dependencies = [
[[package]]
name = "rustfs-s3select-api"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"arc-swap",
"async-compression",
@@ -10796,7 +10797,7 @@ dependencies = [
[[package]]
name = "rustfs-s3select-query"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"async-trait",
"bytes",
@@ -10818,7 +10819,7 @@ dependencies = [
[[package]]
name = "rustfs-scanner"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"async-trait",
"bytes",
@@ -10869,7 +10870,7 @@ dependencies = [
[[package]]
name = "rustfs-scanner-metrics"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"chrono",
"jiff",
@@ -10884,7 +10885,7 @@ dependencies = [
[[package]]
name = "rustfs-security-governance"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"hotpath",
"thiserror 2.0.21",
@@ -10892,7 +10893,7 @@ dependencies = [
[[package]]
name = "rustfs-signer"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"base64-simd",
"bytes",
@@ -10911,7 +10912,7 @@ dependencies = [
[[package]]
name = "rustfs-storage-api"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"async-trait",
"hotpath",
@@ -10926,7 +10927,7 @@ dependencies = [
[[package]]
name = "rustfs-targets"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"arc-swap",
"async-nats",
@@ -10980,7 +10981,7 @@ dependencies = [
[[package]]
name = "rustfs-test-utils"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"hotpath",
"rustfs-data-usage",
@@ -10996,7 +10997,7 @@ dependencies = [
[[package]]
name = "rustfs-tls-runtime"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"arc-swap",
"hotpath",
@@ -11035,7 +11036,7 @@ dependencies = [
[[package]]
name = "rustfs-trusted-proxies"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"async-trait",
"axum",
@@ -11072,7 +11073,7 @@ dependencies = [
[[package]]
name = "rustfs-utils"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"aws-lc-rs",
"base64-simd",
@@ -11115,7 +11116,7 @@ dependencies = [
[[package]]
name = "rustfs-zip"
version = "1.0.5"
version = "1.0.6"
dependencies = [
"async-compression",
"futures",
+51 -51
View File
@@ -75,7 +75,7 @@ edition = "2024"
license = "Apache-2.0"
repository = "https://github.com/rustfs/rustfs"
rust-version = "1.99.0"
version = "1.0.5"
version = "1.0.6"
homepage = "https://rustfs.com"
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
@@ -92,56 +92,56 @@ redundant_clone = "warn"
[workspace.dependencies]
# RustFS Internal Crates
rustfs = { path = "./rustfs", version = "1.0.5" }
rustfs-heal = { path = "crates/heal", version = "1.0.5" }
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.5" }
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.5" }
rustfs-audit = { path = "crates/audit", version = "1.0.5" }
rustfs-checksums = { path = "crates/checksums", version = "1.0.5" }
rustfs-common = { path = "crates/common", version = "1.0.5" }
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.5" }
rustfs-config = { path = "./crates/config", version = "1.0.5" }
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.5" }
rustfs-credentials = { path = "crates/credentials", version = "1.0.5" }
rustfs-crypto = { path = "crates/crypto", version = "1.0.5" }
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.5" }
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.5" }
rustfs-iam = { path = "crates/iam", version = "1.0.5" }
rustfs-keystone = { path = "crates/keystone", version = "1.0.5" }
rustfs-license = { path = "crates/license", version = "1.0.5" }
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.5" }
rustfs-kms = { path = "crates/kms", version = "1.0.5" }
rustfs-lock = { path = "crates/lock", version = "1.0.5" }
rustfs-madmin = { path = "crates/madmin", version = "1.0.5" }
rustfs-notify = { path = "crates/notify", version = "1.0.5" }
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.5" }
rustfs-io-core = { path = "crates/io-core", version = "1.0.5" }
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.5" }
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.5", default-features = false }
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.5" }
rustfs-obs = { path = "crates/obs", version = "1.0.5" }
rustfs-policy = { path = "crates/policy", version = "1.0.5" }
rustfs-protos = { path = "crates/protos", version = "1.0.5" }
rustfs-protocols = { path = "crates/protocols", version = "1.0.5" }
rustfs-replication = { path = "crates/replication", version = "1.0.5" }
rustfs-rio = { path = "crates/rio", version = "1.0.5" }
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.5" }
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.5" }
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.5" }
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.5" }
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.5" }
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.5" }
rustfs-scanner = { path = "crates/scanner", version = "1.0.5" }
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.5" }
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.5" }
rustfs-signer = { path = "crates/signer", version = "1.0.5" }
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.5" }
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.5" }
rustfs-targets = { path = "crates/targets", version = "1.0.5" }
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.5" }
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.5" }
rustfs-utils = { path = "crates/utils", version = "1.0.5" }
rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
rustfs = { path = "./rustfs", version = "1.0.6" }
rustfs-heal = { path = "crates/heal", version = "1.0.6" }
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.6" }
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.6" }
rustfs-audit = { path = "crates/audit", version = "1.0.6" }
rustfs-checksums = { path = "crates/checksums", version = "1.0.6" }
rustfs-common = { path = "crates/common", version = "1.0.6" }
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.6" }
rustfs-config = { path = "./crates/config", version = "1.0.6" }
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.6" }
rustfs-credentials = { path = "crates/credentials", version = "1.0.6" }
rustfs-crypto = { path = "crates/crypto", version = "1.0.6" }
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.6" }
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.6" }
rustfs-iam = { path = "crates/iam", version = "1.0.6" }
rustfs-keystone = { path = "crates/keystone", version = "1.0.6" }
rustfs-license = { path = "crates/license", version = "1.0.6" }
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.6" }
rustfs-kms = { path = "crates/kms", version = "1.0.6" }
rustfs-lock = { path = "crates/lock", version = "1.0.6" }
rustfs-madmin = { path = "crates/madmin", version = "1.0.6" }
rustfs-notify = { path = "crates/notify", version = "1.0.6" }
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.6" }
rustfs-io-core = { path = "crates/io-core", version = "1.0.6" }
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.6" }
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.6", default-features = false }
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.6" }
rustfs-obs = { path = "crates/obs", version = "1.0.6" }
rustfs-policy = { path = "crates/policy", version = "1.0.6" }
rustfs-protos = { path = "crates/protos", version = "1.0.6" }
rustfs-protocols = { path = "crates/protocols", version = "1.0.6" }
rustfs-replication = { path = "crates/replication", version = "1.0.6" }
rustfs-rio = { path = "crates/rio", version = "1.0.6" }
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.6" }
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.6" }
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.6" }
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.6" }
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.6" }
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.6" }
rustfs-scanner = { path = "crates/scanner", version = "1.0.6" }
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.6" }
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.6" }
rustfs-signer = { path = "crates/signer", version = "1.0.6" }
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.6" }
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.6" }
rustfs-targets = { path = "crates/targets", version = "1.0.6" }
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.6" }
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.6" }
rustfs-utils = { path = "crates/utils", version = "1.0.6" }
rustfs-zip = { path = "./crates/zip", version = "1.0.6" }
# Async Runtime and Networking
async-channel = "2.5.0"
+11
View File
@@ -121,6 +121,11 @@ when restoring `/rustfs/console` for a custom OEM build.
- `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` (canonical)
- `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
- `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS` (canonical)
- `RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS` (canonical)
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES` (canonical)
- `RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES` (canonical)
- `RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES` (canonical)
- `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` (canonical)
- `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` (canonical)
- `RUSTFS_SCANNER_WALKERS_PER_DISK` (canonical)
@@ -130,6 +135,12 @@ Scanner cycle budget controls:
- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
Scanner bucket quantum controls:
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS`, `RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS`, `RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES`, `RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES`, and `RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES` cap one bucket admission. Their persistent keys are `scanner.bucket_quantum_duration`, `scanner.bucket_quantum_objects`, `scanner.bucket_quantum_directories`, `scanner.bucket_quantum_entries`, and `scanner.bucket_quantum_name_bytes`; duration is in whole seconds and the remaining keys are work counts or bytes.
- All five defaults are `0` for controlled rollout. A positive value enables that dimension; `0` disables that dimension, and all five zero values disable the bucket quantum. Parent cycle limits remain in force. Invalid unsigned integers and durations outside the platform timer range are rejected; environment overrides take precedence but cannot hide invalid persisted values. Admin scanner status reports each effective value and source under `runtime_config.bucket_quantum_*`, using `bucket_quantum_duration_seconds` for duration.
- Disabled defaults are an initial rollout state, not completion of usage-convergence acceptance. Production defaults require matched-topology A/B measurements and the remote progress/fairness checks in [Scanner runtime controls](../../docs/operations/scanner-runtime-controls.md#bucket-quantum-rollout-and-acceptance). These controls do not certify partial usage or acknowledge pending dirty work.
Scanner recovery retry controls:
- `RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS` caps the wait between automatic startup recovery retries. The persistent admin key is `scanner.recovery_backoff_cap`; the environment takes precedence over the persistent value, and the unset default remains 600 seconds (10 minutes). Values must be positive integer seconds within the platform timer range; `0`, malformed values, and overflow are rejected. A valid environment override does not hide an invalid persistent value from `admin config set` validation.
+50
View File
@@ -44,6 +44,21 @@ pub const SCANNER_CYCLE_MAX_OBJECTS: &str = "cycle_max_objects";
/// Scanner config key capping directories entered by one cycle.
pub const SCANNER_CYCLE_MAX_DIRECTORIES: &str = "cycle_max_directories";
/// Per-bucket admission limit in seconds; zero disables this dimension.
pub const SCANNER_BUCKET_QUANTUM_DURATION: &str = "bucket_quantum_duration";
/// Per-bucket admission limit in objects; zero disables this dimension.
pub const SCANNER_BUCKET_QUANTUM_OBJECTS: &str = "bucket_quantum_objects";
/// Per-bucket admission limit in directories; zero disables this dimension.
pub const SCANNER_BUCKET_QUANTUM_DIRECTORIES: &str = "bucket_quantum_directories";
/// Per-bucket admission limit in raw entries; zero disables this dimension.
pub const SCANNER_BUCKET_QUANTUM_ENTRIES: &str = "bucket_quantum_entries";
/// Per-bucket admission limit in raw entry-name bytes; zero disables this dimension.
pub const SCANNER_BUCKET_QUANTUM_NAME_BYTES: &str = "bucket_quantum_name_bytes";
/// Scanner config key setting the periodic bitrot scan cycle in seconds.
pub const SCANNER_BITROT_CYCLE: &str = "bitrot_cycle";
@@ -93,6 +108,11 @@ pub const SCANNER_KEYS: &[&str] = &[
SCANNER_CYCLE_MAX_DURATION,
SCANNER_CYCLE_MAX_OBJECTS,
SCANNER_CYCLE_MAX_DIRECTORIES,
SCANNER_BUCKET_QUANTUM_DURATION,
SCANNER_BUCKET_QUANTUM_OBJECTS,
SCANNER_BUCKET_QUANTUM_DIRECTORIES,
SCANNER_BUCKET_QUANTUM_ENTRIES,
SCANNER_BUCKET_QUANTUM_NAME_BYTES,
SCANNER_BITROT_CYCLE,
SCANNER_IDLE_MODE,
SCANNER_CACHE_SAVE_TIMEOUT,
@@ -141,6 +161,36 @@ pub const ENV_SCANNER_CYCLE_MAX_OBJECTS: &str = "RUSTFS_SCANNER_CYCLE_MAX_OBJECT
/// - Example: `export RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES=100000`
pub const ENV_SCANNER_CYCLE_MAX_DIRECTORIES: &str = "RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES";
/// Per-bucket admission limit in seconds; zero disables this dimension.
pub const ENV_SCANNER_BUCKET_QUANTUM_DURATION_SECS: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS";
/// Disabled until controlled-fleet acceptance selects production limits.
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_DURATION_SECS: u64 = 0;
/// Per-bucket admission limit in objects; zero disables this dimension.
pub const ENV_SCANNER_BUCKET_QUANTUM_OBJECTS: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS";
/// Disabled until controlled-fleet acceptance selects production limits.
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_OBJECTS: u64 = 0;
/// Per-bucket admission limit in directories; zero disables this dimension.
pub const ENV_SCANNER_BUCKET_QUANTUM_DIRECTORIES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES";
/// Disabled until controlled-fleet acceptance selects production limits.
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_DIRECTORIES: u64 = 0;
/// Per-bucket admission limit in raw entries; zero disables this dimension.
pub const ENV_SCANNER_BUCKET_QUANTUM_ENTRIES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES";
/// Disabled until controlled-fleet acceptance selects production limits.
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_ENTRIES: u64 = 0;
/// Per-bucket admission limit in raw entry-name bytes; zero disables this dimension.
pub const ENV_SCANNER_BUCKET_QUANTUM_NAME_BYTES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES";
/// Disabled until controlled-fleet acceptance selects production limits.
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_NAME_BYTES: u64 = 0;
/// Environment variable that selects the scanner speed preset.
/// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
/// Controls the sleep factor, maximum sleep duration, and cycle interval.
+3
View File
@@ -39,5 +39,8 @@ serde = { workspace = true, features = ["derive"] }
rmp-serde = { workspace = true }
rustfs-filemeta = { workspace = true }
[dev-dependencies]
serde_json.workspace = true
[lib]
doctest = false
+32 -9
View File
@@ -474,6 +474,12 @@ pub struct BucketTargetUsageInfo {
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct BucketUsageInfo {
pub size: u64,
/// Provenance of this value. Missing and unknown states remain unproven.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage_value_state: Option<crate::BucketUsageValueState>,
/// Certification of this exact value, independent of aggregation time.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage_certification: Option<crate::BucketUsageCertification>,
/// Bucket identity certified by every set contributing to this snapshot.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bucket_incarnation: Option<String>,
@@ -580,6 +586,18 @@ pub struct DataUsageInfo {
/// which last-known-good generation contributed to the view.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub usage_snapshot_set_states: Vec<DataUsageSnapshotSetState>,
/// Complete mixed observations for admin display only. Legacy bucket maps
/// and quota consumers continue to use the canonical values above.
#[serde(
default,
skip_serializing_if = "HashMap::is_empty",
deserialize_with = "crate::usage_observation::deserialize_bucket_observations"
)]
pub usage_snapshot_bucket_observations: HashMap<String, crate::DataUsageBucketObservation>,
/// Response-only marker: a projected admin view must never be persisted or
/// admitted as a quota/cache baseline, including after serialization.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub usage_snapshot_admin_projection: bool,
/// Buckets whose dirty-usage generation moved while this complete
/// snapshot was being walked. Their `buckets_usage` values are the walk
/// result, which the cycle could not certify as current; the next cycle
@@ -709,7 +727,8 @@ pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &Da
/// This applies the structural, exact-baseline, and generation checks of
/// [`observed_data_usage_is_newer`] without retaining the baseline's bucket maps.
pub fn observed_data_usage_is_newer_than_identity(observed: &DataUsageInfo, authoritative: DataUsageSnapshotIdentity) -> bool {
observed.usage_snapshot_converged == Some(false)
!observed.usage_snapshot_admin_projection
&& observed.usage_snapshot_converged == Some(false)
&& (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot())
&& observed.usage_snapshot_authoritative_baseline == Some(authoritative)
&& data_usage_identity_is_newer(observed.snapshot_identity(), authoritative)
@@ -1764,12 +1783,7 @@ impl DataUsageCache {
let mut found = HashSet::new();
found.insert(top);
mark(self, &top_e, &mut found);
self.cache.retain(|k, _| {
if !found.contains(k) {
return false;
}
true
});
self.cache.retain(|k, _| found.contains(k));
}
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
@@ -2029,7 +2043,8 @@ impl DataUsageInfo {
/// Whether this snapshot authoritatively covers every reported bucket.
pub fn is_complete_bucket_usage_snapshot(&self) -> bool {
!self.usage_snapshot_bootstrap_pending
!self.usage_snapshot_admin_projection
&& !self.usage_snapshot_bootstrap_pending
&& self.usage_snapshot_complete
&& self.last_update.is_some()
&& u64::try_from(self.buckets_usage.len()).ok() == Some(self.buckets_count)
@@ -2039,7 +2054,8 @@ impl DataUsageInfo {
/// admin display. Partial data is accepted only with unique set states,
/// a plan digest for every state, and at least one usable generation.
pub fn is_valid_partial_snapshot(&self) -> bool {
if self.usage_snapshot_bootstrap_pending
if self.usage_snapshot_admin_projection
|| self.usage_snapshot_bootstrap_pending
|| !self.usage_snapshot_partial
|| self.usage_snapshot_converged != Some(false)
|| self.last_update.is_none()
@@ -2077,9 +2093,11 @@ impl DataUsageInfo {
Ok(name) => name,
Err(_) => return,
};
self.usage_snapshot_bucket_observations.remove(&bucket_name);
// Update bucket statistics
if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
bucket_usage.mark_observed();
bucket_usage.size += meta_object.size as u64;
bucket_usage.objects_count += 1;
bucket_usage.versions_count += 1; // Simplified: assume 1 version per object
@@ -2132,6 +2150,7 @@ impl DataUsageInfo {
Ok(name) => name,
Err(_) => return,
};
self.usage_snapshot_bucket_observations.remove(&bucket_name);
// Calculate accurate statistics from all versions
let mut total_size = 0u64;
@@ -2160,6 +2179,7 @@ impl DataUsageInfo {
// Update bucket statistics
if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
bucket_usage.mark_observed();
bucket_usage.size += total_size;
bucket_usage.objects_count += 1;
bucket_usage.versions_count += versions_count;
@@ -2340,6 +2360,9 @@ impl BucketUsageInfo {
/// Add size summary to this bucket usage
/// Merge another BucketUsageInfo into this one
pub fn merge(&mut self, other: &BucketUsageInfo) {
// Adding contributions produces a new value, not either input's
// whole-bucket certificate. The scanner may certify the final sum.
self.mark_observed();
if self.bucket_incarnation != other.bucket_incarnation {
self.bucket_incarnation = None;
}
+2
View File
@@ -13,5 +13,7 @@
// limitations under the License.
pub mod data_usage;
pub mod usage_observation;
pub use data_usage::*;
pub use usage_observation::*;
+265
View File
@@ -0,0 +1,265 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
//! Diagnostic provenance and response-only projections of complete bucket observations.
use crate::{BucketUsageInfo, DataUsageInfo, DataUsageSnapshotIdentity};
use serde::{Deserialize, Serialize};
use std::{collections::HashMap, time::SystemTime};
pub const DATA_USAGE_BUCKET_OBSERVATION_VERSION: u16 = 1;
/// Provenance of the returned bucket value, independent of quota authority.
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
#[serde(rename_all = "snake_case")]
pub enum BucketUsageValueState {
Certified,
Carried,
Observed,
#[serde(other)]
Unknown,
}
/// A certification of this exact bucket value, never its publication timestamp.
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct BucketUsageCertification {
pub epoch: u64,
pub cycle: u64,
pub certified_at: SystemTime,
}
/// Immutable provenance of one complete traversal contribution.
///
/// The scanner validates its storage-owned sweep identity before constructing
/// this public projection. Equal start/request plans are allowed when a fresh
/// set contributes alongside another set's mixed traversal.
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct DataUsageBucketObservationSource {
pub pool_index: u64,
pub set_index: u64,
pub bucket_incarnation: String,
pub scanner_epoch: u64,
pub scanner_cycle: u64,
pub scan_plan_digest: [u8; 32],
pub started_plan: [u8; 32],
pub requested_plan: [u8; 32],
pub completed_at: SystemTime,
}
/// An additive companion; legacy bucket maps never contain this mixed value.
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
pub struct DataUsageBucketObservation {
pub version: u16,
pub usage: BucketUsageInfo,
/// Exact enclosing candidate identity. Rewrapping preserves source times.
pub snapshot_identity: DataUsageSnapshotIdentity,
/// Oldest contribution time, so a newer set cannot hide an older value.
pub completed_at: SystemTime,
/// Strictly ordered, complete coverage of the enclosing live set inventory.
pub sources: Vec<DataUsageBucketObservationSource>,
}
/// Companion evolution must not make an otherwise usable canonical snapshot
/// unreadable. Unsupported or malformed diagnostic entries never grant proof.
pub(crate) fn deserialize_bucket_observations<'de, D>(
deserializer: D,
) -> Result<HashMap<String, DataUsageBucketObservation>, D::Error>
where
D: serde::Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum Entry {
Observation(Box<DataUsageBucketObservation>),
Ignored(serde::de::IgnoredAny),
}
Ok(HashMap::<String, Entry>::deserialize(deserializer)?
.into_iter()
.filter_map(|(bucket, entry)| match entry {
Entry::Observation(observation) if observation.version == DATA_USAGE_BUCKET_OBSERVATION_VERSION => {
Some((bucket, *observation))
}
Entry::Observation(_) | Entry::Ignored(_) => None,
})
.collect())
}
impl BucketUsageInfo {
/// Attach a certificate only after the caller proves complete bucket coverage.
pub fn certify(&mut self, certification: BucketUsageCertification) {
self.usage_value_state = Some(BucketUsageValueState::Certified);
self.usage_certification = Some(certification);
}
/// Preserve the certificate when carrying the exact prior bucket value.
pub fn mark_carried(&mut self) {
self.normalize_usage_provenance();
self.usage_value_state = Some(BucketUsageValueState::Carried);
}
/// The value changed without a complete bucket certificate.
pub fn mark_observed(&mut self) {
self.usage_value_state = Some(BucketUsageValueState::Observed);
self.usage_certification = None;
}
/// Future or legacy provenance cannot authorize a retained certificate.
pub fn normalize_usage_provenance(&mut self) {
if !matches!(
self.usage_value_state,
Some(BucketUsageValueState::Certified | BucketUsageValueState::Carried)
) {
self.usage_certification = None;
}
if self.usage_value_state == Some(BucketUsageValueState::Certified) && self.usage_certification.is_none() {
self.usage_value_state = Some(BucketUsageValueState::Unknown);
}
}
}
impl DataUsageBucketObservation {
/// Verify the public provenance against the enclosing candidate and current
/// storage-owned incarnation. This never grants publication or ACK authority.
pub fn matches_snapshot(&self, snapshot: &DataUsageInfo, incarnation: &str) -> bool {
let Some(root_generation) = snapshot.scanner_epoch.zip(snapshot.scanner_cycle) else {
return false;
};
if self.version != DATA_USAGE_BUCKET_OBSERVATION_VERSION
|| snapshot.usage_snapshot_admin_projection
|| self.snapshot_identity != snapshot.snapshot_identity()
|| incarnation.is_empty()
|| self.usage.bucket_incarnation.as_deref() != Some(incarnation)
|| self.sources.is_empty()
|| self.usage.usage_certification.is_some()
|| self.usage.usage_value_state != Some(BucketUsageValueState::Observed)
|| self.sources.iter().map(|source| source.completed_at).min() != Some(self.completed_at)
{
return false;
}
let mut previous = None;
let mut plan = None;
for state in &snapshot.usage_snapshot_set_states {
let key = (state.pool_index, state.set_index);
if previous.is_some_and(|previous| previous >= key)
|| state.scan_plan_digest.is_none()
|| plan.is_some_and(|plan| Some(plan) != state.scan_plan_digest)
{
return false;
}
previous = Some(key);
plan = state.scan_plan_digest;
}
let mut sources = self.sources.iter();
for state in snapshot.usage_snapshot_set_states.iter().filter(|state| !state.tombstone) {
let Some(source) = sources.next() else {
return false;
};
let Some(set_generation) = state.scanner_epoch.zip(state.scanner_cycle) else {
return false;
};
if (source.pool_index, source.set_index) != (state.pool_index, state.set_index)
|| source.bucket_incarnation != incarnation
|| Some(source.scan_plan_digest) != state.scan_plan_digest
|| (source.scanner_epoch, source.scanner_cycle) > set_generation
|| set_generation > root_generation
{
return false;
}
}
sources.next().is_none()
}
}
impl DataUsageInfo {
/// Overlay validated companions on an owned admin response only.
///
/// `incarnations` must come from current storage metadata, not the companion.
/// The projection is marked explicitly so persistence and quota boundaries
/// can reject accidental reuse. Invalid companions leave canonical values
/// unchanged. Counter overflow rejects the whole projection.
pub fn project_bucket_observations_for_admin(&mut self, incarnations: &HashMap<String, String>) -> usize {
if self.usage_snapshot_admin_projection
|| (!self.is_complete_bucket_usage_snapshot() && !self.is_valid_partial_snapshot())
{
return 0;
}
let observations =
self.usage_snapshot_bucket_observations
.iter()
.filter(|(bucket, observation)| {
if self.usage_snapshot_complete
&& (!self.usage_snapshot_pending_buckets.iter().any(|pending| pending == *bucket)
|| u64::try_from(self.usage_snapshot_pending_buckets.len()).ok()
!= Some(self.usage_snapshot_pending_bucket_count))
{
return false;
}
if self
.buckets_usage
.get(*bucket)
.and_then(|usage| usage.usage_certification)
.is_some_and(|certificate| {
observation.sources.iter().all(|source| {
(source.scanner_epoch, source.scanner_cycle) <= (certificate.epoch, certificate.cycle)
})
})
{
return false;
}
incarnations.get(*bucket).is_some_and(|incarnation| {
self.buckets_usage
.get(*bucket)
.and_then(|usage| usage.bucket_incarnation.as_ref())
.is_none_or(|canonical| canonical == incarnation)
&& observation.matches_snapshot(self, incarnation)
})
})
.collect::<HashMap<_, _>>();
if observations.is_empty() {
return 0;
}
let mut values = self
.buckets_usage
.iter()
.filter(|(bucket, _)| !observations.contains_key(bucket))
.map(|(_, usage)| usage)
.chain(observations.values().map(|observation| &observation.usage));
let totals = values.try_fold([0_u64; 4], |total, usage| {
Some([
total[0].checked_add(usage.size)?,
total[1].checked_add(usage.objects_count)?,
total[2].checked_add(usage.versions_count)?,
total[3].checked_add(usage.delete_markers_count)?,
])
});
let Some([size, objects, versions, markers]) = totals else {
return 0;
};
let projected = observations.len();
for (bucket, observation) in observations {
self.buckets_usage.insert(bucket.clone(), observation.usage.clone());
}
self.bucket_sizes = self
.buckets_usage
.iter()
.map(|(bucket, usage)| (bucket.clone(), usage.size))
.collect();
self.buckets_count = u64::try_from(self.buckets_usage.len()).unwrap_or(u64::MAX);
self.objects_total_size = size;
self.objects_total_count = objects;
self.versions_total_count = versions;
self.delete_markers_total_count = markers;
self.replication_info.clear();
self.tier_stats = None;
self.unknown_tier_stats = None;
self.usage_snapshot_complete = false;
self.usage_snapshot_partial = true;
self.usage_snapshot_converged = Some(false);
self.usage_snapshot_observational = true;
self.usage_snapshot_admin_projection = true;
projected
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,323 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
use crate::DataUsageSnapshotSetState;
use std::time::Duration;
const INCARNATION: &str = "11111111-1111-4111-8111-111111111111";
fn time(seconds: u64) -> SystemTime {
SystemTime::UNIX_EPOCH + Duration::from_secs(seconds)
}
fn usage(size: u64) -> BucketUsageInfo {
BucketUsageInfo {
size,
objects_count: size,
versions_count: size,
bucket_incarnation: Some(INCARNATION.to_string()),
..Default::default()
}
}
fn snapshot() -> DataUsageInfo {
let mut main = usage(10);
main.certify(BucketUsageCertification {
epoch: 2,
cycle: 3,
certified_at: time(30),
});
main.mark_carried();
let mut snapshot = DataUsageInfo {
last_update: Some(time(90)),
scanner_epoch: Some(2),
scanner_cycle: Some(9),
buckets_count: 1,
buckets_usage: HashMap::from([("bucket".to_string(), main)]),
bucket_sizes: HashMap::from([("bucket".to_string(), 10)]),
usage_snapshot_complete: true,
usage_snapshot_converged: Some(false),
usage_snapshot_pending_buckets: vec!["bucket".to_string()],
usage_snapshot_pending_bucket_count: 1,
usage_snapshot_set_states: (0..2)
.map(|set_index| DataUsageSnapshotSetState {
pool_index: 0,
set_index,
scanner_epoch: Some(2),
scanner_cycle: Some(9),
scan_plan_digest: Some([1; 32]),
complete: true,
..Default::default()
})
.collect(),
..Default::default()
};
snapshot.calculate_totals();
let mut observed = usage(25);
observed.mark_observed();
let observation = DataUsageBucketObservation {
version: DATA_USAGE_BUCKET_OBSERVATION_VERSION,
usage: observed,
snapshot_identity: snapshot.snapshot_identity(),
completed_at: time(40),
sources: (0..2)
.map(|set_index| DataUsageBucketObservationSource {
pool_index: 0,
set_index,
bucket_incarnation: INCARNATION.to_string(),
scanner_epoch: 2,
scanner_cycle: 4 + set_index,
scan_plan_digest: [1; 32],
started_plan: [2; 32],
requested_plan: [3; 32],
completed_at: time(40 + set_index),
})
.collect(),
};
snapshot
.usage_snapshot_bucket_observations
.insert("bucket".to_string(), observation);
snapshot
}
fn incarnations() -> HashMap<String, String> {
HashMap::from([("bucket".to_string(), INCARNATION.to_string())])
}
#[test]
fn provenance_is_additive_and_unknown_states_do_not_keep_certificates() {
let legacy = usage(10);
let bytes = serde_json::to_vec(&legacy).expect("encode a legacy bucket without provenance");
let text = std::str::from_utf8(&bytes).expect("JSON is UTF-8");
assert!(!text.contains("usage_value_state"));
assert!(!text.contains("usage_certification"));
let decoded: BucketUsageInfo = serde_json::from_slice(&bytes).expect("read legacy bucket");
assert_eq!(decoded.usage_value_state, None);
assert_eq!(decoded.usage_certification, None);
let mut wire = serde_json::to_value(legacy).expect("encode future bucket fixture");
wire["usage_value_state"] = serde_json::json!("future_state");
wire["usage_certification"] =
serde_json::json!({"epoch":2,"cycle":3,"certified_at":{"secs_since_epoch":30,"nanos_since_epoch":0}});
let mut future: BucketUsageInfo = serde_json::from_value(wire).expect("unknown diagnostic enum remains readable");
future.normalize_usage_provenance();
assert_eq!(future.usage_value_state, Some(BucketUsageValueState::Unknown));
assert_eq!(future.usage_certification, None);
}
#[test]
fn carrying_preserves_time_but_adding_a_source_invalidates_the_certificate() {
let mut bucket = snapshot().buckets_usage.remove("bucket").expect("canonical bucket");
bucket.mark_carried();
assert_eq!(bucket.usage_certification.map(|certification| certification.certified_at), Some(time(30)));
bucket.merge(&usage(1));
assert_eq!(bucket.size, 11);
assert_eq!(bucket.usage_value_state, Some(BucketUsageValueState::Observed));
assert_eq!(bucket.usage_certification, None);
}
#[test]
fn direct_object_accumulation_does_not_reuse_the_old_certificate() {
let mut simple = snapshot();
simple.add_object("bucket/object", &rustfs_filemeta::MetaObject::default());
assert_eq!(simple.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
assert_eq!(simple.buckets_usage["bucket"].usage_certification, None);
assert!(simple.usage_snapshot_bucket_observations.is_empty());
let mut versioned = snapshot();
versioned.add_object_from_file_meta("bucket/object", &rustfs_filemeta::FileMeta::default());
assert_eq!(versioned.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
assert_eq!(versioned.buckets_usage["bucket"].usage_certification, None);
assert!(versioned.usage_snapshot_bucket_observations.is_empty());
}
#[test]
fn future_or_malformed_companions_do_not_break_the_canonical_reader() {
let canonical = snapshot();
for entry in [
serde_json::json!({"version": 2, "future_payload": {"new": true}}),
serde_json::json!({"version": 1, "usage": "broken"}),
] {
let mut wire = serde_json::to_value(&canonical).expect("canonical JSON");
wire["usage_snapshot_bucket_observations"]["bucket"] = entry;
let decoded: DataUsageInfo = serde_json::from_value(wire).expect("diagnostics cannot break canonical data");
assert_eq!(decoded.buckets_usage, canonical.buckets_usage);
assert!(decoded.usage_snapshot_bucket_observations.is_empty());
assert!(decoded.is_complete_bucket_usage_snapshot());
}
}
#[test]
fn complete_companion_projects_only_the_admin_copy_and_keeps_measurement_times() {
let canonical = snapshot();
let canonical_bytes = serde_json::to_vec(&canonical).expect("encode canonical candidate");
let mut response = canonical.clone();
assert_eq!(response.project_bucket_observations_for_admin(&incarnations()), 1);
assert_eq!(response.buckets_usage["bucket"].size, 25);
assert_eq!(response.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
assert_eq!(response.buckets_usage["bucket"].usage_certification, None);
assert_eq!(response.objects_total_size, 25);
assert_eq!(response.bucket_sizes["bucket"], 25);
assert_eq!(response.last_update, canonical.last_update);
assert_eq!(response.usage_snapshot_bucket_observations["bucket"].completed_at, time(40));
assert!(response.usage_snapshot_admin_projection);
assert!(response.usage_snapshot_partial);
assert!(response.usage_snapshot_observational);
assert_eq!(response.usage_snapshot_converged, Some(false));
assert!(!response.is_complete_bucket_usage_snapshot());
assert!(
!response.is_valid_partial_snapshot(),
"response-only values cannot be a storage candidate"
);
assert_eq!(serde_json::to_vec(&canonical).expect("reencode original"), canonical_bytes);
assert_eq!(canonical.buckets_usage["bucket"].size, 10);
assert_eq!(
canonical.buckets_usage["bucket"]
.usage_certification
.map(|certificate| certificate.certified_at),
Some(time(30))
);
assert_eq!(response.project_bucket_observations_for_admin(&incarnations()), 0);
}
#[test]
fn invalid_companions_preserve_the_canonical_bucket() {
for invalid in 0..11 {
let mut candidate = snapshot();
let observation = candidate
.usage_snapshot_bucket_observations
.get_mut("bucket")
.expect("companion");
match invalid {
0 => observation.version += 1,
1 => observation.snapshot_identity.scanner_cycle = Some(8),
2 => {
observation.sources.pop();
}
3 => observation.sources[1].set_index = 0,
4 => observation.sources[1].scanner_cycle = 10,
5 => observation.sources[1].scan_plan_digest = [9; 32],
6 => observation.sources[1].bucket_incarnation = "other".to_string(),
7 => observation.completed_at = time(41),
8 => observation.usage.certify(BucketUsageCertification {
epoch: 2,
cycle: 9,
certified_at: time(90),
}),
9 => candidate.usage_snapshot_set_states[1].scanner_cycle = None,
10 => {
candidate
.buckets_usage
.get_mut("bucket")
.expect("canonical")
.bucket_incarnation = Some("recreated".to_string())
}
_ => unreachable!("bounded fixture variants"),
}
let before = candidate.clone();
assert_eq!(
candidate.project_bucket_observations_for_admin(&incarnations()),
0,
"invalid case {invalid}"
);
assert_eq!(candidate, before, "invalid case {invalid} changed canonical data");
}
}
#[test]
fn a_fresh_set_can_contribute_without_refreshing_an_older_mixed_set() {
let mut candidate = snapshot();
let observation = candidate
.usage_snapshot_bucket_observations
.get_mut("bucket")
.expect("companion");
observation.sources[1].started_plan = observation.sources[1].requested_plan;
observation.sources[1].completed_at = time(90);
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 1);
assert_eq!(candidate.usage_snapshot_bucket_observations["bucket"].completed_at, time(40));
}
#[test]
fn companion_only_partial_keeps_legacy_maps_empty_until_admin_projection() {
let mut candidate = snapshot();
candidate.buckets_usage.clear();
candidate.bucket_sizes.clear();
candidate.buckets_count = 0;
candidate.calculate_totals();
candidate.usage_snapshot_complete = false;
candidate.usage_snapshot_partial = true;
candidate.usage_snapshot_observational = true;
for state in &mut candidate.usage_snapshot_set_states {
state.complete = false;
}
assert!(candidate.is_valid_partial_snapshot());
let wire = serde_json::to_value(&candidate).expect("encode companion-only observation");
assert_eq!(wire["buckets_usage"], serde_json::json!({}));
assert_eq!(wire["bucket_sizes"], serde_json::json!({}));
assert_eq!(wire["objects_total_size"], 0);
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 1);
assert_eq!(candidate.objects_total_size, 25);
assert!(candidate.usage_snapshot_partial);
assert!(!candidate.usage_snapshot_complete);
}
#[test]
fn projection_rejects_counter_overflow_without_partial_mutation() {
let mut candidate = snapshot();
candidate.buckets_usage.insert("cold".to_string(), usage(u64::MAX));
candidate.buckets_count = 2;
let before = candidate.clone();
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 0);
assert_eq!(candidate, before);
}
#[test]
fn a_newer_canonical_certificate_and_missing_pending_prevent_old_overlay() {
for newer_certificate in [false, true] {
let mut candidate = snapshot();
if newer_certificate {
candidate
.buckets_usage
.get_mut("bucket")
.expect("canonical")
.certify(BucketUsageCertification {
epoch: 2,
cycle: 9,
certified_at: time(90),
});
} else {
candidate.usage_snapshot_pending_buckets.clear();
candidate.usage_snapshot_pending_bucket_count = 0;
}
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 0);
assert_eq!(candidate.buckets_usage["bucket"].size, 10);
}
}
#[test]
fn provenance_and_companions_change_candidate_bytes_and_round_trip_as_maps() {
let canonical = snapshot();
let original = serde_json::to_vec(&canonical).expect("encode candidate");
let mut changed = canonical.clone();
changed
.buckets_usage
.get_mut("bucket")
.expect("bucket")
.usage_certification
.as_mut()
.expect("certificate")
.certified_at = time(31);
assert_ne!(serde_json::to_vec(&changed).expect("encode changed certificate"), original);
let mut changed = canonical.clone();
changed
.usage_snapshot_bucket_observations
.get_mut("bucket")
.expect("observation")
.sources[0]
.started_plan = [8; 32];
assert_ne!(serde_json::to_vec(&changed).expect("encode changed source"), original);
let named = rmp_serde::to_vec_named(&canonical).expect("encode named map");
let restored: DataUsageInfo = rmp_serde::from_slice(&named).expect("read named map with additive fields");
assert_eq!(restored, canonical);
}
@@ -766,9 +766,10 @@ impl Run {
|| async {
let mut ready = true;
let mut observations = Vec::with_capacity(4);
for node in 0..4 {
assert!(self.dist.cluster.nodes.len() >= 4);
for (node, cluster_node) in self.dist.cluster.nodes.iter().enumerate().take(4) {
match http
.get(format!("{}/health/ready", self.dist.cluster.nodes[node].url))
.get(format!("{}/health/ready", cluster_node.url))
.timeout(Duration::from_secs(3))
.send()
.await
@@ -1332,14 +1332,15 @@ mod tests {
.collect();
assert_eq!(confirmed.len(), 1);
let mut receivers = Vec::new();
for drive in 0..phase.disks.get(1).map_or(0, Vec::len) {
let receiver_endpoints = phase.disks.get(1).map_or(&[][..], |disks| &phase.endpoints[1][..disks.len()]);
for endpoint in receiver_endpoints {
for cas in [prepares[1], commits[1]] {
let matching: Vec<_> = events[1]
.iter()
.copied()
.filter(|event| {
event["kind"] == "receiver"
&& event["disk"] == phase.endpoints[1][drive]
&& event["disk"] == *endpoint
&& event["dst_volume"] == ".rustfs.sys"
&& event["dst_path"] == "pool.bin"
&& event["etag"] == cas["etag"]
@@ -22,10 +22,11 @@ use rustfs_protos::{
models::PingBodyBuilder,
proto_gen::node_service::{
BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseReleaseResponse, ScannerPublicationLeaseRequest,
ScannerPublicationLeaseResponse, SnapshotLeaseMutationResponse, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
SnapshotLeaseRequest, SnapshotLeaseResponse, node_service_server::NodeService,
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, MrfLifecycleRequest, MrfLifecycleResponse, PingRequest,
PingResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseReleaseResponse,
ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, SnapshotLeaseMutationResponse,
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse,
node_service_server::NodeService,
},
};
use std::pin::Pin;
@@ -995,6 +996,10 @@ impl NodeService for MinimalLockNodeService {
Err(Status::unimplemented("lock-only test server"))
}
async fn mrf_lifecycle(&self, _request: Request<MrfLifecycleRequest>) -> Result<Response<MrfLifecycleResponse>, Status> {
Err(Status::unimplemented("lock-only test server"))
}
async fn background_heal_status(
&self,
_request: Request<rustfs_protos::proto_gen::node_service::BackgroundHealStatusRequest>,
+38 -7
View File
@@ -553,6 +553,10 @@ pub mod runtime {
global_lock_client, global_lock_clients, global_tier_config_mgr, local_disk_map_read, object_store_handle, region,
rustfs_port, setup_is_dist_erasure, setup_is_erasure, setup_is_erasure_sd, transition_state_handle,
};
pub use crate::runtime::usage_watch::{
SCANNER_USAGE_WATCH_COVERAGE_VERSION, SCANNER_USAGE_WATCH_LEASE_MS, ScannerUsageWatchKnown, ScannerUsageWatchObservation,
ScannerUsageWatchRequest,
};
}
pub mod layout {
@@ -636,6 +640,30 @@ pub mod rio {
}
pub mod rpc {
#[cfg(feature = "test-util")]
pub use crate::cluster::rpc::internode_data_transport::{
InternodeDataTransport, InternodeDataTransportCapabilities, NsScannerCapabilityRequest, NsScannerStreamRequest,
ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest,
};
#[cfg(feature = "test-util")]
pub mod test_util {
/// Build a remote disk using a controlled transport and no health monitor.
///
/// # Errors
/// Returns the normal remote-disk construction error for the supplied endpoint.
pub async fn remote_disk_with_transport(
endpoint: &crate::layout::endpoint::Endpoint,
transport: std::sync::Arc<dyn crate::cluster::rpc::internode_data_transport::InternodeDataTransport>,
) -> crate::disk::error::Result<crate::disk::DiskStore> {
let options = crate::disk::DiskOption {
cleanup: false,
health_check: false,
};
let disk = crate::cluster::rpc::remote_disk::RemoteDisk::new(endpoint, &options, transport).await?;
Ok(std::sync::Arc::new(crate::disk::Disk::Remote(Box::new(disk))))
}
}
pub use crate::cluster::rpc::http_auth::with_rpc_background_request;
pub use crate::cluster::rpc::{
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
@@ -648,13 +676,16 @@ pub mod rpc {
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
rpc_auth_is_resource_exhausted, rpc_background_admission, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_durable_progress,
sign_ns_scanner_capability_with_tier_registry_generation, sign_ns_scanner_capability_with_usage_capabilities,
sign_ns_scanner_usage_watch_capability, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
verify_ns_scanner_capability_with_usage_capabilities, verify_ns_scanner_usage_watch_capability,
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
};
}
+3 -1
View File
@@ -544,8 +544,10 @@ mod tests {
] {
let first: Error = make_error("buckets/first/.metadata.bin".into()).into();
let second: Error = make_error("buckets/second/.metadata.bin".into()).into();
let cloned = first.clone();
assert_eq!(first, second, "record paths must not fragment error grouping");
assert_eq!(first.clone(), second, "cloning must preserve error grouping");
assert_eq!(cloned, second, "cloning must preserve error grouping");
assert_eq!(std::io::Error::from(cloned).to_string(), message);
let io_error = std::io::Error::from(first);
let detail = io_error
@@ -20,6 +20,8 @@ use crate::test_metrics::CapturingRecorder;
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
use std::time::SystemTime;
mod native_compatibility;
async fn stop_reporting_fixture(store: &crate::store::ECStore) {
if let Some(cancel) = store.ctx.background_cancel_token() {
cancel.cancel();
@@ -90,6 +92,10 @@ async fn install_reporting_snapshot(
#[tokio::test(flavor = "current_thread")]
#[serial]
async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger_admission() {
assert_durable_quota_reporting_preserves_admission().await;
}
async fn assert_durable_quota_reporting_preserves_admission() {
let recorder = CapturingRecorder::default();
let _recorder = metrics::set_default_local_recorder(&recorder);
for protocol in [1, 2] {
@@ -196,6 +202,11 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
.expect("fixture cache cleanup");
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
stop_reporting_fixture(&store).await;
for dir in _dirs {
let path = dir.path().to_path_buf();
dir.close().expect("remove durable quota reporting fixture");
assert!(!path.exists());
}
}
}
}
@@ -203,6 +214,10 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
#[tokio::test]
#[serial]
async fn snapshot_quota_admission_retains_degraded_floors_until_migration() {
assert_snapshot_quota_preserves_degraded_floors().await;
}
async fn assert_snapshot_quota_preserves_degraded_floors() {
for source in ["missing", "legacy", "incomplete", "observed"] {
let (_dirs, store, _) = isolated_durable_quota_store().await;
let bucket = format!("quota-compat-{}", Uuid::new_v4().simple());
@@ -233,12 +248,21 @@ async fn snapshot_quota_admission_retains_degraded_floors_until_migration() {
.expect("fixture cache cleanup");
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
stop_reporting_fixture(&store).await;
for dir in _dirs {
let path = dir.path().to_path_buf();
dir.close().expect("remove legacy quota reporting fixture");
assert!(!path.exists());
}
}
}
#[tokio::test(flavor = "current_thread")]
#[serial]
async fn quota_without_limit_keeps_the_zero_usage_io_fast_path() {
assert_unlimited_quota_preserves_fast_path().await;
}
async fn assert_unlimited_quota_preserves_fast_path() {
let (_dirs, store, sys) = isolated_durable_quota_store().await;
let bucket = format!("quota-unlimited-{}", Uuid::new_v4().simple());
create_bucket_with_quota_json(
@@ -259,6 +283,11 @@ async fn quota_without_limit_keeps_the_zero_usage_io_fast_path() {
assert_eq!(recorder.counter_value(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL, &[]), 0);
assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket), 0);
stop_reporting_fixture(&store).await;
for dir in _dirs {
let path = dir.path().to_path_buf();
dir.close().expect("remove unlimited quota reporting fixture");
assert!(!path.exists());
}
}
#[tokio::test]
@@ -0,0 +1,265 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
use super::*;
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use rustfs_data_usage::{
DATA_USAGE_BUCKET_OBSERVATION_VERSION, DataUsageBucketObservation, DataUsageBucketObservationSource,
DataUsageSnapshotSetState,
};
#[tokio::test(flavor = "current_thread")]
#[serial]
async fn usage_quota_modes_preserve_actual_admission_decisions() {
let (dirs, store, sys) = isolated_durable_quota_store().await;
let bucket = format!("usage-no-quota-{}", Uuid::new_v4().simple());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("unconfigured quota bucket");
let check = QuotaChecker::new(sys)
.check_quota(&bucket, QuotaOperation::PutObject, u64::MAX)
.await
.expect("no quota");
assert!(check.allowed && check.quota_limit.is_none());
store
.put_object(
&bucket,
"allowed",
&mut PutObjReader::from_vec(vec![0x5a; 512]),
&ObjectOptions::default(),
)
.await
.expect("real no-quota write");
stop_reporting_fixture(&store).await;
let paths = dirs.iter().map(|dir| dir.path().to_path_buf()).collect::<Vec<_>>();
drop(store);
drop(dirs);
assert!(paths.iter().all(|path| !path.exists()));
assert_unlimited_quota_preserves_fast_path().await;
assert_snapshot_quota_preserves_degraded_floors().await;
assert_durable_quota_reporting_preserves_admission().await;
println!(
"usage_assertion: {}",
serde_json::json!({
"case_id": "USAGE-19", "assertion_id": "USAGE-19-A01", "status": "PASS",
"evidence": {
"quota_modes": ["unconfigured", "unlimited", "legacy-snapshot", "durable-v1", "durable-v2"],
"durable_snapshot_sources": ["missing", "legacy", "incomplete", "observed", "complete"],
"durable_storage_decisions_per_fixture": ["reject-8192", "accept-512", "accept-3584", "reject-1"],
"legacy_missing_baseline_fails_closed": true,
"legacy_grounded_baselines": ["legacy", "incomplete", "observed"]
}
})
);
println!(
"usage_cleanup: {}",
serde_json::json!({"case_id": "USAGE-19", "status": "PASS", "fixtures_removed": true})
);
}
async fn persist_companion_fixture(
store: Arc<crate::store::ECStore>,
bucket: &str,
primary: bool,
companion_size: u64,
) -> (String, Vec<u8>) {
let incarnation = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("real companion bucket identity");
let completed_at = SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(20);
let mut snapshot = DataUsageInfo {
last_update: Some(completed_at),
scanner_epoch: Some(2),
scanner_cycle: Some(3),
usage_snapshot_complete: primary,
usage_snapshot_partial: !primary,
usage_snapshot_observational: true,
usage_snapshot_converged: Some(false),
usage_snapshot_authoritative_baseline: Some(DataUsageInfo::default().snapshot_identity()),
usage_snapshot_pending_buckets: vec![bucket.to_owned()],
usage_snapshot_pending_bucket_count: 1,
usage_snapshot_set_states: vec![DataUsageSnapshotSetState {
pool_index: 0,
set_index: 0,
scanner_epoch: Some(2),
scanner_cycle: Some(3),
scan_plan_digest: Some([1; 32]),
complete: primary,
..Default::default()
}],
..Default::default()
};
if primary {
snapshot.buckets_count = 1;
snapshot.buckets_usage.insert(
bucket.into(),
BucketUsageInfo {
size: 1000,
objects_count: 1,
versions_count: 1,
bucket_incarnation: Some(incarnation.to_string()),
..Default::default()
},
);
snapshot.bucket_sizes.insert(bucket.into(), 1000);
snapshot.calculate_totals();
}
let mut usage = BucketUsageInfo {
size: companion_size,
objects_count: 1,
versions_count: 1,
bucket_incarnation: Some(incarnation.to_string()),
..Default::default()
};
usage.mark_observed();
snapshot.usage_snapshot_bucket_observations.insert(
bucket.into(),
DataUsageBucketObservation {
version: DATA_USAGE_BUCKET_OBSERVATION_VERSION,
usage,
snapshot_identity: snapshot.snapshot_identity(),
completed_at,
sources: vec![DataUsageBucketObservationSource {
pool_index: 0,
set_index: 0,
bucket_incarnation: incarnation.to_string(),
scanner_epoch: 2,
scanner_cycle: 3,
scan_plan_digest: [1; 32],
started_plan: [2; 32],
requested_plan: [3; 32],
completed_at,
}],
},
);
let mut admin = snapshot.clone();
assert_eq!(
admin.project_bucket_observations_for_admin(&HashMap::from([(bucket.to_owned(), incarnation.to_string())])),
1,
"the companion must be valid, not discarded as malformed"
);
assert_eq!(admin.buckets_usage[bucket].size, companion_size);
let path = format!(
"{}/{}",
crate::disk::BUCKET_META_PREFIX,
rustfs_data_usage::DATA_USAGE_OBSERVED_OBJECT_NAME
);
let bytes = serde_json::to_vec(&snapshot).expect("encode valid companion snapshot");
crate::config::com::save_config(store, &path, bytes.clone())
.await
.expect("persist companion beside canonical fields");
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
(path, bytes)
}
#[tokio::test(flavor = "current_thread")]
#[serial]
async fn usage_companions_do_not_expand_legacy_fallback_or_durable_admission() {
let mut decisions = Vec::new();
for protocol in [0, 1, 2] {
for (primary, companion_size) in [(true, 0), (true, 1_000_000), (false, 1_000_000)] {
let (dirs, store, sys) = isolated_durable_quota_store().await;
let bucket = format!("usage-companion-quota-{}", Uuid::new_v4().simple());
let quota = match protocol {
0 => legacy_quota_json(4096),
1 => serde_json::to_vec(&BucketQuota::new(Some(4096))).unwrap(),
_ => serde_json::to_vec(&BucketQuota::new_sharded(4096)).unwrap(),
};
create_bucket_with_quota_json(&store, &bucket, quota).await;
let (path, bytes) = persist_companion_fixture(store.clone(), &bucket, primary, companion_size).await;
let checker = if protocol == 0 {
QuotaChecker::new(Arc::new(RwLock::new(BucketMetadataSys::new(store.clone()))))
} else {
QuotaChecker::new(sys)
};
if protocol == 0 {
let small = checker.check_quota(&bucket, QuotaOperation::PutObject, 512).await;
if primary {
let small = small.expect("grounded primary fallback");
assert!(small.allowed && !small.uses_durable_reservations);
assert_eq!(small.current_usage, Some(1000));
assert!(
!checker
.check_quota(&bucket, QuotaOperation::PutObject, 4096)
.await
.expect("legacy limit")
.allowed
);
} else {
assert!(
matches!(small, Err(QuotaError::UsageUnavailable { .. })),
"a companion cannot invent a legacy baseline"
);
}
} else {
assert!(
checker
.check_quota(&bucket, QuotaOperation::PutObject, 512)
.await
.expect("durable checker")
.uses_durable_reservations
);
if protocol == 1 {
assert!(matches!(
store
.put_object(
&bucket,
"bootstrap",
&mut PutObjReader::from_vec(vec![0x5a; 512]),
&ObjectOptions::default()
)
.await,
Err(StorageError::SlowDown)
));
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
.await
.expect("real v1 ledger bootstrap");
}
store
.put_object(&bucket, "fits", &mut PutObjReader::from_vec(vec![0x5a; 4096]), &ObjectOptions::default())
.await
.expect("ledger admits exactly the physical quota regardless of companion");
assert!(
matches!(
store
.put_object(&bucket, "exceeds", &mut PutObjReader::from_vec(vec![0x5a; 1]), &ObjectOptions::default())
.await,
Err(StorageError::QuotaExceeded { .. })
),
"a low companion cannot expand the ledger"
);
}
assert_eq!(
crate::config::com::read_config(store.clone(), &path)
.await
.expect("retained scanner evidence"),
bytes
);
decisions
.push(serde_json::json!({"protocol": protocol, "primary_baseline": primary, "companion_size": companion_size}));
crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
.await
.expect("clear fixture caches");
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
stop_reporting_fixture(&store).await;
let paths = dirs.iter().map(|dir| dir.path().to_path_buf()).collect::<Vec<_>>();
drop(store);
drop(dirs);
assert!(paths.iter().all(|path| !path.exists()));
}
}
println!(
"usage_assertion: {}",
serde_json::json!({
"case_id": "USAGE-19", "assertion_id": "USAGE-19-A02", "status": "PASS",
"evidence": {"validated_companion_fixtures": decisions, "legacy_primary_bytes": 1000, "durable_limit_bytes": 4096, "companion_only_legacy_admission": "UsageUnavailable"}
})
);
println!(
"usage_cleanup: {}",
serde_json::json!({"case_id": "USAGE-19", "status": "PASS", "fixtures_removed": true})
);
}
+270 -20
View File
@@ -139,6 +139,8 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
const REPLAY_CACHE_METADATA_BYTES: u64 = 1024 * 1024;
const REPLAY_CACHE_TABLE_OVERHEAD_BYTES: u64 = 512;
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
const NS_SCANNER_COMPLETE_OBSERVATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-complete-observation-v1";
const NS_SCANNER_DURABLE_PROGRESS_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-durable-progress-v1";
const NS_SCANNER_CHECKPOINT_COVERAGE_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-checkpoint-coverage-v1";
const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
@@ -795,6 +797,8 @@ fn update_ns_scanner_capability_mac(
server_epoch: Uuid,
supports_tier_registry_generation: bool,
supports_checkpoint_coverage: bool,
supports_durable_progress: bool,
supports_complete_observation: bool,
) {
mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
@@ -809,14 +813,22 @@ fn update_ns_scanner_capability_mac(
if supports_checkpoint_coverage {
mac.update(NS_SCANNER_CHECKPOINT_COVERAGE_AUTH_DOMAIN);
}
if supports_durable_progress {
mac.update(NS_SCANNER_DURABLE_PROGRESS_AUTH_DOMAIN);
}
if supports_complete_observation {
mac.update(NS_SCANNER_COMPLETE_OBSERVATION_AUTH_DOMAIN);
}
}
fn generate_ns_scanner_capability_proof(
fn generate_ns_scanner_capability_proof_with_observation(
secret: &str,
challenge: Uuid,
server_epoch: Uuid,
supports_tier_registry_generation: bool,
supports_checkpoint_coverage: bool,
supports_durable_progress: bool,
supports_complete_observation: bool,
) -> std::io::Result<Vec<u8>> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
@@ -829,17 +841,22 @@ fn generate_ns_scanner_capability_proof(
server_epoch,
supports_tier_registry_generation,
supports_checkpoint_coverage,
supports_durable_progress,
supports_complete_observation,
);
Ok(mac.finalize().into_bytes().to_vec())
}
fn verify_ns_scanner_capability_proof(
#[allow(clippy::too_many_arguments)]
fn verify_ns_scanner_capability_proof_with_observation(
secret: &str,
challenge: Uuid,
server_epoch: Uuid,
proof: &[u8],
supports_tier_registry_generation: bool,
supports_checkpoint_coverage: bool,
supports_durable_progress: bool,
supports_complete_observation: bool,
) -> std::io::Result<()> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
@@ -852,11 +869,35 @@ fn verify_ns_scanner_capability_proof(
server_epoch,
supports_tier_registry_generation,
supports_checkpoint_coverage,
supports_durable_progress,
supports_complete_observation,
);
mac.verify_slice(proof)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
}
fn generate_ns_scanner_capability_proof(
secret: &str,
challenge: Uuid,
server_epoch: Uuid,
tier: bool,
checkpoint: bool,
progress: bool,
) -> std::io::Result<Vec<u8>> {
generate_ns_scanner_capability_proof_with_observation(secret, challenge, server_epoch, tier, checkpoint, progress, false)
}
fn verify_ns_scanner_capability_proof(
secret: &str,
challenge: Uuid,
server_epoch: Uuid,
proof: &[u8],
tier: bool,
checkpoint: bool,
progress: bool,
) -> std::io::Result<()> {
verify_ns_scanner_capability_proof_with_observation(secret, challenge, server_epoch, proof, tier, checkpoint, progress, false)
}
pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, false)
}
@@ -878,6 +919,22 @@ pub fn sign_ns_scanner_capability_with_checkpoint_coverage(
server_epoch: Uuid,
supports_tier_registry_generation: bool,
supports_checkpoint_coverage: bool,
) -> std::io::Result<Vec<u8>> {
sign_ns_scanner_capability_with_durable_progress(
challenge,
server_epoch,
supports_tier_registry_generation,
supports_checkpoint_coverage,
false,
)
}
pub fn sign_ns_scanner_capability_with_durable_progress(
challenge: Uuid,
server_epoch: Uuid,
supports_tier_registry_generation: bool,
supports_checkpoint_coverage: bool,
supports_durable_progress: bool,
) -> std::io::Result<Vec<u8>> {
generate_ns_scanner_capability_proof(
&get_shared_secret()?,
@@ -885,6 +942,7 @@ pub fn sign_ns_scanner_capability_with_checkpoint_coverage(
server_epoch,
supports_tier_registry_generation,
supports_checkpoint_coverage,
supports_durable_progress,
)
}
@@ -909,6 +967,24 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
proof: &[u8],
supports_tier_registry_generation: bool,
supports_checkpoint_coverage: bool,
) -> std::io::Result<()> {
verify_ns_scanner_capability_with_durable_progress(
challenge,
server_epoch,
proof,
supports_tier_registry_generation,
supports_checkpoint_coverage,
false,
)
}
pub fn verify_ns_scanner_capability_with_durable_progress(
challenge: Uuid,
server_epoch: Uuid,
proof: &[u8],
supports_tier_registry_generation: bool,
supports_checkpoint_coverage: bool,
supports_durable_progress: bool,
) -> std::io::Result<()> {
verify_ns_scanner_capability_proof(
&get_shared_secret()?,
@@ -917,6 +993,74 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
proof,
supports_tier_registry_generation,
supports_checkpoint_coverage,
supports_durable_progress,
)
}
pub fn sign_ns_scanner_capability_with_usage_capabilities(
challenge: Uuid,
server_epoch: Uuid,
tier: bool,
checkpoint: bool,
progress: bool,
observation: bool,
) -> std::io::Result<Vec<u8>> {
generate_ns_scanner_capability_proof_with_observation(
&get_shared_secret()?,
challenge,
server_epoch,
tier,
checkpoint,
progress,
observation,
)
}
fn ns_scanner_usage_watch_capability_mac(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<HmacSha256> {
if challenge.is_nil() || server_epoch.is_nil() {
return Err(std::io::Error::other("Invalid namespace scanner usage watch capability scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
mac.update(b"rustfs-ns-scanner-usage-watch-capability-v1\0");
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
mac.update(challenge.as_bytes());
mac.update(server_epoch.as_bytes());
Ok(mac)
}
/// Sign the independent worker capability to validate bucket watches before
/// reusing or durably publishing a complete scanner cache.
pub fn sign_ns_scanner_usage_watch_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
Ok(ns_scanner_usage_watch_capability_mac(&get_shared_secret()?, challenge, server_epoch)?
.finalize()
.into_bytes()
.to_vec())
}
pub fn verify_ns_scanner_usage_watch_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
ns_scanner_usage_watch_capability_mac(&get_shared_secret()?, challenge, server_epoch)?
.verify_slice(proof)
.map_err(|_| std::io::Error::other("Invalid namespace scanner usage watch capability proof"))
}
pub fn verify_ns_scanner_capability_with_usage_capabilities(
challenge: Uuid,
server_epoch: Uuid,
proof: &[u8],
tier: bool,
checkpoint: bool,
progress: bool,
observation: bool,
) -> std::io::Result<()> {
verify_ns_scanner_capability_proof_with_observation(
&get_shared_secret()?,
challenge,
server_epoch,
proof,
tier,
checkpoint,
progress,
observation,
)
}
@@ -2135,13 +2279,119 @@ mod tests {
let secret = "test-scanner-capability-secret";
let challenge = Uuid::new_v4();
let server_epoch = Uuid::new_v4();
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false)
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false, false)
.expect("capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false, false).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false, false).is_err());
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false, false).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false, false).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false, false, false).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false, false, false).is_err());
assert!(
verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false, false, false).is_err()
);
}
#[test]
fn namespace_scanner_watch_capability_is_independent_and_rejects_replay() {
let _ = rustfs_credentials::set_global_rpc_secret("usage-watch-capability-test-secret".into());
let challenge = Uuid::new_v4();
let epoch = Uuid::new_v4();
let proof = sign_ns_scanner_usage_watch_capability(challenge, epoch).expect("watch capability");
verify_ns_scanner_usage_watch_capability(challenge, epoch, &proof).expect("exact watch capability scope");
assert!(verify_ns_scanner_usage_watch_capability(Uuid::new_v4(), epoch, &proof).is_err());
assert!(verify_ns_scanner_usage_watch_capability(challenge, Uuid::new_v4(), &proof).is_err());
for progress in [false, true] {
for observation in [false, true] {
let other =
sign_ns_scanner_capability_with_usage_capabilities(challenge, epoch, true, true, progress, observation)
.expect("independent older capabilities");
assert!(verify_ns_scanner_usage_watch_capability(challenge, epoch, &other).is_err());
assert!(
verify_ns_scanner_capability_with_usage_capabilities(
challenge,
epoch,
&proof,
true,
true,
progress,
observation
)
.is_err()
);
}
}
}
#[test]
fn namespace_scanner_observation_and_progress_scopes_are_independent() {
let challenge = Uuid::new_v4();
let epoch = Uuid::new_v4();
let secret = "independent-usage-capabilities";
for progress in [false, true] {
for observation in [false, true] {
let proof = generate_ns_scanner_capability_proof_with_observation(
secret,
challenge,
epoch,
true,
true,
progress,
observation,
)
.expect("proof");
assert!(
verify_ns_scanner_capability_proof_with_observation(
secret,
challenge,
epoch,
&proof,
true,
true,
progress,
observation
)
.is_ok()
);
assert!(
verify_ns_scanner_capability_proof_with_observation(
secret,
challenge,
epoch,
&proof,
true,
true,
!progress,
observation
)
.is_err()
);
assert!(
verify_ns_scanner_capability_proof_with_observation(
secret,
challenge,
epoch,
&proof,
true,
true,
progress,
!observation
)
.is_err()
);
}
}
}
#[test]
fn namespace_scanner_capability_proof_binds_durable_progress_independently() {
let challenge = Uuid::new_v4();
let epoch = Uuid::new_v4();
let secret = "durable-progress-test";
for supported in [false, true] {
let proof = generate_ns_scanner_capability_proof(secret, challenge, epoch, true, true, supported).expect("proof");
assert!(verify_ns_scanner_capability_proof(secret, challenge, epoch, &proof, true, true, supported).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, challenge, epoch, &proof, true, true, !supported).is_err());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), epoch, &proof, true, true, supported).is_err());
}
}
#[test]
@@ -2149,16 +2399,16 @@ mod tests {
let secret = "checkpoint-capability-test-secret";
let challenge = Uuid::new_v4();
let server_epoch = Uuid::new_v4();
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, true)
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, true, false)
.expect("checkpoint capability proof");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, true).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, true).is_err());
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false)
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, true, false).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false, false).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, true, false).is_err());
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false, false)
.expect("legacy generation capability proof");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, true).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &[], true, true).is_err());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, true, true).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, true, false).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &[], true, true, false).is_err());
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, true, true, false).is_err());
}
#[test]
@@ -2166,14 +2416,14 @@ mod tests {
let secret = "test-scanner-capability-secret";
let challenge = Uuid::new_v4();
let server_epoch = Uuid::new_v4();
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false)
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false, false)
.expect("generation capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false).is_err());
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false)
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false, false).is_ok());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false, false).is_err());
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false, false)
.expect("legacy capability proof should be generated");
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, false).is_err());
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, false, false).is_err());
}
/// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,
@@ -14,14 +14,16 @@
use crate::cluster::rpc::{
build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_checkpoint_coverage,
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_capability,
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
verify_ns_scanner_capability_with_usage_capabilities, verify_put_file_capability,
};
use crate::disk::error::{Error, Result};
use crate::disk::{FileReader, FileWriter};
use crate::storage_api_contracts::internode::{
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CHECKPOINT_COVERAGE_QUERY,
NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY,
NS_SCANNER_COMPLETE_OBSERVATION_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_DURABLE_PROGRESS_QUERY,
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY,
NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY,
NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1,
PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY,
PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
@@ -267,6 +269,9 @@ pub struct NsScannerCapabilityRequest {
pub endpoint: String,
pub supports_tier_registry_generation: bool,
pub supports_checkpoint_coverage: bool,
pub supports_durable_progress: bool,
pub supports_complete_observation: bool,
pub supports_usage_watch: bool,
}
/// Data-plane stream opener used by `RemoteDisk`.
@@ -307,6 +312,9 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
proof: Vec::new(),
supports_tier_registry_generation: None,
supports_checkpoint_coverage: None,
supports_durable_progress: None,
supports_complete_observation: None,
usage_watch_proof: None,
})
}
// Interface facet nobody calls yet: every transport implements both, but no
@@ -401,11 +409,53 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
}
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
let request = if request.supports_usage_watch {
match self.probe_ns_scanner_capability_once(&request).await {
Ok(response) => return Ok(response),
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
supports_usage_watch: false,
..request
},
Err(error) => return Err(error),
}
} else {
request
};
let request = if request.supports_complete_observation {
match self.probe_ns_scanner_capability_once(&request).await {
Ok(response) => return Ok(response),
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
supports_complete_observation: false,
supports_usage_watch: false,
..request
},
Err(error) => return Err(error),
}
} else {
request
};
let request = if request.supports_durable_progress {
match self.probe_ns_scanner_capability_once(&request).await {
Ok(response) => return Ok(response),
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
supports_durable_progress: false,
supports_complete_observation: false,
supports_usage_watch: false,
..request
},
Err(error) => return Err(error),
}
} else {
request
};
let request = if request.supports_checkpoint_coverage {
match self.probe_ns_scanner_capability_once(&request).await {
Ok(response) => return Ok(response),
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
supports_checkpoint_coverage: false,
supports_durable_progress: false,
supports_complete_observation: false,
supports_usage_watch: false,
..request
},
Err(error) => return Err(error),
@@ -425,6 +475,9 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
endpoint: request.endpoint.clone(),
supports_tier_registry_generation: false,
supports_checkpoint_coverage: false,
supports_durable_progress: false,
supports_complete_observation: false,
supports_usage_watch: false,
};
match self.probe_ns_scanner_capability_once(&legacy_request).await {
Ok(mut response) => {
@@ -482,13 +535,62 @@ impl TcpHttpInternodeDataTransport {
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
return Err(Error::other("incompatible remote namespace scanner capability response"));
}
if let Err(err) = verify_ns_scanner_capability_with_checkpoint_coverage(
if request.supports_usage_watch {
if let Some(proof) = response.usage_watch_proof.as_deref() {
crate::cluster::rpc::verify_ns_scanner_usage_watch_capability(challenge, response.server_epoch, proof)
.map_err(|_| Error::other("remote namespace scanner usage watch capability authentication failed"))?;
}
} else {
response.usage_watch_proof = None;
}
if let Err(err) = verify_ns_scanner_capability_with_usage_capabilities(
challenge,
response.server_epoch,
&response.proof,
request.supports_tier_registry_generation,
request.supports_checkpoint_coverage,
request.supports_durable_progress,
request.supports_complete_observation,
) {
if request.supports_complete_observation
&& verify_ns_scanner_capability_with_durable_progress(
challenge,
response.server_epoch,
&response.proof,
request.supports_tier_registry_generation,
request.supports_checkpoint_coverage,
request.supports_durable_progress,
)
.is_ok()
{
normalize_ns_scanner_capability_response(
&mut response,
request.supports_tier_registry_generation,
request.supports_checkpoint_coverage,
request.supports_durable_progress,
false,
);
return Ok(response);
}
if request.supports_durable_progress
&& verify_ns_scanner_capability_with_checkpoint_coverage(
challenge,
response.server_epoch,
&response.proof,
request.supports_tier_registry_generation,
request.supports_checkpoint_coverage,
)
.is_ok()
{
normalize_ns_scanner_capability_response(
&mut response,
request.supports_tier_registry_generation,
request.supports_checkpoint_coverage,
false,
false,
);
return Ok(response);
}
if request.supports_checkpoint_coverage
&& verify_ns_scanner_capability_with_tier_registry_generation(
challenge,
@@ -498,7 +600,13 @@ impl TcpHttpInternodeDataTransport {
)
.is_ok()
{
normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation, false);
normalize_ns_scanner_capability_response(
&mut response,
request.supports_tier_registry_generation,
false,
false,
false,
);
return Ok(response);
}
// A permissive older peer can ignore the additive marker and
@@ -508,6 +616,8 @@ impl TcpHttpInternodeDataTransport {
if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) {
response.supports_tier_registry_generation = None;
response.supports_checkpoint_coverage = None;
response.supports_durable_progress = None;
response.supports_complete_observation = None;
return Ok(response);
}
return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}")));
@@ -521,6 +631,8 @@ impl TcpHttpInternodeDataTransport {
&mut response,
request.supports_tier_registry_generation,
request.supports_checkpoint_coverage,
request.supports_durable_progress,
request.supports_complete_observation,
);
Ok(response)
}
@@ -855,9 +967,13 @@ fn normalize_ns_scanner_capability_response(
response: &mut NsScannerCapabilityResponse,
requested_generation_support: bool,
requested_checkpoint_coverage: bool,
requested_durable_progress: bool,
requested_complete_observation: bool,
) {
response.supports_tier_registry_generation = requested_generation_support.then_some(true);
response.supports_checkpoint_coverage = requested_checkpoint_coverage.then_some(true);
response.supports_durable_progress = requested_durable_progress.then_some(true);
response.supports_complete_observation = requested_complete_observation.then_some(true);
}
fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool {
@@ -890,7 +1006,7 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
format!(
"{}{}?{}={}&{}={}{}{}",
"{}{}?{}={}&{}={}{}{}{}{}{}",
request.endpoint,
NS_SCANNER_PATH,
NS_SCANNER_PROTOCOL_VERSION_QUERY,
@@ -906,6 +1022,21 @@ fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challen
format!("&{}=true", NS_SCANNER_CHECKPOINT_COVERAGE_QUERY)
} else {
String::new()
},
if request.supports_durable_progress {
format!("&{}=true", NS_SCANNER_DURABLE_PROGRESS_QUERY)
} else {
String::new()
},
if request.supports_complete_observation {
format!("&{}=true", NS_SCANNER_COMPLETE_OBSERVATION_QUERY)
} else {
String::new()
},
if request.supports_usage_watch {
"&ns_scanner_usage_watch=true".to_string()
} else {
String::new()
}
)
}
@@ -1252,6 +1383,9 @@ mod tests {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: false,
supports_checkpoint_coverage: false,
supports_durable_progress: false,
supports_complete_observation: false,
supports_usage_watch: false,
})
.await
.expect_err("legacy transport should report namespace scanner as unsupported");
@@ -1278,6 +1412,106 @@ mod tests {
assert!(matches!(open_err, Error::MethodNotAllowed));
}
#[tokio::test]
async fn namespace_scanner_watch_capability_preserves_older_capabilities_and_rejects_stale_proof() {
crate::runtime::sources::ensure_test_rpc_secret();
for peer in ["current", "strict-old", "missing-watch", "stale-watch"] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("watch capability listener");
let address = listener.local_addr().expect("listener address");
let epoch = Uuid::new_v4();
let server = tokio::spawn(async move {
let requests = if peer == "strict-old" { 2 } else { 1 };
for index in 0..requests {
let (mut socket, _) = listener.accept().await.expect("watch capability request");
let mut bytes = Vec::new();
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
let mut chunk = [0; 1024];
let read = socket.read(&mut chunk).await.expect("request headers");
assert!(read > 0 && bytes.len() < 16 * 1024);
bytes.extend_from_slice(&chunk[..read]);
}
let headers = std::str::from_utf8(&bytes).expect("HTTP headers");
let target = headers.split_whitespace().nth(1).expect("target");
let url = reqwest::Url::parse(&format!("http://{address}{target}")).expect("request URL");
let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
assert_eq!(query.contains_key("ns_scanner_usage_watch"), index == 0);
assert!(query.contains_key(NS_SCANNER_DURABLE_PROGRESS_QUERY));
assert!(query.contains_key(NS_SCANNER_COMPLETE_OBSERVATION_QUERY));
if peer == "strict-old" && index == 0 {
socket
.write_all(b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
.await
.expect("old parser rejects only the new field");
continue;
}
let challenge = query[NS_SCANNER_CAPABILITY_CHALLENGE_QUERY].parse().expect("challenge");
let proof = crate::cluster::rpc::sign_ns_scanner_capability_with_usage_capabilities(
challenge, epoch, true, true, true, true,
)
.expect("prior capability proof");
let usage_watch_proof = match peer {
"current" => Some(
crate::cluster::rpc::sign_ns_scanner_usage_watch_capability(challenge, epoch).expect("watch proof"),
),
"stale-watch" => Some(
crate::cluster::rpc::sign_ns_scanner_usage_watch_capability(Uuid::new_v4(), epoch)
.expect("stale watch proof"),
),
_ => None,
};
let body = rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch: epoch,
proof,
supports_tier_registry_generation: Some(true),
supports_checkpoint_coverage: Some(true),
supports_durable_progress: Some(true),
supports_complete_observation: Some(true),
usage_watch_proof,
})
.expect("capability body");
socket
.write_all(
format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", body.len()).as_bytes(),
)
.await
.expect("response headers");
socket.write_all(&body).await.expect("response body");
}
requests
});
let response = tokio::time::timeout(
Duration::from_secs(10),
TcpHttpInternodeDataTransport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
endpoint: format!("http://{address}"),
supports_tier_registry_generation: true,
supports_checkpoint_coverage: true,
supports_durable_progress: true,
supports_complete_observation: true,
supports_usage_watch: true,
}),
)
.await
.expect("bounded probe");
if peer == "stale-watch" {
assert!(
response
.expect_err("stale response must fail authentication")
.to_string()
.contains("authentication failed")
);
} else {
let response = response.expect("verified capability result");
assert_eq!(response.usage_watch_proof.is_some(), peer == "current");
assert_eq!(response.supports_durable_progress, Some(true));
assert_eq!(response.supports_complete_observation, Some(true));
}
assert_eq!(server.await.expect("capability server"), if peer == "strict-old" { 2 } else { 1 });
}
}
#[tokio::test]
async fn namespace_scanner_checkpoint_capability_authenticates_new_and_legacy_peers() {
crate::runtime::sources::ensure_test_rpc_secret();
@@ -1330,6 +1564,9 @@ mod tests {
supports_tier_registry_generation: Some(true),
// Strip a supported bit, or forge it for an older peer.
supports_checkpoint_coverage: (peer != "current").then_some(true),
supports_durable_progress: None,
supports_complete_observation: None,
usage_watch_proof: None,
})
.expect("capability response");
socket
@@ -1348,6 +1585,9 @@ mod tests {
endpoint: format!("http://{address}"),
supports_tier_registry_generation: true,
supports_checkpoint_coverage: true,
supports_durable_progress: false,
supports_complete_observation: false,
supports_usage_watch: false,
}),
)
.await
@@ -1369,6 +1609,206 @@ mod tests {
}
}
#[tokio::test]
async fn namespace_scanner_durable_progress_capability_authenticates_new_and_legacy_peers() {
crate::runtime::sources::ensure_test_rpc_secret();
for peer in ["current", "strict-old", "permissive-old", "invalid-proof", "missing-proof"] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("capability listener");
let address = listener.local_addr().expect("capability listener address");
let server_epoch = Uuid::new_v4();
let server = tokio::spawn(async move {
let requests = if peer == "strict-old" { 2 } else { 1 };
for index in 0..requests {
let (mut socket, _) = listener.accept().await.expect("capability request");
let mut bytes = Vec::new();
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
let mut chunk = [0; 1024];
let read = socket.read(&mut chunk).await.expect("capability headers");
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete capability headers");
bytes.extend_from_slice(&chunk[..read]);
}
let request = std::str::from_utf8(&bytes).expect("HTTP headers");
let target = request.split_whitespace().nth(1).expect("HTTP request target");
let url = reqwest::Url::parse(&format!("http://{address}{target}")).expect("capability URL");
let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
assert_eq!(query.get(NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY).map(String::as_str), Some("true"));
assert_eq!(query.contains_key(NS_SCANNER_DURABLE_PROGRESS_QUERY), index == 0);
if peer == "strict-old" && index == 0 {
socket
.write_all(b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
.await
.expect("legacy parser rejection");
continue;
}
let challenge = query[NS_SCANNER_CAPABILITY_CHALLENGE_QUERY].parse().expect("challenge UUID");
let proof = match peer {
"invalid-proof" => vec![0; 32],
"missing-proof" => Vec::new(),
_ => crate::cluster::rpc::sign_ns_scanner_capability_with_durable_progress(
challenge,
server_epoch,
true,
true,
peer == "current",
)
.expect("server capability proof"),
};
let body = rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch,
proof,
supports_tier_registry_generation: Some(true),
// Strip a supported bit, or forge it for an older peer.
supports_checkpoint_coverage: Some(true),
supports_durable_progress: (peer != "current").then_some(true),
supports_complete_observation: None,
usage_watch_proof: None,
})
.expect("capability response");
socket
.write_all(
format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", body.len()).as_bytes(),
)
.await
.expect("capability response headers");
socket.write_all(&body).await.expect("capability response body");
}
requests
});
let response = tokio::time::timeout(
Duration::from_secs(10),
TcpHttpInternodeDataTransport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
endpoint: format!("http://{address}"),
supports_tier_registry_generation: true,
supports_checkpoint_coverage: true,
supports_durable_progress: true,
supports_complete_observation: false,
supports_usage_watch: false,
}),
)
.await
.expect("bounded capability probe");
if peer.ends_with("proof") {
assert!(
response
.expect_err("unverified capability must fail closed")
.to_string()
.contains("authentication failed")
);
} else {
let response = response.expect("authenticated capability");
assert_eq!(response.server_epoch, server_epoch);
assert_eq!(response.supports_tier_registry_generation, Some(true));
assert_eq!(response.supports_checkpoint_coverage, Some(true));
assert_eq!(response.supports_durable_progress, (peer == "current").then_some(true));
}
assert_eq!(server.await.expect("capability server"), if peer == "strict-old" { 2 } else { 1 });
}
}
#[tokio::test]
async fn namespace_scanner_complete_observation_capability_authenticates_new_and_legacy_peers() {
crate::runtime::sources::ensure_test_rpc_secret();
for peer in ["current", "strict-old", "permissive-old", "invalid-proof", "missing-proof"] {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("capability listener");
let address = listener.local_addr().expect("capability listener address");
let server_epoch = Uuid::new_v4();
let server = tokio::spawn(async move {
let requests = if peer == "strict-old" { 2 } else { 1 };
for index in 0..requests {
let (mut socket, _) = listener.accept().await.expect("capability request");
let mut bytes = Vec::new();
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
let mut chunk = [0; 1024];
let read = socket.read(&mut chunk).await.expect("capability headers");
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete capability headers");
bytes.extend_from_slice(&chunk[..read]);
}
let request = std::str::from_utf8(&bytes).expect("HTTP headers");
let target = request.split_whitespace().nth(1).expect("HTTP request target");
let url = reqwest::Url::parse(&format!("http://{address}{target}")).expect("capability URL");
let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
assert_eq!(query.get(NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY).map(String::as_str), Some("true"));
assert_eq!(query.contains_key(NS_SCANNER_COMPLETE_OBSERVATION_QUERY), index == 0);
if peer == "strict-old" && index == 0 {
socket
.write_all(b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
.await
.expect("legacy parser rejection");
continue;
}
let challenge = query[NS_SCANNER_CAPABILITY_CHALLENGE_QUERY].parse().expect("challenge UUID");
let proof = match peer {
"invalid-proof" => vec![0; 32],
"missing-proof" => Vec::new(),
_ => crate::cluster::rpc::sign_ns_scanner_capability_with_usage_capabilities(
challenge,
server_epoch,
true,
true,
true,
peer == "current",
)
.expect("server capability proof"),
};
let body = rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
version: NS_SCANNER_PROTOCOL_VERSION,
server_epoch,
proof,
supports_tier_registry_generation: Some(true),
// Strip a supported bit, or forge it for an older peer.
supports_checkpoint_coverage: Some(true),
supports_durable_progress: Some(true),
supports_complete_observation: (peer != "current").then_some(true),
usage_watch_proof: None,
})
.expect("capability response");
socket
.write_all(
format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", body.len()).as_bytes(),
)
.await
.expect("capability response headers");
socket.write_all(&body).await.expect("capability response body");
}
requests
});
let response = tokio::time::timeout(
Duration::from_secs(10),
TcpHttpInternodeDataTransport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
endpoint: format!("http://{address}"),
supports_tier_registry_generation: true,
supports_checkpoint_coverage: true,
supports_durable_progress: true,
supports_complete_observation: true,
supports_usage_watch: false,
}),
)
.await
.expect("bounded capability probe");
if peer.ends_with("proof") {
assert!(
response
.expect_err("unverified capability must fail closed")
.to_string()
.contains("authentication failed")
);
} else {
let response = response.expect("authenticated capability");
assert_eq!(response.server_epoch, server_epoch);
assert_eq!(response.supports_tier_registry_generation, Some(true));
assert_eq!(response.supports_checkpoint_coverage, Some(true));
assert_eq!(response.supports_durable_progress, Some(true));
assert_eq!(response.supports_complete_observation, (peer == "current").then_some(true));
}
assert_eq!(server.await.expect("capability server"), if peer == "strict-old" { 2 } else { 1 });
}
}
#[tokio::test]
async fn walk_dir_restores_explicit_remote_missing_error() {
let _ = rustfs_credentials::set_global_rpc_secret("walk-dir-error-test-secret".to_string());
@@ -2112,6 +2552,9 @@ mod tests {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: false,
supports_checkpoint_coverage: false,
supports_durable_progress: false,
supports_complete_observation: false,
supports_usage_watch: false,
},
challenge,
);
@@ -2132,6 +2575,9 @@ mod tests {
endpoint: "http://node1:9000".to_string(),
supports_tier_registry_generation: true,
supports_checkpoint_coverage: false,
supports_durable_progress: false,
supports_complete_observation: false,
supports_usage_watch: false,
},
challenge,
);
@@ -2171,13 +2617,16 @@ mod tests {
proof: Vec::new(),
supports_tier_registry_generation: None,
supports_checkpoint_coverage: None,
supports_durable_progress: None,
supports_complete_observation: None,
usage_watch_proof: None,
};
normalize_ns_scanner_capability_response(&mut response, true, true);
normalize_ns_scanner_capability_response(&mut response, true, true, false, false);
assert_eq!(response.supports_tier_registry_generation, Some(true));
response.supports_tier_registry_generation = Some(false);
normalize_ns_scanner_capability_response(&mut response, false, false);
normalize_ns_scanner_capability_response(&mut response, false, false, false, false);
assert_eq!(response.supports_tier_registry_generation, None);
}
@@ -2192,6 +2641,9 @@ mod tests {
.expect("placeholder proof should be generated"),
supports_tier_registry_generation: None,
supports_checkpoint_coverage: None,
supports_durable_progress: None,
supports_complete_observation: None,
usage_watch_proof: None,
};
// A proof bound to a different challenge cannot authorize the legacy
// fallback, even though the response has the expected shape.
+9 -6
View File
@@ -40,12 +40,15 @@ pub use http_auth::{
gen_tonic_signature_headers, normalize_tonic_rpc_audience, rpc_auth_is_resource_exhausted, rpc_background_admission,
set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest,
set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
sign_ns_scanner_capability_with_durable_progress, sign_ns_scanner_capability_with_tier_registry_generation,
sign_ns_scanner_capability_with_usage_capabilities, sign_ns_scanner_usage_watch_capability, sign_put_file_capability,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
verify_ns_scanner_capability_with_usage_capabilities, verify_ns_scanner_usage_watch_capability, verify_put_file_auth_trailer,
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
};
#[cfg(test)]
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
@@ -97,6 +97,7 @@ const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
const SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT: Duration = Duration::from_secs(5);
const SCANNER_USAGE_WATCH_PEER_CONCURRENCY: usize = 4;
/// Reserve time for the acquire response's network/clock uncertainty. The
/// server owns the real expiry; this local deadline is intentionally earlier
/// so a coordinator never starts a bounded persistence operation at the edge
@@ -298,6 +299,22 @@ fn scanner_scoped_dirty_usage_ack_payload(
Ok(payload)
}
/// Authentication binds a response to the complete current request before any
/// returned watch can be consumed. Semantic validation still rejects a signed
/// response for the wrong producer session, bucket, or token.
fn validate_scanner_usage_watch_response(
request: &rustfs_protos::proto_gen::node_service::ScannerUsageWatchRequest,
response: &rustfs_protos::proto_gen::node_service::ScannerUsageWatchResponse,
) -> Result<()> {
use rustfs_protos::usage_watch::{
canonical_usage_watch_request, canonical_usage_watch_response, validate_usage_watch_response,
};
let canonical = canonical_usage_watch_request(request).map_err(|error| Error::other(error.to_string()))?;
let body = canonical_usage_watch_response(&canonical, response).map_err(|error| Error::other(error.to_string()))?;
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
validate_usage_watch_response(request, response).map_err(|error| Error::other(error.to_string()))
}
fn scanner_scoped_dirty_usage_ack_response_matches(
request: &ScannerScopedDirtyUsageAckRequest,
response: &ScannerScopedDirtyUsageAckResponse,
@@ -741,6 +758,9 @@ pub struct PeerRestClient {
topology_member: String,
offline: Arc<AtomicBool>,
recovery_running: Arc<AtomicBool>,
/// All clones used by admission, keepalive and root verification share
/// this peer's bounded sender capacity.
scanner_usage_watch_admission: Arc<tokio::sync::Semaphore>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -900,9 +920,20 @@ impl PeerRestClient {
topology_member,
offline: Arc::new(AtomicBool::new(false)),
recovery_running: Arc::new(AtomicBool::new(false)),
scanner_usage_watch_admission: Arc::new(tokio::sync::Semaphore::new(SCANNER_USAGE_WATCH_PEER_CONCURRENCY)),
}
}
async fn acquire_usage_watch_send_permit(&self) -> Result<tokio::sync::OwnedSemaphorePermit> {
timeout(
SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT,
self.scanner_usage_watch_admission.clone().acquire_owned(),
)
.await
.map_err(|_| Error::other("usage watch sender admission timed out"))?
.map_err(|_| Error::other("usage watch sender admission is closed"))
}
fn parse_topology_host(peer_host_port: &str, grid_host: &str) -> Result<XHost> {
let url = url::Url::parse(grid_host).map_err(|_| Error::other("peer grid host is not a valid URL"))?;
if !matches!(url.scheme(), "http" | "https")
@@ -1639,6 +1670,51 @@ impl PeerRestClient {
Err(Error::NotImplemented)
}
/// Routes one authorized MRF operation without retrying ambiguous mutations.
/// `None` means the selected node predates this protocol; it is never an empty result.
pub async fn mrf_lifecycle(
&self,
topology: String,
node_id: String,
command: Vec<u8>,
timeout: std::time::Duration,
) -> Result<Option<Vec<u8>>> {
use rustfs_protos::proto_gen::node_service::MrfLifecycleRequest;
use rustfs_protos::{MRF_LIFECYCLE_MAX_COMMAND_BYTES, MRF_LIFECYCLE_MAX_MESSAGE_BYTES, MRF_LIFECYCLE_PROTOCOL_VERSION};
if command.len() > MRF_LIFECYCLE_MAX_COMMAND_BYTES {
return Err(Error::other("MRF command exceeds size limit"));
}
let payload = MrfLifecycleRequest {
protocol_version: MRF_LIFECYCLE_PROTOCOL_VERSION,
topology_fingerprint: topology,
node_id,
command: command.into(),
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
};
let mut request = Request::new(payload.clone());
request.set_timeout(timeout);
set_tonic_mutation_body_digest(&mut request)?;
let mut client = self
.get_client()
.await?
.max_decoding_message_size(MRF_LIFECYCLE_MAX_MESSAGE_BYTES)
.max_encoding_message_size(MRF_LIFECYCLE_MAX_MESSAGE_BYTES);
let response = match client.mrf_lifecycle(request).await {
Ok(response) => response.into_inner(),
// RUSTFS_COMPAT_TODO(backlog-3177): old peers lack MRF management. Remove after every supported peer implements this RPC.
Err(status) if status.code() == tonic::Code::Unimplemented => return Ok(None),
Err(status) => return Err(status.into()),
};
let canonical = rustfs_protos::canonical_mrf_lifecycle_response_body(&payload, &response)
.map_err(|_| Error::other("MRF response is too large to authenticate"))?;
verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
.map_err(|_| Error::other("MRF response authentication failed"))?;
if response.protocol_version != MRF_LIFECYCLE_PROTOCOL_VERSION {
return Err(Error::other("unsupported MRF response protocol"));
}
Ok(Some(response.result.to_vec()))
}
pub async fn background_heal_status(&self) -> Result<Option<Vec<u8>>> {
self.finalize_result(
async {
@@ -2429,6 +2505,129 @@ impl PeerRestClient {
.await
}
/// Independently negotiated bucket watches; an old peer returns an error
/// and callers retain the global-generation fallback for that bucket.
pub async fn scanner_usage_watch(
&self,
owner_id: Uuid,
mut entries: Vec<crate::runtime::usage_watch::ScannerUsageWatchRequest>,
) -> Result<Vec<crate::runtime::usage_watch::ScannerUsageWatchObservation>> {
use crate::runtime::usage_watch::{
SCANNER_USAGE_WATCH_COVERAGE_VERSION, SCANNER_USAGE_WATCH_LEASE_MS, ScannerUsageWatchKnown,
ScannerUsageWatchObservation,
};
use rustfs_protos::proto_gen::node_service::{ScannerUsageWatchEntry, ScannerUsageWatchRequest};
use rustfs_protos::usage_watch::*;
entries.sort_by(|left, right| left.bucket.cmp(&right.bucket));
if owner_id.is_nil() || entries.windows(2).any(|pair| pair[0].bucket == pair[1].bucket) {
return Err(Error::other("usage watch request contains duplicate buckets or an invalid owner"));
}
// Local queue pressure is not a network failure. Keep admission errors
// outside finalize_result so they cannot mark a healthy peer offline.
let _send_permit = self.acquire_usage_watch_send_permit().await?;
self.finalize_result(
async {
let mut client = super::client::scanner_control_time_out_client(
&self.grid_host,
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
)
.await?;
let mut observations = Vec::with_capacity(entries.len());
for batch in entries.chunks(USAGE_WATCH_MAX_ENTRIES) {
let payload = ScannerUsageWatchRequest {
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
protocol_version: USAGE_WATCH_PROTOCOL_VERSION,
owner_id: owner_id.to_string(),
entries: batch
.iter()
.map(|entry| ScannerUsageWatchEntry {
bucket: entry.bucket.clone(),
bucket_incarnation: entry.bucket_incarnation.as_bytes().to_vec().into(),
watch_id: entry.watch_id.map_or_else(Bytes::new, |id| id.as_bytes().to_vec().into()),
release: entry.release,
session_id: entry.session_id.map_or_else(Bytes::new, |id| id.as_bytes().to_vec().into()),
})
.collect(),
};
let canonical = canonical_usage_watch_request(&payload).map_err(|error| Error::other(error.to_string()))?;
let mut request = Request::new(payload.clone());
set_tonic_canonical_body_digest(&mut request, &canonical)?;
let started = std::time::Instant::now();
let response = timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT, client.scanner_usage_watch(request))
.await
.map_err(|_| Error::other("usage watch request timed out"))??
.into_inner();
let budget_used = u64::try_from(started.elapsed().as_millis())
.unwrap_or(u64::MAX)
.saturating_add(250);
validate_scanner_usage_watch_response(&payload, &response)?;
let session = Uuid::from_slice(response.session_id.as_ref())
.map_err(|_| Error::other("invalid usage watch session"))?;
if response.protocol_version != USAGE_WATCH_PROTOCOL_VERSION
|| response.owner_id != payload.owner_id
|| session.is_nil()
|| response.entries.len() != payload.entries.len()
{
return Err(Error::other("usage watch response does not match request"));
}
for (requested, entry) in batch.iter().zip(response.entries) {
if entry.bucket != requested.bucket {
return Err(Error::other("usage watch response bucket order differs"));
}
let known = if entry.known {
let token = Uuid::from_slice(entry.watch_id.as_ref())
.map_err(|_| Error::other("invalid usage watch token"))?;
if token.is_nil()
|| entry.bucket_incarnation.as_ref() != requested.bucket_incarnation.as_bytes()
|| requested.watch_id.is_some_and(|expected| expected != token)
|| entry.revision == 0
|| entry.revision == u64::MAX
|| entry.inflight == u64::MAX
|| entry.coverage_version != SCANNER_USAGE_WATCH_COVERAGE_VERSION
|| entry.coverage_epoch == 0
|| entry.lease_remaining_ms == 0
|| entry.lease_remaining_ms > SCANNER_USAGE_WATCH_LEASE_MS
{
return Err(Error::other("usage watch response contains an invalid proof"));
}
let lease_remaining_ms = entry.lease_remaining_ms.saturating_sub(budget_used);
(lease_remaining_ms != 0).then_some(ScannerUsageWatchKnown {
owner_id,
session_id: session,
watch_id: token,
bucket_incarnation: requested.bucket_incarnation,
revision: entry.revision,
inflight: entry.inflight,
coverage_version: entry.coverage_version,
coverage_epoch: entry.coverage_epoch,
lease_remaining_ms,
})
} else {
if !entry.watch_id.is_empty()
|| !entry.bucket_incarnation.is_empty()
|| entry.revision != 0
|| entry.inflight != 0
|| entry.coverage_version != 0
|| entry.coverage_epoch != 0
|| entry.lease_remaining_ms != 0
{
return Err(Error::other("Unknown usage watch entry contains proof fields"));
}
None
};
observations.push(ScannerUsageWatchObservation {
bucket: entry.bucket,
known,
});
}
}
Ok(observations)
}
.await,
)
.await
}
pub async fn acknowledge_scanner_scoped_dirty_usage(
&self,
owner_id: String,
@@ -3263,6 +3462,74 @@ mod tests {
)
}
#[tokio::test]
async fn usage_watch_client_clones_share_receiver_sized_send_admission() {
let receiver = Arc::new(tokio::sync::Semaphore::new(4));
let occupied = (0..4)
.map(|_| receiver.clone().try_acquire_owned().expect("four receiver slots"))
.collect::<Vec<_>>();
assert!(receiver.try_acquire().is_err(), "an uncoordinated fifth request is rejected");
drop(occupied);
let client = test_peer_client();
let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let peak = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let tasks = (0..12).map(|_| {
// Simulate the shared client's independent admission, keeper and
// publication callers. Each uses the actual production gate.
let client = client.clone();
let receiver = receiver.clone();
let active = active.clone();
let peak = peak.clone();
tokio::spawn(async move {
let _send = client
.acquire_usage_watch_send_permit()
.await
.expect("bounded sender admission");
let _receive = receiver
.try_acquire_owned()
.expect("coexisting client clones must never send more than the receiver's four slots");
let count = active.fetch_add(1, Ordering::AcqRel) + 1;
peak.fetch_max(count, Ordering::AcqRel);
tokio::time::sleep(Duration::from_millis(10)).await;
active.fetch_sub(1, Ordering::AcqRel);
})
});
for result in futures::future::join_all(tasks).await {
result.expect("all queued callers finish");
}
assert_eq!(peak.load(Ordering::Acquire), 4);
assert_eq!(active.load(Ordering::Acquire), 0);
assert_eq!(client.scanner_usage_watch_admission.available_permits(), 4);
assert_eq!(receiver.available_permits(), 4);
}
#[tokio::test(start_paused = true)]
async fn usage_watch_sender_queue_timeout_does_not_mark_the_peer_offline() {
let client = test_peer_client();
let _occupied = client
.scanner_usage_watch_admission
.clone()
.acquire_many_owned(4)
.await
.unwrap();
let result = client
.scanner_usage_watch(
Uuid::from_u128(1),
vec![crate::runtime::usage_watch::ScannerUsageWatchRequest {
bucket: "bucket".into(),
bucket_incarnation: Uuid::from_u128(2),
watch_id: None,
session_id: None,
release: false,
}],
)
.await;
assert!(result.unwrap_err().to_string().contains("sender admission timed out"));
assert!(!client.offline.load(Ordering::Acquire));
assert!(!client.recovery_running.load(Ordering::Acquire));
}
fn decode_test_scanner_activity(response: ScannerActivityResponse) -> Result<ScannerPeerActivity> {
decode_scanner_activity_with_verifier(response, &[9; 16], |_canonical, proof| {
(proof == b"proof")
@@ -4338,6 +4605,112 @@ mod tests {
assert!(!client.offline.load(Ordering::Acquire));
}
#[test]
fn usage_watch_consumer_authenticates_the_current_request_and_rejects_replay() {
use rustfs_protos::proto_gen::node_service::{
ScannerUsageWatchEntry, ScannerUsageWatchRequest, ScannerUsageWatchResponse, ScannerUsageWatchResult,
};
use rustfs_protos::usage_watch::{canonical_usage_watch_request, canonical_usage_watch_response};
runtime_sources::ensure_test_rpc_secret();
let request = ScannerUsageWatchRequest {
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
protocol_version: 1,
owner_id: Uuid::new_v4().to_string(),
entries: vec![ScannerUsageWatchEntry {
bucket: "usage-watch".into(),
bucket_incarnation: Uuid::new_v4().as_bytes().to_vec().into(),
watch_id: Uuid::new_v4().as_bytes().to_vec().into(),
session_id: Uuid::new_v4().as_bytes().to_vec().into(),
release: false,
}],
};
let sign = |request: &ScannerUsageWatchRequest, response: &mut ScannerUsageWatchResponse| {
let canonical = canonical_usage_watch_request(request).expect("valid request to sign");
let body = canonical_usage_watch_response(&canonical, response).expect("bounded canonical watch response");
response.response_proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&body)
.expect("real response HMAC")
.into();
};
let mut response = ScannerUsageWatchResponse {
protocol_version: 1,
owner_id: request.owner_id.clone(),
session_id: request.entries[0].session_id.clone(),
entries: vec![ScannerUsageWatchResult {
bucket: request.entries[0].bucket.clone(),
known: true,
bucket_incarnation: request.entries[0].bucket_incarnation.clone(),
watch_id: request.entries[0].watch_id.clone(),
revision: 7,
inflight: 0,
coverage_version: 1,
coverage_epoch: 3,
lease_remaining_ms: 60_000,
}],
response_proof: Bytes::new(),
};
sign(&request, &mut response);
validate_scanner_usage_watch_response(&request, &response).expect("current request accepts its signed response");
let mut rejected_replays = 0;
for field in 0..8 {
let mut current = request.clone();
match field {
0 => current.challenge = Uuid::new_v4().as_bytes().to_vec().into(),
1 => current.owner_id = Uuid::new_v4().to_string(),
2 => current.entries[0].bucket.push('x'),
3 => current.entries[0].bucket_incarnation = Uuid::new_v4().as_bytes().to_vec().into(),
4 => current.entries[0].watch_id = Uuid::new_v4().as_bytes().to_vec().into(),
5 => current.entries[0].session_id = Uuid::new_v4().as_bytes().to_vec().into(),
6 => current.entries[0].release = true,
_ => current.protocol_version += 1,
}
assert!(
validate_scanner_usage_watch_response(&current, &response).is_err(),
"a previously signed response must not authorize request mutation {field}"
);
rejected_replays += 1;
}
let mut tampered = response.clone();
tampered.entries[0].revision += 1;
assert!(validate_scanner_usage_watch_response(&request, &tampered).is_err());
let mut unsigned = response.clone();
unsigned.response_proof = Bytes::new();
assert!(validate_scanner_usage_watch_response(&request, &unsigned).is_err());
let mut rejected_signed_mismatches = 0;
for field in 0..5 {
let mut mismatched = response.clone();
match field {
0 => mismatched.owner_id = Uuid::new_v4().to_string(),
1 => mismatched.session_id = Uuid::new_v4().as_bytes().to_vec().into(),
2 => mismatched.entries[0].bucket_incarnation = Uuid::new_v4().as_bytes().to_vec().into(),
3 => mismatched.entries[0].watch_id = Uuid::new_v4().as_bytes().to_vec().into(),
_ => mismatched.entries[0].known = false,
}
sign(&request, &mut mismatched);
assert!(
validate_scanner_usage_watch_response(&request, &mismatched).is_err(),
"an authentic HMAC cannot replace requested watch identity or authorize malformed Unknown: {field}"
);
rejected_signed_mismatches += 1;
}
println!(
"usage_assertion: {}",
serde_json::json!({
"case_id": "USAGE-12", "assertion_id": "USAGE-12-A07", "status": "PASS",
"evidence": {"current_response_authenticated": true, "replayed_requests_rejected": rejected_replays,
"signed_identity_mismatches_rejected": rejected_signed_mismatches,
"tampered_response_rejected": true, "missing_hmac_rejected": true}
})
);
drop((request, response, tampered, unsigned));
println!(
"usage_cleanup: {}",
serde_json::json!({"case_id": "USAGE-12", "status": "PASS", "scope": "process-local"})
);
}
#[test]
fn heal_control_capability_proof_must_authenticate_exact_ack() {
runtime_sources::ensure_test_rpc_secret();
+21 -1
View File
@@ -1221,6 +1221,15 @@ impl RemoteDisk {
}
pub(crate) async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
Ok(self
.ns_scanner_progress_capability()
.await?
.map(|capability| capability.server_epoch))
}
pub(crate) async fn ns_scanner_progress_capability(
&self,
) -> Result<Option<crate::storage_api_contracts::internode::NsScannerCapabilities>> {
if self.health.is_faulty() {
return Err(DiskError::FaultyDisk);
}
@@ -1228,6 +1237,9 @@ impl RemoteDisk {
endpoint: self.endpoint.grid_host(),
supports_tier_registry_generation: true,
supports_checkpoint_coverage: true,
supports_durable_progress: true,
supports_complete_observation: true,
supports_usage_watch: true,
});
let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
.await
@@ -1238,7 +1250,12 @@ impl RemoteDisk {
if response.supports_tier_registry_generation == Some(true)
&& response.supports_checkpoint_coverage == Some(true) =>
{
Ok(Some(response.server_epoch))
Ok(Some(crate::storage_api_contracts::internode::NsScannerCapabilities {
server_epoch: response.server_epoch,
supports_durable_progress: response.supports_durable_progress == Some(true),
supports_complete_observation: response.supports_complete_observation == Some(true),
supports_usage_watch: response.usage_watch_proof.is_some(),
}))
}
Ok(_) => Ok(None),
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
@@ -7065,6 +7082,9 @@ mod tests {
proof: Vec::new(),
supports_tier_registry_generation,
supports_checkpoint_coverage: *self.ns_scanner_checkpoint_support.lock().expect("checkpoint capability lock"),
supports_durable_progress: None,
supports_complete_observation: None,
usage_watch_proof: None,
})
}
+10
View File
@@ -314,6 +314,16 @@ mod tests {
assert_eq!(scanner_kvs.get(SCANNER_DELAY), "");
assert_eq!(scanner_kvs.get(SCANNER_MAX_WAIT), "");
assert_eq!(scanner_kvs.get(SCANNER_CYCLE_MAX_OBJECTS), "0");
for key in [
rustfs_config::SCANNER_BUCKET_QUANTUM_DURATION,
rustfs_config::SCANNER_BUCKET_QUANTUM_OBJECTS,
rustfs_config::SCANNER_BUCKET_QUANTUM_DIRECTORIES,
rustfs_config::SCANNER_BUCKET_QUANTUM_ENTRIES,
rustfs_config::SCANNER_BUCKET_QUANTUM_NAME_BYTES,
] {
assert!(rustfs_config::SCANNER_KEYS.contains(&key), "admin key {key} is registered");
assert_eq!(scanner_kvs.get(key), "0", "quantum dimension {key} stays disabled until rollout");
}
let heal_kvs = cfg
.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER)
+31
View File
@@ -25,6 +25,12 @@ use rustfs_config::{
SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_RECOVERY_BACKOFF_CAP, SCANNER_SPEED, SCANNER_START_DELAY,
SCANNER_WALKERS_PER_DISK, SCANNER_YIELD_EVERY_N_OBJECTS,
};
use rustfs_config::{
DEFAULT_SCANNER_BUCKET_QUANTUM_DIRECTORIES, DEFAULT_SCANNER_BUCKET_QUANTUM_DURATION_SECS,
DEFAULT_SCANNER_BUCKET_QUANTUM_ENTRIES, DEFAULT_SCANNER_BUCKET_QUANTUM_NAME_BYTES, DEFAULT_SCANNER_BUCKET_QUANTUM_OBJECTS,
SCANNER_BUCKET_QUANTUM_DIRECTORIES, SCANNER_BUCKET_QUANTUM_DURATION, SCANNER_BUCKET_QUANTUM_ENTRIES,
SCANNER_BUCKET_QUANTUM_NAME_BYTES, SCANNER_BUCKET_QUANTUM_OBJECTS,
};
use std::sync::LazyLock;
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
@@ -69,6 +75,31 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
value: DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES.to_string(),
hidden_if_empty: false,
},
KV {
key: SCANNER_BUCKET_QUANTUM_DURATION.to_owned(),
value: DEFAULT_SCANNER_BUCKET_QUANTUM_DURATION_SECS.to_string(),
hidden_if_empty: false,
},
KV {
key: SCANNER_BUCKET_QUANTUM_OBJECTS.to_owned(),
value: DEFAULT_SCANNER_BUCKET_QUANTUM_OBJECTS.to_string(),
hidden_if_empty: false,
},
KV {
key: SCANNER_BUCKET_QUANTUM_DIRECTORIES.to_owned(),
value: DEFAULT_SCANNER_BUCKET_QUANTUM_DIRECTORIES.to_string(),
hidden_if_empty: false,
},
KV {
key: SCANNER_BUCKET_QUANTUM_ENTRIES.to_owned(),
value: DEFAULT_SCANNER_BUCKET_QUANTUM_ENTRIES.to_string(),
hidden_if_empty: false,
},
KV {
key: SCANNER_BUCKET_QUANTUM_NAME_BYTES.to_owned(),
value: DEFAULT_SCANNER_BUCKET_QUANTUM_NAME_BYTES.to_string(),
hidden_if_empty: false,
},
KV {
key: SCANNER_BITROT_CYCLE.to_owned(),
value: DEFAULT_SCANNER_BITROT_CYCLE_SECS.to_string(),
+4 -4
View File
@@ -20944,12 +20944,12 @@ impl ECStore {
let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
let mut proof = None::<DecommissionDurableIlmReceipt>;
let mut nonterminal_receipt_found = false;
for pool_idx in 0..self.pools.len() {
for (pool_idx, pool) in self.pools.iter().enumerate() {
if pool_idx == source_pool_idx {
continue;
}
let data = match read_config_limited_preserve_empty(
self.pools[pool_idx].clone(),
pool.clone(),
&receipt_path,
DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
)
@@ -28412,7 +28412,7 @@ mod tests {
for wrapped in [
err.clone(),
data_movement::data_movement_stage_error_for_test("decommission_object", "put_object", "b", "o", err.clone()),
with_decommission_entry_context("migrate_object", "b", "o", err.clone()),
with_decommission_entry_context("migrate_object", "b", "o", err),
] {
assert_eq!(
classify_decommission_capacity_failure(&wrapped),
@@ -40280,7 +40280,7 @@ mod pools_tests {
)
.expect("finalize records the completion");
let mut adopted = meta.clone();
let mut adopted = meta;
let plan = decommission_capacity_intent_adoption_plan(&adopted, 0, generation)
.expect("plan adoption")
.expect("the interrupted completion should be planned");
+1 -1
View File
@@ -4242,7 +4242,7 @@ mod tests {
"compatible aliases for {suffix} should match"
);
let mut source_without_marker = source.clone();
let mut source_without_marker = source;
rustfs_utils::http::remove_str(Arc::make_mut(&mut source_without_marker.user_defined), suffix);
let mut target_with_generated_marker = source_without_marker.clone();
rustfs_utils::http::insert_str(
+76 -10
View File
@@ -56,6 +56,8 @@ pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
const DATA_COMPRESSION_TOTAL_NAME: &str = ".compression.json";
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
const DATA_USAGE_CACHE_TTL_SECS: u64 = 30;
const ADMIN_USAGE_INCARNATION_CONCURRENCY: usize = 4;
const ADMIN_USAGE_INCARNATION_TIMEOUT: Duration = Duration::from_secs(2);
// A scanner save invalidates only the saving process's snapshot cache; every
// other node sees it only after reloading. A fail-closed usage lookup for a
// bucket the cached snapshot does not mention at all may reload a copy this
@@ -505,6 +507,7 @@ fn stale_data_usage_persist_reason_for_source(
/// Store data usage info to backend storage
#[instrument(skip(store))]
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
reject_admin_usage_projection(&data_usage_info)?;
if data_usage_info.usage_snapshot_converged == Some(false) {
return Err(Error::other(
"nonconverged data usage observations cannot replace the quota-authoritative snapshot",
@@ -534,6 +537,7 @@ async fn save_data_usage_in_backend(
store: Arc<ECStore>,
expected_publication_epoch: u64,
) -> Result<(), Error> {
reject_admin_usage_projection(&data_usage_info)?;
let data =
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
@@ -663,8 +667,9 @@ fn remove_bucket_usage_from_info(data_usage_info: &mut DataUsageInfo, bucket: &s
let removed_usage = data_usage_info.buckets_usage.remove(bucket).is_some();
let removed_size = data_usage_info.bucket_sizes.remove(bucket).is_some();
let removed_observation = data_usage_info.usage_snapshot_bucket_observations.remove(bucket).is_some();
if !removed_usage && !removed_size {
if !removed_usage && !removed_size && !removed_observation {
return false;
}
@@ -816,7 +821,9 @@ fn map_data_usage_metadata_read_error(err: Error, object: &str) -> Error {
}
fn data_usage_contains_bucket(data_usage_info: &DataUsageInfo, bucket: &str) -> bool {
data_usage_info.buckets_usage.contains_key(bucket) || data_usage_info.bucket_sizes.contains_key(bucket)
data_usage_info.buckets_usage.contains_key(bucket)
|| data_usage_info.bucket_sizes.contains_key(bucket)
|| data_usage_info.usage_snapshot_bucket_observations.contains_key(bucket)
}
fn advance_data_usage_last_update(data_usage_info: &mut DataUsageInfo) {
@@ -1137,14 +1144,26 @@ fn record_usage_snapshot_decode_failure(operation: &'static str, object: &str, e
}
fn parse_usage_snapshot(buf: &[u8]) -> Result<DataUsageInfo, Error> {
serde_json::from_slice(buf).map_err(|e| {
let mut info: DataUsageInfo = serde_json::from_slice(buf).map_err(|e| {
Error::other(format!(
"Failed to deserialize data usage info: {:?} at line {} column {}",
e.classify(),
e.line(),
e.column()
))
})
})?;
reject_admin_usage_projection(&info)?;
for usage in info.buckets_usage.values_mut() {
usage.normalize_usage_provenance();
}
Ok(info)
}
fn reject_admin_usage_projection(info: &DataUsageInfo) -> Result<(), Error> {
if info.usage_snapshot_admin_projection {
return Err(Error::other("admin usage projections cannot be persisted or used as a storage baseline"));
}
Ok(())
}
/// Decide which usage snapshot to serve: a primary usage object or its
@@ -1327,6 +1346,9 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
/// Fill quota-baseline gaps from an observed (nonconverged) snapshot without
/// overriding any bucket the authoritative baseline already covers.
fn backfill_degraded_baseline_from_observed(degraded_baseline: &mut HashMap<String, u64>, observed: &DataUsageInfo) {
if observed.usage_snapshot_admin_projection {
return;
}
for (bucket, usage) in &observed.buckets_usage {
degraded_baseline.entry(bucket.clone()).or_insert(usage.size);
}
@@ -1419,6 +1441,7 @@ fn merge_partial_observation_for_admin(mut authoritative: DataUsageInfo, observe
authoritative.usage_snapshot_converged = Some(false);
authoritative.usage_snapshot_authoritative_baseline = observed.usage_snapshot_authoritative_baseline;
authoritative.usage_snapshot_set_states = observed.usage_snapshot_set_states;
authoritative.usage_snapshot_bucket_observations = observed.usage_snapshot_bucket_observations;
authoritative.usage_snapshot_bootstrap_pending = false;
authoritative.buckets_count = authoritative.buckets_usage.len() as u64;
authoritative.bucket_sizes = authoritative
@@ -1437,10 +1460,48 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?;
// For the one-shot admin view a failed observed read degrades to "no
// observation", same as before the read was fallible.
let observed = load_observed_data_usage_snapshot(store).await.ok().flatten();
let observed = load_observed_data_usage_snapshot(store.clone()).await.ok().flatten();
let (selected, selected_is_current_format) =
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
let mut response = normalize_loaded_data_usage(selected, selected_is_current_format).await.0;
use futures::StreamExt as _;
let deadline = tokio::time::Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT;
let mut incarnations = HashMap::new();
{
// Display-only identity reads must not serialize every bucket behind a
// slow metadata lock while holding the shared admin refresh permit.
// The common deadline bounds their total added latency; dropping this
// stream cancels all unfinished reads without spawning detached work.
// Own the names and store across awaits so this future remains Send
// when composed into spawned metrics collectors.
let buckets = response
.usage_snapshot_bucket_observations
.keys()
.cloned()
.collect::<Vec<_>>();
let mut identities = futures::stream::iter(buckets)
.map(move |bucket| {
let store = store.clone();
async move {
let identity = store.bucket_incarnation_id_from_disk(&bucket).await;
(bucket, identity)
}
})
.buffer_unordered(ADMIN_USAGE_INCARNATION_CONCURRENCY);
while tokio::time::Instant::now() < deadline {
let Ok(Some((bucket, identity))) = tokio::time::timeout_at(deadline, identities.next()).await else {
break;
};
// Unavailable, timed-out or recreated buckets retain canonical values.
if let Ok(incarnation) = identity
&& !incarnation.is_nil()
{
incarnations.insert(bucket, incarnation.to_string());
}
}
}
response.project_bucket_observations_for_admin(&incarnations);
Ok(response)
}
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
@@ -1512,7 +1573,7 @@ async fn normalize_loaded_data_usage(
&& !cfg.role.is_empty()
{
data_usage_info.replication_info.insert(
cfg.role.clone(),
cfg.role,
BucketTargetUsageInfo {
replication_failed_size: bui.replication_failed_size_v1,
replication_failed_count: bui.replication_failed_count_v1,
@@ -1725,7 +1786,7 @@ fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapsh
match aggregated.buckets_usage.entry(bucket.clone()) {
Entry::Occupied(mut entry) => entry.get_mut().merge(&usage),
Entry::Vacant(entry) => {
entry.insert(usage.clone());
entry.insert(usage);
}
}
@@ -2082,7 +2143,9 @@ where
fn apply_live_bucket_usage_to_response(data_usage_info: &mut DataUsageInfo, bucket: &str, usage: &BucketUsageInfo) {
let inserted_bucket = !data_usage_info.buckets_usage.contains_key(bucket);
data_usage_info.bucket_sizes.insert(bucket.to_string(), usage.size);
data_usage_info.buckets_usage.insert(bucket.to_string(), usage.clone());
let mut observed = usage.clone();
observed.mark_observed();
data_usage_info.buckets_usage.insert(bucket.to_string(), observed);
if inserted_bucket {
data_usage_info.usage_snapshot_complete = false;
}
@@ -2774,7 +2837,9 @@ async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mu
}
added_bucket |= !data_usage_info.buckets_usage.contains_key(bucket);
data_usage_info.buckets_usage.insert(bucket.clone(), cached.usage.clone());
let mut observed = cached.usage.clone();
observed.mark_observed();
data_usage_info.buckets_usage.insert(bucket.clone(), observed);
data_usage_info.bucket_sizes.insert(bucket.clone(), cached.usage.size);
changed = true;
}
@@ -3024,6 +3089,7 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
#[cfg(test)]
mod tests {
mod companion_projection;
mod snapshot_projection;
use super::*;
@@ -0,0 +1,310 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
use rustfs_data_usage::{
BucketUsageCertification, BucketUsageValueState, DATA_USAGE_BUCKET_OBSERVATION_VERSION, DataUsageBucketObservation,
DataUsageBucketObservationSource, DataUsageSnapshotSetState,
};
mod native_compatibility;
fn companion_snapshot(bucket: &str, incarnation: uuid::Uuid) -> DataUsageInfo {
let completed_at = SystemTime::UNIX_EPOCH + Duration::from_secs(40);
let mut snapshot = data_usage_info_for_test(bucket, 10, 100, completed_at + Duration::from_secs(10));
snapshot.scanner_epoch = Some(2);
snapshot.scanner_cycle = Some(9);
snapshot.usage_snapshot_converged = Some(false);
snapshot.usage_snapshot_pending_buckets = vec![bucket.to_string()];
snapshot.usage_snapshot_pending_bucket_count = 1;
let canonical = snapshot.buckets_usage.get_mut(bucket).expect("canonical bucket");
canonical.bucket_incarnation = Some(incarnation.to_string());
canonical.certify(BucketUsageCertification {
epoch: 2,
cycle: 3,
certified_at: SystemTime::UNIX_EPOCH + Duration::from_secs(30),
});
canonical.mark_carried();
snapshot.usage_snapshot_set_states = vec![DataUsageSnapshotSetState {
pool_index: 0,
set_index: 0,
scanner_epoch: Some(2),
scanner_cycle: Some(9),
scan_plan_digest: Some([1; 32]),
complete: true,
..Default::default()
}];
let mut observed = BucketUsageInfo {
size: 250,
objects_count: 25,
versions_count: 25,
bucket_incarnation: Some(incarnation.to_string()),
..Default::default()
};
observed.mark_observed();
snapshot.usage_snapshot_bucket_observations.insert(
bucket.to_string(),
DataUsageBucketObservation {
version: DATA_USAGE_BUCKET_OBSERVATION_VERSION,
usage: observed,
snapshot_identity: snapshot.snapshot_identity(),
completed_at,
sources: vec![DataUsageBucketObservationSource {
pool_index: 0,
set_index: 0,
bucket_incarnation: incarnation.to_string(),
scanner_epoch: 2,
scanner_cycle: 4,
scan_plan_digest: [1; 32],
started_plan: [2; 32],
requested_plan: [3; 32],
completed_at,
}],
},
);
snapshot
}
#[test]
fn companion_cannot_backfill_quota_and_projection_is_rejected_at_storage_boundaries() {
let incarnation = uuid::Uuid::from_u128(31);
let mut snapshot = companion_snapshot("bucket", incarnation);
let mut canonical_quota = HashMap::new();
backfill_degraded_baseline_from_observed(&mut canonical_quota, &snapshot);
assert_eq!(canonical_quota.get("bucket"), Some(&100), "quota reads only the unchanged main map");
assert_eq!(
snapshot.project_bucket_observations_for_admin(&HashMap::from([("bucket".to_string(), incarnation.to_string())])),
1
);
let mut accidental_quota = HashMap::new();
backfill_degraded_baseline_from_observed(&mut accidental_quota, &snapshot);
assert!(accidental_quota.is_empty());
assert!(reject_admin_usage_projection(&snapshot).is_err());
assert!(certified_bucket_usage(&snapshot, "bucket", incarnation).is_none());
let wire = serde_json::to_vec(&snapshot).expect("admin response remains serializable");
assert!(parse_usage_snapshot(&wire).is_err(), "a serialized response is not a persisted snapshot");
}
#[test]
fn companion_only_observed_snapshot_does_not_create_a_legacy_quota_baseline() {
let incarnation = uuid::Uuid::from_u128(31);
let mut snapshot = companion_snapshot("bucket", incarnation);
snapshot.buckets_usage.clear();
snapshot.bucket_sizes.clear();
snapshot.buckets_count = 0;
snapshot.calculate_totals();
snapshot.usage_snapshot_complete = false;
snapshot.usage_snapshot_partial = true;
snapshot.usage_snapshot_observational = true;
snapshot.usage_snapshot_set_states[0].complete = false;
let decoded = parse_usage_snapshot(&serde_json::to_vec(&snapshot).expect("encode companion-only snapshot"))
.expect("read valid partial snapshot");
assert!(decoded.is_valid_partial_snapshot());
let mut baseline = HashMap::new();
backfill_degraded_baseline_from_observed(&mut baseline, &decoded);
assert!(baseline.is_empty(), "unknown usage must not become either zero or mixed quota usage");
println!(
"usage_assertion: {}",
serde_json::json!({
"case_id": "USAGE-08", "assertion_id": "USAGE-08-A02", "status": "PASS",
"evidence": {"primary_buckets": decoded.buckets_usage.len(), "legacy_bucket_sizes": decoded.bucket_sizes.len(), "quota_baseline_buckets": baseline.len(), "companion_buckets": decoded.usage_snapshot_bucket_observations.len(), "partial": decoded.is_valid_partial_snapshot()}
})
);
println!(
"usage_cleanup: {}",
serde_json::json!({"case_id": "USAGE-08", "status": "PASS", "scope": "process-local"})
);
}
#[test]
fn bucket_removal_discards_companions_even_without_a_canonical_entry() {
let mut snapshot = companion_snapshot("bucket", uuid::Uuid::from_u128(31));
snapshot.buckets_usage.clear();
snapshot.bucket_sizes.clear();
assert!(data_usage_contains_bucket(&snapshot, "bucket"));
assert!(remove_bucket_usage_from_info(&mut snapshot, "bucket"));
assert!(!data_usage_contains_bucket(&snapshot, "bucket"));
assert!(snapshot.usage_snapshot_bucket_observations.is_empty());
}
#[test]
fn partial_selection_replaces_value_provenance_together() {
let incarnation = uuid::Uuid::from_u128(31);
let authoritative = companion_snapshot("bucket", incarnation);
let mut observed = authoritative.clone();
observed.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
observed.scanner_cycle = Some(10);
observed.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(60));
observed.usage_snapshot_complete = false;
observed.usage_snapshot_partial = true;
observed.usage_snapshot_observational = true;
observed.usage_snapshot_bucket_observations.clear();
let usage = observed.buckets_usage.get_mut("bucket").expect("observed main bucket");
usage.size = 175;
usage.mark_observed();
let (selected, _) = select_admin_data_usage_snapshot(authoritative, true, Some(observed));
assert_eq!(selected.buckets_usage["bucket"].size, 175);
assert_eq!(selected.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
assert_eq!(selected.buckets_usage["bucket"].usage_certification, None);
assert!(
selected.usage_snapshot_bucket_observations.is_empty(),
"old root companions cannot cross a baseline change"
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn companion_admin_cache_isolated_from_quota_memory_and_persistence() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "usage-companion";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create bucket for incarnation validation");
let incarnation = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("trusted incarnation");
let canonical = companion_snapshot(bucket, incarnation);
let bytes = serde_json::to_vec(&canonical).expect("encode canonical root with companion");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
.await
.expect("save canonical root");
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
let mut samples = Vec::new();
let mut rejected_writes = 0;
for _ in 0..2 {
let response = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("read admin cache");
assert_eq!(response.buckets_usage[bucket].size, 250);
assert!(response.usage_snapshot_admin_projection);
replace_bucket_usage_memory_from_info(&response).await;
assert!(memory_cache().read().await.is_empty(), "response cannot seed quota memory");
let admin_bytes = response.buckets_usage[bucket].size;
let rejected = store_data_usage_in_backend(response, store.clone()).await.is_err();
assert!(rejected);
rejected_writes += usize::from(rejected);
let quota = load_data_usage_from_backend_cached(store.clone())
.await
.expect("read independent quota cache");
assert_eq!(quota.buckets_usage[bucket].size, 100);
assert!(!quota.usage_snapshot_admin_projection);
samples.push((admin_bytes, quota.buckets_usage[bucket].size));
invalidate_data_usage_snapshot_cache().await;
}
let persisted = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("read unchanged canonical root");
assert_eq!(persisted, bytes);
println!(
"usage_assertion: {}",
serde_json::json!({
"case_id": "USAGE-18", "assertion_id": "USAGE-18-A03", "status": "PASS",
"evidence": {"admin_and_quota_byte_samples": samples, "rejected_projected_writes": rejected_writes, "quota_memory_entries": memory_cache().read().await.len(), "persisted_canonical_unchanged": persisted == bytes}
})
);
let paths = _dirs.iter().map(|dir| dir.path().to_path_buf()).collect::<Vec<_>>();
drop(_dirs);
assert!(paths.iter().all(|path| !path.exists()));
println!(
"usage_cleanup: {}",
serde_json::json!({"case_id": "USAGE-18", "status": "PASS", "fixtures_removed": paths.iter().all(|path| !path.exists())})
);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buckets() {
use crate::bucket::metadata_sys::{
acquire_metadata_transaction_guard, init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
// Admin reads also feed spawned metrics collectors across crate boundaries.
fn assert_send<T: Send>(future: T) -> T {
future
}
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let blocked = "usage-blocked-identity";
let healthy = "usage-healthy-identity";
let mut snapshots = Vec::new();
for bucket in [blocked, healthy] {
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create companion identity fixture");
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("real incarnation");
snapshots.push(companion_snapshot(bucket, incarnation));
}
let mut canonical = snapshots.remove(0);
let other = snapshots.remove(0);
canonical.buckets_usage.extend(other.buckets_usage);
canonical.bucket_sizes.extend(other.bucket_sizes);
canonical
.usage_snapshot_bucket_observations
.extend(other.usage_snapshot_bucket_observations);
canonical.usage_snapshot_pending_buckets.push(healthy.to_string());
canonical.usage_snapshot_pending_bucket_count = 2;
canonical.buckets_count = 2;
canonical.calculate_totals();
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&canonical).expect("encode both canonical values and companions"),
)
.await
.expect("persist usage fixture");
invalidate_data_usage_snapshot_cache().await;
let blocked_guard = acquire_metadata_transaction_guard(
store.clone(),
blocked,
"admin_identity_timeout_test",
rustfs_lock::LockType::Exclusive,
)
.await
.expect("hold the real bucket metadata writer lock");
let response = tokio::time::timeout(
ADMIN_USAGE_INCARNATION_TIMEOUT + Duration::from_secs(3),
assert_send(load_admin_data_usage_from_backend_cached(store.clone())),
)
.await
.expect("optional projection must return before the blocked metadata lock's long timeout")
.expect("canonical admin response stays available");
assert_eq!(response.buckets_usage[blocked].size, 100);
assert_eq!(response.buckets_usage[blocked].usage_value_state, Some(BucketUsageValueState::Carried));
assert_eq!(response.buckets_usage[healthy].size, 250);
assert_eq!(response.buckets_usage[healthy].usage_value_state, Some(BucketUsageValueState::Observed));
assert!(response.usage_snapshot_admin_projection);
assert!(
ADMIN_DATA_USAGE_SNAPSHOT_REFRESH
.get()
.expect("admin refresh mutex")
.try_lock()
.is_ok()
);
drop(blocked_guard);
invalidate_data_usage_snapshot_cache().await;
let recovered = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("refresh after lock recovery");
assert_eq!(
recovered.buckets_usage[blocked].size, 250,
"unavailable identity can recover on the next refresh"
);
assert_eq!(recovered.buckets_usage[healthy].size, 250);
let quota = load_data_usage_from_backend_cached(store)
.await
.expect("separate canonical quota read");
assert_eq!(quota.buckets_usage[blocked].size, 100);
assert_eq!(quota.buckets_usage[healthy].size, 100);
}
@@ -0,0 +1,96 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
use super::*;
// Freeze the pre-companion reader's primary fields. Serde's unknown-field
// behavior models the existing JSON reader without importing the new schema.
#[derive(serde::Deserialize)]
struct LegacyPrimarySnapshot {
buckets_usage: HashMap<String, LegacyPrimaryBucket>,
bucket_sizes: HashMap<String, u64>,
objects_total_count: u64,
objects_total_size: u64,
#[serde(default)]
usage_snapshot_complete: bool,
}
#[derive(serde::Deserialize)]
struct LegacyPrimaryBucket {
size: u64,
objects_count: u64,
}
#[test]
fn usage_legacy_reader_never_promotes_companion_values_into_primary_lkg() {
let incarnation = uuid::Uuid::from_u128(31);
let mut snapshot = companion_snapshot("bucket", incarnation);
let original_certificate = snapshot.buckets_usage["bucket"].usage_certification;
let mut decoded_sizes = Vec::new();
for cycle in 10..=12 {
snapshot.scanner_cycle = Some(cycle);
let identity = snapshot.snapshot_identity();
snapshot
.usage_snapshot_bucket_observations
.get_mut("bucket")
.unwrap()
.snapshot_identity = identity;
let bytes = serde_json::to_vec(&snapshot).expect("production persisted snapshot encoding");
let old: LegacyPrimarySnapshot = serde_json::from_slice(&bytes).expect("old reader tolerates additive fields");
assert!(old.usage_snapshot_complete);
assert_eq!(old.buckets_usage["bucket"].size, 100);
assert_eq!(old.buckets_usage["bucket"].objects_count, 10);
assert_eq!(old.bucket_sizes["bucket"], 100);
assert_eq!(old.objects_total_size, 100);
assert_eq!(old.objects_total_count, 10);
let current = parse_usage_snapshot(&bytes).expect("current reader accepts persisted primary plus companion");
assert_eq!(current.buckets_usage["bucket"].usage_certification, original_certificate);
assert_eq!(current.usage_snapshot_bucket_observations["bucket"].usage.size, 250);
decoded_sizes.push(old.buckets_usage["bucket"].size);
}
snapshot.buckets_usage.clear();
snapshot.bucket_sizes.clear();
snapshot.buckets_count = 0;
snapshot.calculate_totals();
snapshot.usage_snapshot_complete = false;
snapshot.usage_snapshot_partial = true;
snapshot.usage_snapshot_observational = true;
snapshot.usage_snapshot_set_states[0].complete = false;
let bytes = serde_json::to_vec(&snapshot).expect("persisted companion-only encoding");
let old: LegacyPrimarySnapshot = serde_json::from_slice(&bytes).expect("old reader handles the absent baseline");
assert!(old.buckets_usage.is_empty() && old.bucket_sizes.is_empty());
assert!(!old.usage_snapshot_complete);
assert_eq!((old.objects_total_count, old.objects_total_size), (0, 0));
let current = parse_usage_snapshot(&bytes).expect("valid current observational snapshot");
assert!(current.is_valid_partial_snapshot());
assert_eq!(current.usage_snapshot_bucket_observations["bucket"].usage.size, 250);
assert_eq!(
snapshot.project_bucket_observations_for_admin(&HashMap::from([("bucket".into(), incarnation.to_string())])),
1
);
let response_bytes = serde_json::to_vec(&snapshot).expect("admin response encoding");
assert!(
parse_usage_snapshot(&response_bytes).is_err(),
"a projected response cannot be written as a baseline"
);
println!(
"usage_assertion: {}",
serde_json::json!({
"case_id": "USAGE-16", "assertion_id": "USAGE-16-A01", "status": "PASS",
"evidence": {
"wire_format": "production JSON, frozen legacy named primary fields",
"legacy_primary_sizes_across_rounds": decoded_sizes,
"new_companion_size": 250,
"companion_only_legacy_primary_count": old.buckets_usage.len(),
"companion_only_legacy_complete": old.usage_snapshot_complete,
"projected_response_rejected_by_persistence_parser": true
}
})
);
println!(
"usage_cleanup: {}",
serde_json::json!({"case_id": "USAGE-16", "status": "PASS", "scope": "process-local"})
);
}
@@ -521,7 +521,7 @@ impl EndpointTopology {
}
};
topology.aliases.entry(host_port.clone()).or_insert(member_index);
topology.aliases.entry(host_port).or_insert(member_index);
topology.aliases.entry(ep.to_string()).or_insert(member_index);
if let Some(display_endpoint) = ep.url.host_str() {
match display_aliases.entry(display_endpoint.to_owned()) {
+1 -1
View File
@@ -1626,7 +1626,7 @@ mod tests {
for disk_error in disk_errors {
let original_message = disk_error.to_string();
let io_error: std::io::Error = disk_error.clone().into();
let io_error: std::io::Error = disk_error.into();
// The io::Error should contain the original error message
assert!(io_error.to_string().contains(&original_message));
+1
View File
@@ -175,6 +175,7 @@ fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
false
}
#[cfg(test)]
#[tracing::instrument(level = "debug", skip_all)]
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
let path = path.as_ref().to_path_buf();
+129 -30
View File
@@ -76,11 +76,11 @@ use crate::disk::{
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
format::FormatV3,
fs::{
O_APPEND, O_CREATE, O_RDONLY, O_WRONLY, access, cached_access, invalidate_bucket_cache, lstat, lstat_std, remove,
remove_all_std, remove_std, rename,
O_APPEND, O_CREATE, O_RDONLY, O_WRONLY, access, cached_access, invalidate_bucket_cache, lstat, lstat_std, remove_all_std,
remove_std, rename,
},
is_quota_mutation_fence_path, os,
os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source},
os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all_ignore_missing_source},
quota_mutation_fence_path,
};
use crate::erasure::coding::{self, bitrot_verify};
@@ -3339,11 +3339,7 @@ pub(crate) fn has_delete_version_test_failpoints() -> bool {
if os::prepared_publication_test_hooks::has_any_hooks() {
return true;
}
#[cfg(feature = "e2e-test-hooks")]
if std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").is_some() {
return true;
}
false
cfg!(feature = "e2e-test-hooks") && std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").is_some()
}
#[cfg(test)]
@@ -7587,6 +7583,7 @@ pub struct LocalDisk {
pub root: PathBuf,
publication_root: os::PublicationRoot,
multipart_proof_authority: OnceLock<crate::disk::multipart_source::ProofVerificationContext>,
pub(crate) usage_watch_binding: Arc<super::usage_watch::DiskUsageWatchBinding>,
multipart_index: Arc<multipart_index::IndexOwner>,
/// I/O root pinned to the mount instance that was opened while the disk
/// was initialized. On Linux this is `/proc/self/fd/<dirfd>/.`; resolving
@@ -8137,6 +8134,7 @@ impl LocalDisk {
root: root.clone(),
publication_root,
multipart_proof_authority: OnceLock::new(),
usage_watch_binding: Arc::new(super::usage_watch::DiskUsageWatchBinding::default()),
io_root: io_root.clone(),
#[cfg(target_os = "linux")]
mount_lease: mount_lease.clone(),
@@ -8980,6 +8978,7 @@ impl LocalDisk {
fi: FileInfo,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<()> {
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
if volume == RUSTFS_META_BUCKET {
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
}
@@ -9189,6 +9188,7 @@ impl LocalDisk {
opts: DeleteOptions,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Vec<Option<Error>> {
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
let concurrency = delete_version_batch_concurrency(&versions, &opts);
let mut errors = vec![None; versions.len()];
let mut pending = stream::iter(
@@ -9221,6 +9221,7 @@ impl LocalDisk {
opts: DeleteOptions,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Vec<Option<DiskError>> {
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
let payload = super::GuardedDeleteVersionsPayload {
versions,
opts,
@@ -9256,6 +9257,8 @@ impl LocalDisk {
mut fi: FileInfo,
mutation: DeleteVersionMutation,
) -> Result<()> {
let mut mutation = mutation;
mutation.namespace_owner = self.usage_watch_binding.owner(volume, mutation.namespace_owner.take());
if volume == RUSTFS_META_BUCKET {
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
}
@@ -9878,7 +9881,12 @@ impl LocalDisk {
.await
.err()
} else {
match rename(&delete_path, &trash_path).await {
let renamed = if namespace_owner.is_some() {
os::rename_with_namespace_owner(delete_path, &trash_path, namespace_owner.clone()).await
} else {
rename(delete_path, &trash_path).await
};
match renamed {
Ok(()) => None,
Err(err)
if err.kind() == ErrorKind::NotFound && os::rename_source_is_missing(delete_path, &self.publication_root) =>
@@ -9930,6 +9938,7 @@ impl LocalDisk {
opt: DeleteOptions,
namespace_owner: Option<Arc<dyn Send + Sync>>,
) -> Result<()> {
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
if volume == RUSTFS_META_BUCKET {
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
}
@@ -10365,6 +10374,7 @@ impl LocalDisk {
Ok(())
}
#[cfg(test)]
async fn delete_file(&self, base_path: &Path, delete_path: &PathBuf, recursive: bool, immediate_purge: bool) -> Result<()> {
self.delete_file_with_namespace_owner(base_path, delete_path, recursive, immediate_purge, None, None)
.await
@@ -11642,6 +11652,15 @@ impl LocalDisk {
// Executes the given SyncMode verbatim: durability policy (tier gating,
// system-critical pinning) is resolved by callers via effective_durability.
async fn write_all_internal(&self, file_path: &Path, data: Bytes, sync: SyncMode, skip_parent: &Path) -> Result<()> {
let relative = file_path
.strip_prefix(self.io_root())
.or_else(|_| file_path.strip_prefix(&self.root));
let bucket = relative
.ok()
.and_then(|path| path.components().next())
.and_then(|component| component.as_os_str().to_str())
.unwrap_or("");
let usage_owner = self.usage_watch_binding.owner(bucket, None);
let skip_parent = if skip_parent.as_os_str().is_empty() {
self.io_root()
} else {
@@ -11660,6 +11679,7 @@ impl LocalDisk {
let root = self.publication_root.clone();
let operation = move |is_cancelled: &dyn Fn() -> bool| {
let _usage_owner = usage_owner;
if is_cancelled() {
return Err(std::io::Error::other("blocking namespace operation cancelled"));
}
@@ -14562,7 +14582,15 @@ impl DiskAPI for LocalDisk {
multipart_staging::reject_native_control_write(&self.root, self.io_root(), src_volume, src_path)?;
multipart_staging::reject_native_control_write(&self.root, self.io_root(), dst_volume, dst_path)?;
multipart_staging::reject_noncanonical_scope(&self.root, self.io_root(), src_volume, src_path)?;
let source_owner = self.multipart_source_owner(dst_volume, dst_path, true, false, None).await?;
let usage_owner = self.usage_watch_binding.owner(src_volume, None);
let usage_owner = if src_volume == dst_volume {
usage_owner
} else {
self.usage_watch_binding.owner(dst_volume, usage_owner)
};
let source_owner = self
.multipart_source_owner(dst_volume, dst_path, true, false, usage_owner)
.await?;
let admission = self.multipart_source_admission(dst_volume, dst_path, true, false)?;
let (src_volume_dir, src_file_path) = self.io_get_bucket_and_object_path(src_volume, src_path)?;
let (dst_volume_dir, dst_file_path) = self.io_get_bucket_and_object_path(dst_volume, dst_path)?;
@@ -14751,7 +14779,8 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(level = "trace", skip_all)]
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
let source_owner = self.multipart_source_owner(volume, path, true, false, None).await?;
let usage_owner = self.usage_watch_binding.owner(volume, None);
let source_owner = self.multipart_source_owner(volume, path, true, false, usage_owner).await?;
let admission = self.multipart_source_admission(volume, path, true, false)?;
let admission_source = admission.as_ref().map(|a| a.source().to_path_buf());
let txn_rel_path = crate::disk::part_transaction_path(path);
@@ -14902,7 +14931,15 @@ impl DiskAPI for LocalDisk {
multipart_staging::reject_native_control_write(&self.root, self.io_root(), src_volume, src_path)?;
multipart_staging::reject_native_control_write(&self.root, self.io_root(), dst_volume, dst_path)?;
multipart_staging::reject_noncanonical_scope(&self.root, self.io_root(), src_volume, src_path)?;
let source_owner = self.multipart_source_owner(dst_volume, dst_path, true, false, None).await?;
let usage_owner = self.usage_watch_binding.owner(src_volume, None);
let usage_owner = if src_volume == dst_volume {
usage_owner
} else {
self.usage_watch_binding.owner(dst_volume, usage_owner)
};
let source_owner = self
.multipart_source_owner(dst_volume, dst_path, true, false, usage_owner)
.await?;
multipart_staging::reject_reserved_anchor(dst_path)?;
let admission = self.multipart_source_admission(dst_volume, dst_path, true, false)?;
let admission_source = admission.as_ref().map(|a| a.source().to_path_buf());
@@ -15214,8 +15251,15 @@ impl DiskAPI for LocalDisk {
}
if !published_transaction_meta && let Some(parent) = src_file_for_cleanup.parent() {
self.delete_file(&src_volume_for_cleanup, &parent.to_path_buf(), false, false)
.await?;
self.delete_file_with_namespace_owner(
&src_volume_for_cleanup,
&parent.to_path_buf(),
false,
false,
source_owner,
None,
)
.await?;
}
Ok(())
@@ -15223,6 +15267,12 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(level = "trace", skip_all)]
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
let usage_owner = self.usage_watch_binding.owner(src_volume, None);
let usage_owner = if src_volume == dst_volume {
usage_owner
} else {
self.usage_watch_binding.owner(dst_volume, usage_owner)
};
multipart_staging::reject_native_control_write(&self.root, self.io_root(), src_volume, src_path)?;
multipart_staging::reject_native_control_write(&self.root, self.io_root(), dst_volume, dst_path)?;
if src_volume == RUSTFS_META_MULTIPART_BUCKET || dst_volume == RUSTFS_META_MULTIPART_BUCKET {
@@ -15280,14 +15330,21 @@ impl DiskAPI for LocalDisk {
// destination is the normal case when renaming a directory to a new
// location, so tolerate NotFound instead of aborting the whole rename
// (MinIO's RenameFile ignores osIsNotExist here).
if let Err(e) = remove(&dst_file_path).await
if let Err(e) = os::remove_with_owner(&dst_file_path, usage_owner.clone()).await
&& e.kind() != ErrorKind::NotFound
{
return Err(to_file_error(e).into());
}
}
rename_all(&src_file_path, &dst_file_path, &dst_volume_dir, &self.publication_root).await?;
os::rename_all_with_owner(
&src_file_path,
&dst_file_path,
&dst_volume_dir,
&self.publication_root,
usage_owner.clone(),
)
.await?;
// Both ends changed identity: the source path no longer exists and the
// destination now resolves to a different inode (backlog#1145).
@@ -15300,7 +15357,9 @@ impl DiskAPI for LocalDisk {
}
if let Some(parent) = src_file_path.parent() {
let _ = self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await;
let _ = self
.delete_file_with_namespace_owner(&src_volume_dir, &parent.to_path_buf(), false, false, usage_owner, None)
.await;
}
Ok(())
@@ -15308,6 +15367,10 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(level = "trace", skip_all)]
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, _file_size: i64) -> Result<FileWriter> {
// Normal PUT streams into the internal staging volume and is covered
// by rename_data. A raw stream into a live user volume has no retained
// physical completion owner, so it revokes the watch capability.
self.usage_watch_binding.mark_uncovered_user_stream(volume);
multipart_staging::reject_native_control_write(&self.root, self.io_root(), volume, path)?;
multipart_staging::reject_reserved_anchor(path)?;
multipart_staging::reject_noncanonical_scope(&self.root, self.io_root(), volume, path)?;
@@ -15350,6 +15413,12 @@ impl DiskAPI for LocalDisk {
}
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
let usage_owner = self.usage_watch_binding.owner(src_volume, None);
let usage_owner = if src_volume == dst_volume {
usage_owner
} else {
self.usage_watch_binding.owner(dst_volume, usage_owner)
};
let source = self.io_get_object_path(src_volume, src_path)?;
let destination = self.io_get_object_path(dst_volume, dst_path)?;
check_path_length(source.to_string_lossy().as_ref())?;
@@ -15360,9 +15429,11 @@ impl DiskAPI for LocalDisk {
return Err(DiskError::FileAccessDenied);
}
if durability.syncs_data_shards() {
let sync_owner = usage_owner.clone();
#[cfg(target_os = "linux")]
let mount_lease = self.mount_lease.clone();
spawn_blocking_with_disk_io_permit(None, move || {
let _usage_owner = sync_owner;
#[cfg(target_os = "linux")]
let _mount_lease = mount_lease;
os::sync_file(&source)
@@ -15373,7 +15444,7 @@ impl DiskAPI for LocalDisk {
}
self.rename_file(src_volume, src_path, dst_volume, dst_path).await?;
if durability.syncs_commit_metadata() {
os::fsync_dir(destination.parent().ok_or(DiskError::InvalidPath)?)
os::fsync_dir_with_owner(destination.parent().ok_or(DiskError::InvalidPath)?, usage_owner.clone())
.await
.map_err(to_file_error)?;
}
@@ -15383,6 +15454,7 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(level = "trace", skip_all)]
// async fn append_file(&self, volume: &str, path: &str, mut r: DuplexStream) -> Result<File> {
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
self.usage_watch_binding.mark_uncovered_user_stream(volume);
multipart_staging::reject_native_control_write(&self.root, self.io_root(), volume, path)?;
multipart_staging::reject_reserved_anchor(path)?;
multipart_staging::reject_noncanonical_scope(&self.root, self.io_root(), volume, path)?;
@@ -15732,6 +15804,7 @@ impl DiskAPI for LocalDisk {
} else {
self.multipart_source_owner(volume, path, true, true, None).await?
};
let owner = self.usage_watch_binding.owner(volume, owner);
self.move_to_trash_with_namespace_owner(&file_path, false, false, owner)
.await?;
@@ -15887,6 +15960,7 @@ impl DiskAPI for LocalDisk {
#[tracing::instrument(level = "trace", skip_all)]
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
let status_only = super::usage_watch::status_only_metadata_update(&fi, opts);
multipart_staging::reject_native_control_write(&self.root, self.io_root(), volume, path)?;
pool_meta_native::reject_raw_path(&self.root, volume, path)?;
let object_path = self.get_object_path(volume, path)?;
@@ -15899,13 +15973,18 @@ impl DiskAPI for LocalDisk {
condition,
)
.await?;
let owner: Option<Arc<dyn Send + Sync>> = Some(authority.clone());
let owner = self.usage_watch_binding.owner(volume, Some(authority.clone()));
let owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, owner).await);
return self
.reconcile_transition_state_metadata(volume, path, fi.version_id, condition, owner, authority)
.await;
}
let source_owner = self.multipart_source_owner(volume, path, false, true, None).await?;
let source_owner = if status_only {
source_owner
} else {
self.usage_watch_binding.owner(volume, source_owner)
};
let namespace_owner: Option<Arc<dyn Send + Sync>> =
Some(os::acquire_metadata_mutation_lease(&object_path, source_owner).await);
if !fi.metadata.is_empty() {
@@ -16293,18 +16372,35 @@ impl DiskAPI for LocalDisk {
// non-recursive rmdir calls. A file that exists during the scan, or
// appears before its parent is removed, fails closed with
// VolumeNotEmpty. Only an explicit force delete removes recursively.
let res = if force_delete {
fs::remove_dir_all(&p).await
} else {
#[cfg(target_os = "linux")]
{
remove_empty_directory_tree_under_mount_lease(&self.mount_lease, volume, p.clone()).await
let usage_owner = self.usage_watch_binding.owner(volume, None);
let physical_path = p.clone();
let physical_volume = volume.to_owned();
let backend = Arc::clone(&self.io_backend);
#[cfg(target_os = "linux")]
let mount_lease = Arc::clone(&self.mount_lease);
let res = tokio::spawn(async move {
let _usage_owner = usage_owner;
let _volume_mutation_guard = _volume_mutation_guard;
let result = if force_delete {
fs::remove_dir_all(&physical_path).await
} else {
#[cfg(target_os = "linux")]
{
remove_empty_directory_tree_under_mount_lease(&mount_lease, &physical_volume, physical_path.clone()).await
}
#[cfg(not(target_os = "linux"))]
{
remove_empty_directory_tree(&physical_path).await
}
};
if result.is_ok() {
backend.invalidate_cached_fds_for_volume(&physical_volume);
invalidate_bucket_cache(&physical_path);
}
#[cfg(not(target_os = "linux"))]
{
remove_empty_directory_tree(&p).await
}
};
result
})
.await
.map_err(DiskError::from)?;
if let Err(err) = res {
let e: DiskError = classify_delete_volume_error(err);
@@ -16484,6 +16580,9 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf
Ok((disk_info, root_drive))
}
#[cfg(all(test, not(windows)))]
mod usage_watch_tests;
#[cfg(test)]
mod test {
use super::commit::create_local_inline_rollback_backup;
+4
View File
@@ -500,6 +500,10 @@ impl LocalDisk {
dst_path: &str,
state: &mut RenameDataState,
) -> Result<RenameDataResp> {
state.namespace_owner = self.usage_watch_binding.owner(dst_volume, state.namespace_owner.take());
if src_volume != dst_volume {
state.namespace_owner = self.usage_watch_binding.owner(src_volume, state.namespace_owner.take());
}
super::multipart_staging::reject_noncanonical_scope(&self.root, self.io_root(), src_volume, src_path).inspect_err(
|_| {
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
@@ -1827,6 +1827,7 @@ impl LocalDisk {
action: DeleteRecoveryAction,
owner: Option<Arc<dyn Send + Sync>>,
) -> Result<DeleteRecoveryInspection> {
let owner = self.usage_watch_binding.owner(volume, owner);
self.reject_multipart_delete_recovery_mutation(volume, object)?;
if let Ok(root) = self.io_get_object_path(volume, object) {
let _ = take_prepared_delete_receipt(&root.join(transaction.to_string()));
@@ -2779,6 +2780,7 @@ impl LocalDisk {
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
if transaction.is_nil() {
return Err(DiskError::InvalidPath);
}
@@ -2974,6 +2976,7 @@ impl LocalDisk {
namespace_owner: Option<Arc<dyn Send + Sync>>,
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
) -> Result<bool> {
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
fn ignore_not_found(res: std::io::Result<()>) -> std::io::Result<()> {
match res {
Ok(()) => Ok(()),
@@ -0,0 +1,234 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
use crate::runtime::usage_watch::{ScannerUsageWatchRequest, UsageWatchRegistry};
fn watch_request(bucket: &str) -> ScannerUsageWatchRequest {
ScannerUsageWatchRequest {
bucket: bucket.into(),
bucket_incarnation: Uuid::from_u128(2),
watch_id: None,
release: false,
session_id: None,
}
}
async fn fixture() -> (tempfile::TempDir, Arc<LocalDisk>, Arc<UsageWatchRegistry>) {
let root = tempfile::tempdir().expect("watch physical fixture directory");
let endpoint = Endpoint::try_from(root.path().to_string_lossy().as_ref()).expect("watch physical endpoint");
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("watch physical disk"));
disk.make_volume("bucket").await.expect("watch test bucket");
let registry = Arc::new(UsageWatchRegistry::default());
disk.usage_watch_binding.bind(&registry);
(root, disk, registry)
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn usage_watch_stays_inflight_after_metadata_is_visible_and_waiter_is_canceled() {
let (_root, disk, registry) = fixture().await;
let target = disk
.io_get_object_path("bucket", "object")
.expect("target metadata directory");
let (entered, wait_for_enter) = tokio::sync::oneshot::channel();
let (release, hold) = std::sync::mpsc::channel::<()>();
let _hook = os::prepared_publication_test_hooks::install_at(
os::prepared_publication_test_hooks::Stage::DirFsync,
&target,
move || {
let _ = entered.send(());
let _ = hold.recv();
},
);
let writer_disk = Arc::clone(&disk);
let mut writer = tokio::spawn(async move {
let info =
super::test::test_file_info("object", Uuid::new_v4(), Some(Uuid::new_v4()), Some(Bytes::from_static(b"payload")));
writer_disk.write_metadata("", "bucket", "object", info).await
});
tokio::time::timeout(Duration::from_secs(10), wait_for_enter)
.await
.expect("reach physical fsync barrier")
.expect("barrier entered");
assert!(
target.join(STORAGE_FORMAT_FILE).exists(),
"metadata is already visible at the blocked fsync"
);
let during = registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("install during active physical mutation");
assert!(during.inflight > 0, "installation must include pre-existing writes");
writer.abort();
assert!((&mut writer).await.expect_err("cancel logical waiter").is_cancelled());
let canceled = registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("watch survives logical cancellation");
assert!(canceled.inflight > 0, "physical tail retains its guard");
drop(release);
let settled = tokio::time::timeout(Duration::from_secs(10), async {
loop {
let observation = registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("settlement proof");
if observation.inflight == 0 {
break observation;
}
tokio::task::yield_now().await;
}
})
.await
.expect("physical tail settles");
assert!(settled.revision > during.revision);
assert!(!settled.unchanged_since(&during));
println!(
"usage_assertion: {}",
serde_json::json!({
"case_id": "USAGE-12", "assertion_id": "USAGE-12-A03", "status": "PASS",
"evidence": {"inflight_at_install": during.inflight, "inflight_after_waiter_cancel": canceled.inflight, "inflight_after_physical_end": settled.inflight, "revision_at_install": during.revision, "revision_after_physical_end": settled.revision}
})
);
drop(disk);
drop(registry);
let root_path = _root.path().to_path_buf();
drop(_root);
assert!(!root_path.exists());
println!(
"usage_cleanup: {}",
serde_json::json!({"case_id": "USAGE-12", "status": "PASS", "fixture_removed": !root_path.exists()})
);
}
#[tokio::test]
async fn usage_watch_status_only_metadata_keeps_maintenance_semantics() {
let (_root, disk, registry) = fixture().await;
let info = super::test::test_file_info("object", Uuid::new_v4(), Some(Uuid::new_v4()), Some(Bytes::from_static(b"payload")));
disk.write_metadata("", "bucket", "object", info.clone())
.await
.expect("initial counted version");
let before = registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("idle watch");
let mut status = info.clone();
status.metadata.clear();
rustfs_utils::http::metadata_compat::insert_str(
&mut status.metadata,
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
"arn=COMPLETED;".into(),
);
disk.update_metadata("bucket", "object", status, &UpdateMetadataOpts::default())
.await
.expect("status-class metadata update");
let after = registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("status result watch");
assert!(after.unchanged_since(&before), "replication counters retain maintenance cadence");
let mut next = info;
next.version_id = Some(Uuid::new_v4());
disk.write_metadata("", "bucket", "object", next).await.expect("new version");
let changed = registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("new version watch");
assert!(changed.revision > after.revision);
assert_eq!(changed.inflight, 0);
}
#[tokio::test]
async fn usage_watch_raw_live_stream_revokes_coverage_and_internal_staging_does_not() {
let (_root, disk, registry) = fixture().await;
let before = registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("idle watch");
disk.usage_watch_binding.mark_uncovered_user_stream(RUSTFS_META_TMP_BUCKET);
assert!(
registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("internal staging stays covered")
.unchanged_since(&before)
);
let _stream = disk.create_file("bucket", "bucket", "raw-file", 0).await;
assert_eq!(registry.observe(Uuid::from_u128(1), &watch_request("bucket")), None);
assert!(!disk.usage_watch_binding.matches(&registry));
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn usage_watch_delete_paths_retains_ownership_through_canceled_trash_rename() {
let (_root, disk, registry) = fixture().await;
let target = disk.io_get_object_path("bucket", "delete-me").expect("delete source");
std::fs::write(&target, b"payload").expect("initial source");
let trash = disk.io_get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET).expect("trash volume");
std::fs::create_dir_all(&trash).expect("trash directory");
let before = registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("initial proof");
let (entered, wait_for_enter) = tokio::sync::oneshot::channel();
let (release, hold) = std::sync::mpsc::channel::<()>();
let _hook =
os::prepared_publication_test_hooks::install_at(os::prepared_publication_test_hooks::Stage::Rename, &target, move || {
let _ = entered.send(());
let _ = hold.recv();
});
let deleting_disk = Arc::clone(&disk);
let mut deleting = tokio::spawn(async move { deleting_disk.delete_paths("bucket", &["delete-me".into()]).await });
tokio::time::timeout(Duration::from_secs(10), wait_for_enter)
.await
.expect("reach raw trash rename")
.expect("rename entered");
deleting.abort();
assert!((&mut deleting).await.expect_err("logical waiter canceled").is_cancelled());
let during = registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("active proof");
assert!(during.inflight > 0, "trash syscall owns the watch after waiter cancellation");
assert!(during.revision > before.revision);
drop(release);
let after = tokio::time::timeout(Duration::from_secs(10), async {
loop {
let observation = registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("draining proof");
if observation.inflight == 0 {
break observation;
}
tokio::task::yield_now().await;
}
})
.await
.expect("physical rename settles");
assert!(!target.exists());
assert!(after.revision > during.revision);
assert!(!after.unchanged_since(&before));
}
#[tokio::test]
async fn owned_stale_destination_removal_preserves_file_directory_and_symlink_semantics() {
let (_root, disk, registry) = fixture().await;
let target = disk.io_get_object_path("bucket", "stale").expect("stale destination");
std::fs::write(&target, b"stale-file").expect("stale file");
os::remove_with_owner(&target, disk.usage_watch_binding.owner("bucket", None))
.await
.expect("remove stale file");
assert!(!target.exists());
std::fs::create_dir(&target).expect("stale directory");
os::remove_with_owner(&target, disk.usage_watch_binding.owner("bucket", None))
.await
.expect("remove empty directory");
assert!(!target.exists());
#[cfg(unix)]
{
let referent = disk.io_get_object_path("bucket", "retained").expect("symlink referent");
std::fs::write(&referent, b"retained").expect("retained file");
std::os::unix::fs::symlink(&referent, &target).expect("stale symlink");
os::remove_with_owner(&target, disk.usage_watch_binding.owner("bucket", None))
.await
.expect("remove symlink itself");
assert!(!target.exists());
assert_eq!(std::fs::read(referent).expect("referent remains"), b"retained");
}
assert_eq!(
registry
.observe(Uuid::from_u128(1), &watch_request("bucket"))
.expect("all work drained")
.inflight,
0
);
}
+11
View File
@@ -30,6 +30,7 @@ pub mod local;
pub mod multipart_index;
pub mod multipart_source;
pub mod os;
mod usage_watch;
pub(crate) use io_admission::memory as background_memory;
pub(crate) use io_admission::remote as remote_io_admission;
@@ -1386,6 +1387,16 @@ impl Disk {
}
impl Disk {
/// The progress flag is derived from a verified capability scope.
pub async fn ns_scanner_progress_capability(
&self,
) -> Result<Option<crate::storage_api_contracts::internode::NsScannerCapabilities>> {
match self {
Disk::Local(_) => Ok(None),
Disk::Remote(remote_disk) => remote_disk.ns_scanner_progress_capability().await,
}
}
pub async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
match self {
Disk::Local(_) => Ok(None),
+2 -2
View File
@@ -2137,7 +2137,7 @@ mod multipart_proof_authority_tests {
assert_eq!(expected, snapshot);
assert_eq!(actual_round, round);
let publication = PublicationIdentity {
lineage: round.lineage.clone(),
lineage: round.lineage,
source_generation: snapshot.generation,
attempt: Uuid::new_v4(),
version_id: None,
@@ -2145,7 +2145,7 @@ mod multipart_proof_authority_tests {
modified_unix_nanos: nanos,
completion_identity: [6; 32],
};
snapshot.publication = Some(publication.clone());
snapshot.publication = Some(publication);
let response = MultipartSourceResponse::TerminalConfirmed(
MultipartSourceProof::issue(
&authority,
+18 -31
View File
@@ -384,14 +384,11 @@ pub(crate) mod prepared_publication_test_hooks {
}
pub(crate) fn has_any_hooks() -> bool {
if !BEFORE_PUBLICATION.lock().is_empty() {
return true;
}
#[cfg(test)]
if !RENAME_DESTINATIONS.lock().is_empty() || !HARDLINK_DESTINATIONS.lock().is_empty() {
return true;
}
false
!BEFORE_PUBLICATION.lock().is_empty()
}
#[cfg(test)]
@@ -3511,20 +3508,6 @@ pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> std::io::Result<Vec
Ok(volumes)
}
#[tracing::instrument(level = "debug", skip_all)]
pub async fn rename_all(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
publication_root: &PublicationRoot,
) -> Result<()> {
reliable_rename(src_file_path, dst_file_path.as_ref(), base_dir, publication_root)
.await
.map_err(to_file_error)?;
Ok(())
}
/// Retain mutation ownership through both metadata publication and its directory
/// barriers, even when the async caller stops waiting after the rename starts.
pub(crate) async fn rename_metadata_and_sync_with_lease(
@@ -3569,6 +3552,19 @@ pub(crate) async fn fsync_dir_with_owner(path: &Path, namespace_owner: Option<Ar
}
}
/// Preserve file, symlink, and empty-directory removal semantics while the
/// actual filesystem executor retains namespace ownership after cancellation.
pub(crate) async fn remove_with_owner(path: impl AsRef<Path>, namespace_owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
let path = path.as_ref().to_path_buf();
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
run_blocking_namespace_operation(lease, move || {
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
super::fs::remove_std(path)
})
.await
}
/// Retain namespace ownership in the actual filesystem executor after timeout.
pub(crate) async fn remove_file_with_owner(
path: impl AsRef<Path>,
@@ -4635,15 +4631,6 @@ pub(crate) fn rename_source_is_missing(src_file_path: &Path, _publication_root:
matches!(std::fs::symlink_metadata(src_file_path), Err(err) if err.kind() == io::ErrorKind::NotFound)
}
async fn reliable_rename(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
base_dir: impl AsRef<Path>,
publication_root: &PublicationRoot,
) -> io::Result<()> {
reliable_rename_inner(src_file_path, dst_file_path, base_dir, publication_root, true).await
}
async fn reliable_rename_inner(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
@@ -8658,7 +8645,7 @@ mod tests {
let dst_file_path = dst_file_path.as_ref();
let base_dir = base_dir.as_ref();
let publication_root = test_publication_root(&[src_file_path, dst_file_path, base_dir]);
super::rename_all(src_file_path, dst_file_path, base_dir, &publication_root).await
super::rename_all_with_owner(src_file_path, dst_file_path, base_dir, &publication_root, None).await
}
async fn rename_all_ignore_missing_source(
@@ -9503,7 +9490,7 @@ mod tests {
let configured_dst = configured_base.join("object");
std::fs::create_dir(&configured_base).expect("create bucket through configured root");
std::fs::write(&configured_src, b"configured").expect("write staged object through configured root");
super::rename_all(&configured_src, &configured_dst, &configured_base, &publication_root)
super::rename_all_with_owner(&configured_src, &configured_dst, &configured_base, &publication_root, None)
.await
.expect("publish a configured path relative to the pinned root");
assert_eq!(
@@ -9516,7 +9503,7 @@ mod tests {
let resolved_dst = resolved_base.join("object");
std::fs::create_dir(&resolved_base).expect("create bucket through resolved root");
std::fs::write(&resolved_src, b"resolved").expect("write staged object through resolved root");
super::rename_all(&resolved_src, &resolved_dst, &resolved_base, &publication_root)
super::rename_all_with_owner(&resolved_src, &resolved_dst, &resolved_base, &publication_root, None)
.await
.expect("publish a resolved path relative to the pinned root");
@@ -9936,7 +9923,7 @@ mod tests {
std::fs::write(&dst, b"uncommitted-metadata").expect("write metadata to replace");
let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root above the object tree");
super::rename_all(&src, &dst, &object_dir, &publication_root)
super::rename_all_with_owner(&src, &dst, &object_dir, &publication_root, None)
.await
.expect("a rollback source below its destination parent must not conflict with its own guards");
+196
View File
@@ -0,0 +1,196 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
//! Explicit instance binding at the physical disk boundary. Lock order is this
//! binding mutex, then the registry mutex; registry code never locks a disk.
use crate::runtime::usage_watch::{UsageWatchMutationGuard, UsageWatchRegistry};
use std::sync::{Arc, Mutex, Weak};
/// Existing-version replication status writes change neither size nor version
/// membership. Replication aggregates keep their existing maintenance cadence.
pub(crate) fn status_only_metadata_update(fi: &rustfs_filemeta::FileInfo, opts: &super::UpdateMetadataOpts) -> bool {
use rustfs_utils::http::*;
!opts.replace_user_metadata
&& opts.transition_reconcile.is_none()
&& !fi.metadata.is_empty()
&& fi.metadata.keys().all(|key| {
strip_internal_prefix_preserving_case(key).is_some_and(|suffix| {
[
SUFFIX_REPLICA_STATUS,
SUFFIX_REPLICA_TIMESTAMP,
SUFFIX_REPLICATION_STATUS,
SUFFIX_REPLICATION_TIMESTAMP,
SUFFIX_REPLICATION_GENERATION,
SUFFIX_REPLICATION_RESET,
SUFFIX_PURGESTATUS,
]
.iter()
.any(|known| suffix.eq_ignore_ascii_case(known))
})
})
}
#[derive(Default)]
struct BindingState {
registry: Option<Weak<UsageWatchRegistry>>,
unbound_inflight: u64,
prebinding_guard: Option<UsageWatchMutationGuard>,
conflicted: bool,
}
#[derive(Default)]
pub(crate) struct DiskUsageWatchBinding {
state: Mutex<BindingState>,
}
struct UnboundGuard(Arc<DiskUsageWatchBinding>);
impl Drop for UnboundGuard {
fn drop(&mut self) {
let mut state = self.0.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
if state.unbound_inflight != 0 {
state.unbound_inflight -= 1;
}
if state.unbound_inflight == 0 {
state.prebinding_guard = None;
}
}
}
impl DiskUsageWatchBinding {
pub(crate) fn mark_uncovered_user_stream(&self, bucket: &str) {
if crate::bucket::utils::is_meta_bucketname(bucket) {
return;
}
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
state.conflicted = true;
if let Some(registry) = state.registry.as_ref().and_then(Weak::upgrade) {
registry.mark_coverage_unavailable();
}
}
pub(crate) fn bind(&self, registry: &Arc<UsageWatchRegistry>) {
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
if let Some(existing) = state.registry.as_ref().and_then(Weak::upgrade) {
if !Arc::ptr_eq(&existing, registry) {
state.conflicted = true;
// A query may already have checked this binding before
// awaiting disk metadata. Never allow it to reinstall a
// proof after this permanently ambiguous ownership change.
existing.mark_coverage_unavailable();
registry.mark_coverage_unavailable();
}
return;
}
// A newly registered disk can have pre-registration physical tails.
// Keep the whole registry Unknown until those exact guards drain.
if state.unbound_inflight != 0 {
state.prebinding_guard = Some(registry.begin(None));
}
if state.conflicted {
registry.mark_coverage_unavailable();
}
state.registry = Some(Arc::downgrade(registry));
}
pub(crate) fn matches(&self, registry: &Arc<UsageWatchRegistry>) -> bool {
let state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
!state.conflicted
&& state
.registry
.as_ref()
.and_then(Weak::upgrade)
.is_some_and(|bound| Arc::ptr_eq(&bound, registry))
}
pub(crate) fn owner(self: &Arc<Self>, bucket: &str, parent: Option<Arc<dyn Send + Sync>>) -> Option<Arc<dyn Send + Sync>> {
if crate::bucket::utils::is_meta_bucketname(bucket) {
return parent;
}
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
let guard: Arc<dyn Send + Sync> = if let Some(registry) = state.registry.as_ref().and_then(Weak::upgrade) {
Arc::new(registry.begin((!state.conflicted).then_some(bucket)))
} else {
match state.unbound_inflight.checked_add(1) {
Some(count) => state.unbound_inflight = count,
None => state.conflicted = true,
}
Arc::new(UnboundGuard(Arc::clone(self)))
};
Some(match parent {
Some(parent) => Arc::new((parent, guard)),
None => guard,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::runtime::usage_watch::ScannerUsageWatchRequest;
use uuid::Uuid;
#[test]
fn binding_preserves_mutations_that_started_before_registration() {
let binding = Arc::new(DiskUsageWatchBinding::default());
let registry = Arc::new(UsageWatchRegistry::default());
let tail = binding.owner("bucket", None);
binding.bind(&registry);
assert!(binding.matches(&registry));
let request = ScannerUsageWatchRequest {
bucket: "bucket".into(),
bucket_incarnation: Uuid::from_u128(2),
watch_id: None,
release: false,
session_id: None,
};
assert_eq!(registry.observe(Uuid::from_u128(1), &request), None);
drop(tail);
assert_eq!(
registry
.observe(Uuid::from_u128(1), &request)
.expect("drained binding")
.inflight,
0
);
}
#[test]
fn composite_owner_and_instance_conflicts_fail_closed() {
let binding = Arc::new(DiskUsageWatchBinding::default());
let registry = Arc::new(UsageWatchRegistry::default());
binding.bind(&registry);
let request = ScannerUsageWatchRequest {
bucket: "bucket".into(),
bucket_incarnation: Uuid::from_u128(2),
watch_id: None,
release: false,
session_id: None,
};
registry.observe(Uuid::from_u128(1), &request).expect("initial watch");
let owner = binding.owner("bucket", None).expect("physical owner");
let nested = binding.owner("bucket", Some(owner));
assert_eq!(
registry
.observe(Uuid::from_u128(1), &request)
.expect("both operations active")
.inflight,
2
);
drop(nested);
assert_eq!(
registry
.observe(Uuid::from_u128(1), &request)
.expect("drained owners")
.inflight,
0
);
let other = Arc::new(UsageWatchRegistry::default());
binding.bind(&other);
assert!(!binding.matches(&registry));
assert!(!binding.matches(&other));
assert_eq!(registry.observe(Uuid::from_u128(1), &request), None);
assert_eq!(other.observe(Uuid::from_u128(1), &request), None);
}
}
+1 -1
View File
@@ -439,7 +439,7 @@ mod tests {
reconstruct_with(&engine, &mut shards).expect("restore missing data and parity");
assert_eq!(shards, expected, "every restored byte must match at width {}", shards.len());
let mut corrupt = expected.clone();
let mut corrupt = expected;
corrupt[0] = None;
corrupt[data_shards] = None;
corrupt[data_shards + 1].as_mut().expect("surplus parity present")[0] ^= 1;
+1 -1
View File
@@ -2002,7 +2002,7 @@ mod tests {
for storage_error in storage_errors {
let original_message = storage_error.to_string();
let io_error: std::io::Error = storage_error.clone().into();
let io_error: std::io::Error = storage_error.into();
// The io::Error should contain the original error message or info
assert!(io_error.to_string().contains(&original_message));
+1 -1
View File
@@ -391,7 +391,7 @@ impl PoolEndpointList {
e.get_mut().extend(host_ip_set.iter());
}
Entry::Vacant(e) => {
e.insert(host_ip_set.clone());
e.insert(host_ip_set);
}
}
}
+1 -1
View File
@@ -47,7 +47,7 @@ pub(crate) fn formats_to_drives_info(
before_drives.push(HealDriveInfo {
uuid,
endpoint: drive,
state: state.to_string(),
state,
});
}
before_drives
+13
View File
@@ -99,6 +99,7 @@ pub(crate) struct ScannerPublicationLeaseEntry {
pub(crate) struct NamespaceCommitGuard {
ctx: Arc<InstanceContext>,
counted: bool,
_usage_watch: super::usage_watch::UsageWatchMutationGuard,
}
impl Drop for NamespaceCommitGuard {
@@ -365,6 +366,7 @@ pub struct InstanceContext {
scanner_publication_state: AtomicU8,
namespace_commits: AtomicU64,
namespace_commit_generation: AtomicU64,
pub(crate) usage_watches: Arc<super::usage_watch::UsageWatchRegistry>,
/// Resolves object-encryption material at the application boundary.
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
delete_recovery_hints: DeleteRecoveryHints,
@@ -444,6 +446,7 @@ impl InstanceContext {
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
namespace_commits: AtomicU64::new(0),
namespace_commit_generation: AtomicU64::new(0),
usage_watches: Arc::new(super::usage_watch::UsageWatchRegistry::default()),
object_encryption_resolver: OnceLock::new(),
delete_recovery_hints: DeleteRecoveryHints::default(),
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
@@ -683,6 +686,15 @@ impl InstanceContext {
}
pub(crate) fn begin_namespace_commit(self: &Arc<Self>) -> Arc<NamespaceCommitGuard> {
self.begin_namespace_commit_scoped(None)
}
pub(crate) fn begin_namespace_commit_for_bucket(self: &Arc<Self>, bucket: &str) -> Arc<NamespaceCommitGuard> {
self.begin_namespace_commit_scoped(Some(bucket))
}
fn begin_namespace_commit_scoped(self: &Arc<Self>, bucket: Option<&str>) -> Arc<NamespaceCommitGuard> {
let usage_watch = self.usage_watches.begin(bucket);
let counted = self
.namespace_commits
.try_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1))
@@ -695,6 +707,7 @@ impl InstanceContext {
Arc::new(NamespaceCommitGuard {
ctx: Arc::clone(self),
counted,
_usage_watch: usage_watch,
})
}
+1
View File
@@ -17,3 +17,4 @@
pub(crate) mod global;
pub(crate) mod instance;
pub(crate) mod sources;
pub(crate) mod usage_watch;
+6 -2
View File
@@ -525,8 +525,11 @@ pub(crate) async fn record_local_disks(instance_ctx: &Arc<InstanceContext>, disk
let map = instance_ctx.local_disk_map();
let mut global_local_disk_map = map.write().await;
for disk in disks {
if let crate::disk::Disk::Local(wrapper) = disk.as_ref() {
wrapper.get_disk().usage_watch_binding.bind(&instance_ctx.usage_watches);
}
let path = disk.endpoint().to_string();
global_local_disk_map.insert(path, Some(disk.clone()));
global_local_disk_map.insert(path, Some(disk));
}
}
@@ -611,6 +614,7 @@ pub(crate) async fn initialize_local_disk_maps(
Err(err) => return Err(err.into()),
};
if let crate::disk::Disk::Local(wrapper) = disk.as_ref() {
wrapper.get_disk().usage_watch_binding.bind(&instance_ctx.usage_watches);
wrapper.get_disk().bind_multipart_proof_authority(&multipart_authority)?;
}
let path = disk.endpoint().to_string();
@@ -625,7 +629,7 @@ pub(crate) async fn initialize_local_disk_maps(
})?;
global_local_disk_map.insert(path, Some(disk.clone()));
global_set_drives[pool_idx][set_idx][disk_idx] = Some(disk.clone());
global_set_drives[pool_idx][set_idx][disk_idx] = Some(disk);
}
}
+334
View File
@@ -0,0 +1,334 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
//! Bounded, same-session bucket mutation watches. The registry lock is a short
//! synchronous critical section shared by installation, mutation begin/end,
//! and observation. No I/O or await occurs while it is held.
use std::{
collections::HashMap,
sync::{Arc, Mutex},
time::{Duration, Instant},
};
use uuid::Uuid;
pub const SCANNER_USAGE_WATCH_COVERAGE_VERSION: u32 = rustfs_protos::usage_watch::USAGE_WATCH_COVERAGE_VERSION;
pub const SCANNER_USAGE_WATCH_MAX_ENTRIES: usize = 256;
pub const SCANNER_USAGE_WATCH_MAX_ACTIVE_BUCKETS: usize = 4096;
pub const SCANNER_USAGE_WATCH_MAX_NAME_BYTES: usize = 16 * 1024;
pub const SCANNER_USAGE_WATCH_MAX_ACTIVE_NAME_BYTES: usize = 256 * 1024;
pub const SCANNER_USAGE_WATCH_LEASE_MS: u64 = rustfs_protos::usage_watch::USAGE_WATCH_MAX_LEASE_MS;
/// A missing watch ID installs a new watch; a supplied ID can only refresh the
/// exact, still-live token. An expired token never silently creates a new one.
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ScannerUsageWatchRequest {
pub bucket: String,
pub bucket_incarnation: Uuid,
pub watch_id: Option<Uuid>,
/// Release only this exact producer session and token after publication.
pub release: bool,
pub session_id: Option<Uuid>,
}
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub struct ScannerUsageWatchKnown {
pub owner_id: Uuid,
pub session_id: Uuid,
pub watch_id: Uuid,
pub bucket_incarnation: Uuid,
pub revision: u64,
pub inflight: u64,
pub coverage_version: u32,
pub coverage_epoch: u64,
/// Duration from the receiver's observation, never a wall-clock deadline.
pub lease_remaining_ms: u64,
}
impl ScannerUsageWatchKnown {
pub fn is_well_formed(&self) -> bool {
!self.owner_id.is_nil()
&& !self.session_id.is_nil()
&& !self.watch_id.is_nil()
&& !self.bucket_incarnation.is_nil()
&& self.revision != 0
&& self.revision != u64::MAX
&& self.inflight != u64::MAX
&& self.coverage_version == SCANNER_USAGE_WATCH_COVERAGE_VERSION
&& self.coverage_epoch != 0
&& self.coverage_epoch != u64::MAX
&& self.lease_remaining_ms != 0
&& self.lease_remaining_ms <= SCANNER_USAGE_WATCH_LEASE_MS
}
/// Equality of stable proof fields; the caller must also respect its
/// conservative local lease deadline and validate the source/owner set.
pub fn unchanged_since(&self, earlier: &Self) -> bool {
self.is_well_formed()
&& earlier.is_well_formed()
&& self.owner_id == earlier.owner_id
&& self.session_id == earlier.session_id
&& self.watch_id == earlier.watch_id
&& self.bucket_incarnation == earlier.bucket_incarnation
&& self.revision == earlier.revision
&& self.inflight == 0
&& earlier.inflight == 0
&& self.coverage_version == SCANNER_USAGE_WATCH_COVERAGE_VERSION
&& self.coverage_version == earlier.coverage_version
&& self.coverage_epoch == earlier.coverage_epoch
&& self.lease_remaining_ms != 0
&& earlier.lease_remaining_ms != 0
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ScannerUsageWatchObservation {
pub bucket: String,
/// Missing means Unknown. It is never a revision-zero clean value.
pub known: Option<ScannerUsageWatchKnown>,
}
struct Watch {
owner: Uuid,
token: Uuid,
incarnation: Uuid,
revision: u64,
expires_at: Instant,
}
#[derive(Default)]
struct State {
watches: HashMap<Arc<str>, Watch>,
active: HashMap<Arc<str>, u64>,
watch_name_bytes: usize,
active_name_bytes: usize,
untracked: u64,
exhausted: bool,
coverage_epoch: u64,
}
pub(crate) struct UsageWatchRegistry {
session: Uuid,
state: Mutex<State>,
}
enum Mutation {
Tracked(Arc<str>),
Untracked,
}
/// The physical mutation owner retains this value through all detached tails.
pub(crate) struct UsageWatchMutationGuard {
registry: Arc<UsageWatchRegistry>,
mutation: Mutation,
}
impl Default for UsageWatchRegistry {
fn default() -> Self {
Self {
session: Uuid::new_v4(),
state: Mutex::new(State {
coverage_epoch: 1,
..Default::default()
}),
}
}
}
impl State {
fn invalidate_all(&mut self) {
self.watches.clear();
self.watch_name_bytes = 0;
match self.coverage_epoch.checked_add(1).filter(|epoch| *epoch != u64::MAX) {
Some(epoch) => self.coverage_epoch = epoch,
None => self.exhausted = true,
}
}
fn advance_bucket(&mut self, bucket: &str) {
if let Some(watch) = self.watches.get_mut(bucket) {
if let Some(next) = watch.revision.checked_add(1).filter(|next| *next != u64::MAX) {
watch.revision = next;
} else {
self.remove_watch(bucket);
}
}
}
fn remove_watch(&mut self, bucket: &str) {
if let Some((name, _)) = self.watches.remove_entry(bucket) {
self.watch_name_bytes -= name.len();
}
}
}
impl UsageWatchRegistry {
pub(crate) fn session_id(&self) -> Uuid {
self.session
}
pub(crate) fn begin(self: &Arc<Self>, bucket: Option<&str>) -> UsageWatchMutationGuard {
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
// Internal metadata is outside the user bucket counters. Callers with
// an unknown user scope must pass None, which invalidates every watch.
let bucket = bucket.filter(|bucket| !bucket.is_empty() && bucket.len() <= 63);
let tracked = bucket.and_then(|bucket| {
if let Some((key, count)) = state.active.get_key_value(bucket) {
let key = Arc::clone(key);
let next = count.checked_add(1)?;
state.active.insert(Arc::clone(&key), next);
return Some(key);
}
if state.active.len() >= SCANNER_USAGE_WATCH_MAX_ACTIVE_BUCKETS
|| state.active_name_bytes + bucket.len() > SCANNER_USAGE_WATCH_MAX_ACTIVE_NAME_BYTES
{
return None;
}
let key: Arc<str> = Arc::from(bucket);
state.active_name_bytes += bucket.len();
state.active.insert(Arc::clone(&key), 1);
Some(key)
});
let mutation = match tracked {
Some(bucket) => {
state.advance_bucket(&bucket);
Mutation::Tracked(bucket)
}
None => {
state.invalidate_all();
match state.untracked.checked_add(1) {
Some(count) => state.untracked = count,
None => state.exhausted = true,
}
Mutation::Untracked
}
};
UsageWatchMutationGuard {
registry: Arc::clone(self),
mutation,
}
}
/// Replay and unsupported runtime mode transitions revoke tokens. They
/// never copy a durable generation into a same-session watch revision.
pub(crate) fn invalidate(&self) {
self.state
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.invalidate_all();
}
/// A write mode without a physical completion owner cannot establish
/// same-session coverage again. Keep the global fallback until restart.
pub(crate) fn mark_coverage_unavailable(&self) {
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
state.exhausted = true;
state.invalidate_all();
}
pub(crate) fn observe(&self, owner: Uuid, request: &ScannerUsageWatchRequest) -> Option<ScannerUsageWatchKnown> {
self.observe_at(owner, request, Instant::now())
}
fn observe_at(&self, owner: Uuid, request: &ScannerUsageWatchRequest, now: Instant) -> Option<ScannerUsageWatchKnown> {
if owner.is_nil() || request.bucket_incarnation.is_nil() || request.bucket.is_empty() || request.bucket.len() > 63 {
return None;
}
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
if request.release {
if request.session_id == Some(self.session)
&& state.watches.get(request.bucket.as_str()).is_some_and(|watch| {
watch.owner == owner
&& watch.incarnation == request.bucket_incarnation
&& request.watch_id == Some(watch.token)
})
{
state.remove_watch(&request.bucket);
}
// Release is resource retirement, never an assertion of cleanliness.
return None;
}
if request.session_id.is_some_and(|session| session != self.session) {
return None;
}
if state.exhausted || state.untracked != 0 {
return None;
}
// Sweeps are bounded by the fixed watch limit. Never accumulate tombstones.
state.watches.retain(|_, watch| watch.expires_at > now);
state.watch_name_bytes = state.watches.keys().map(|key| key.len()).sum();
let matches = state.watches.get(request.bucket.as_str()).is_some_and(|watch| {
watch.owner == owner
&& watch.incarnation == request.bucket_incarnation
&& request.watch_id.is_none_or(|id| id == watch.token)
});
if !matches {
if request.watch_id.is_some() {
return None;
}
state.remove_watch(&request.bucket);
if state.watches.len() >= SCANNER_USAGE_WATCH_MAX_ENTRIES
|| state.watch_name_bytes + request.bucket.len() > SCANNER_USAGE_WATCH_MAX_NAME_BYTES
{
return None;
}
let key: Arc<str> = Arc::from(request.bucket.as_str());
state.watch_name_bytes += key.len();
state.watches.insert(
key,
Watch {
owner,
token: Uuid::new_v4(),
incarnation: request.bucket_incarnation,
revision: 1,
expires_at: now + Duration::from_millis(SCANNER_USAGE_WATCH_LEASE_MS),
},
);
}
let inflight = state.active.get(request.bucket.as_str()).copied().unwrap_or(0);
let coverage_epoch = state.coverage_epoch;
let watch = state.watches.get_mut(request.bucket.as_str())?;
watch.expires_at = now + Duration::from_millis(SCANNER_USAGE_WATCH_LEASE_MS);
Some(ScannerUsageWatchKnown {
owner_id: owner,
session_id: self.session,
watch_id: watch.token,
bucket_incarnation: watch.incarnation,
revision: watch.revision,
inflight,
coverage_version: SCANNER_USAGE_WATCH_COVERAGE_VERSION,
coverage_epoch,
lease_remaining_ms: SCANNER_USAGE_WATCH_LEASE_MS,
})
}
}
impl Drop for UsageWatchMutationGuard {
fn drop(&mut self) {
let mut state = self.registry.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
match &self.mutation {
Mutation::Tracked(bucket) => {
// Advance before exposing inflight=0, including canceled and
// failed operations whose physical outcome may be uncertain.
state.advance_bucket(bucket);
if let Some(count) = state.active.get_mut(bucket) {
*count -= 1;
if *count == 0 {
state.active.remove(bucket);
state.active_name_bytes -= bucket.len();
}
} else {
state.exhausted = true;
state.invalidate_all();
}
}
Mutation::Untracked => {
if state.untracked != 0 {
state.untracked -= 1;
}
}
}
}
}
#[cfg(test)]
mod tests;
@@ -0,0 +1,321 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
fn request(bucket: &str) -> ScannerUsageWatchRequest {
ScannerUsageWatchRequest {
bucket: bucket.into(),
bucket_incarnation: Uuid::from_u128(7),
watch_id: None,
release: false,
session_id: None,
}
}
fn query(registry: &UsageWatchRegistry, request: &ScannerUsageWatchRequest, at: Instant) -> ScannerUsageWatchKnown {
registry
.observe_at(Uuid::from_u128(1), request, at)
.expect("known bucket proof")
}
#[test]
fn watch_installation_detects_an_already_active_mutation() {
let registry = Arc::new(UsageWatchRegistry::default());
let now = Instant::now();
let mutation = registry.begin(Some("bucket"));
let before = query(&registry, &request("bucket"), now);
assert_eq!(before.inflight, 1);
drop(mutation);
let after = query(&registry, &request("bucket"), now);
assert_eq!(after.inflight, 0);
assert!(after.revision > before.revision);
assert!(!after.unchanged_since(&before));
}
#[test]
fn every_begin_and_physical_tail_end_advances_only_the_target_bucket() {
let registry = Arc::new(UsageWatchRegistry::default());
let now = Instant::now();
let before = query(&registry, &request("cold"), now);
let hot = registry.begin(Some("hot"));
assert!(query(&registry, &request("cold"), now).unchanged_since(&before));
drop(hot);
assert!(query(&registry, &request("cold"), now).unchanged_since(&before));
let owner = Arc::new(registry.begin(Some("cold")));
let physical_tail = Arc::clone(&owner);
let during = query(&registry, &request("cold"), now);
assert_eq!(during.revision, before.revision + 1);
assert_eq!(during.inflight, 1);
drop(owner);
assert_eq!(query(&registry, &request("cold"), now).inflight, 1);
drop(physical_tail);
let after = query(&registry, &request("cold"), now);
assert_eq!(after.revision, before.revision + 2);
assert_eq!(after.inflight, 0);
}
#[test]
fn expiry_reincarnation_and_restart_do_not_resurrect_old_tokens() {
let registry = UsageWatchRegistry::default();
let now = Instant::now();
let before = query(&registry, &request("bucket"), now);
let mut refresh = request("bucket");
refresh.watch_id = Some(before.watch_id);
let expired = now + Duration::from_millis(SCANNER_USAGE_WATCH_LEASE_MS);
assert_eq!(registry.observe_at(Uuid::from_u128(1), &refresh, expired), None);
let after = query(&registry, &request("bucket"), expired);
assert_ne!(after.watch_id, before.watch_id);
assert!(!after.unchanged_since(&before));
refresh.bucket_incarnation = Uuid::new_v4();
assert_eq!(registry.observe_at(Uuid::from_u128(1), &refresh, expired), None);
refresh.watch_id = None;
let recreated = query(&registry, &refresh, expired);
assert_ne!(recreated.watch_id, after.watch_id);
let restarted = UsageWatchRegistry::default();
let restart_proof = query(&restarted, &refresh, expired);
assert!(!restart_proof.unchanged_since(&recreated));
println!(
"usage_assertion: {}",
serde_json::json!({
"case_id": "USAGE-12", "assertion_id": "USAGE-12-A05", "status": "PASS",
"evidence": {"expired_token": before.watch_id, "replacement_token": after.watch_id, "lease_ms": SCANNER_USAGE_WATCH_LEASE_MS, "expired_proof_accepted": false}
})
);
println!(
"usage_assertion: {}",
serde_json::json!({
"case_id": "USAGE-12", "assertion_id": "USAGE-12-A06", "status": "PASS",
"evidence": {"previous_session": recreated.session_id, "new_session": restart_proof.session_id, "unchanged": restart_proof.unchanged_since(&recreated)}
})
);
println!(
"usage_cleanup: {}",
serde_json::json!({"case_id": "USAGE-12", "status": "PASS", "scope": "process-local"})
);
}
#[test]
fn watch_capacity_is_bounded_and_rejected_entries_are_unknown() {
let registry = UsageWatchRegistry::default();
let now = Instant::now();
for index in 0..SCANNER_USAGE_WATCH_MAX_ENTRIES {
query(&registry, &request(&format!("bucket-{index}")), now);
}
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("overflow"), now), None);
let state = registry.state.lock().expect("registry state");
assert_eq!(state.watches.len(), SCANNER_USAGE_WATCH_MAX_ENTRIES);
assert!(state.watch_name_bytes <= SCANNER_USAGE_WATCH_MAX_NAME_BYTES);
}
#[test]
fn active_mutation_overflow_revokes_all_proofs_until_untracked_work_ends() {
let registry = Arc::new(UsageWatchRegistry::default());
let now = Instant::now();
let old = query(&registry, &request("cold"), now);
let mut active = Vec::new();
for index in 0..SCANNER_USAGE_WATCH_MAX_ACTIVE_BUCKETS {
active.push(registry.begin(Some(&format!("active-{index}"))));
}
let overflow = registry.begin(Some("not-in-active-map"));
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("cold"), now), None);
{
let state = registry.state.lock().expect("registry state");
assert_eq!(state.active.len(), SCANNER_USAGE_WATCH_MAX_ACTIVE_BUCKETS);
assert!(state.active_name_bytes <= SCANNER_USAGE_WATCH_MAX_ACTIVE_NAME_BYTES);
}
drop(overflow);
let new = query(&registry, &request("cold"), now);
assert_ne!(new.watch_id, old.watch_id);
assert!(new.coverage_epoch > old.coverage_epoch);
drop(active);
println!(
"usage_assertion: {}",
serde_json::json!({
"case_id": "USAGE-12", "assertion_id": "USAGE-12-A04", "status": "PASS",
"evidence": {"active_capacity": SCANNER_USAGE_WATCH_MAX_ACTIVE_BUCKETS, "previous_token": old.watch_id, "replacement_token": new.watch_id, "previous_coverage": old.coverage_epoch, "new_coverage": new.coverage_epoch, "active_after_cleanup": registry.state.lock().expect("drained registry").active.len()}
})
);
println!(
"usage_cleanup: {}",
serde_json::json!({"case_id": "USAGE-12", "status": "PASS", "scope": "process-local"})
);
}
#[test]
fn unknown_scope_and_replay_invalidate_without_reusing_generation() {
let registry = Arc::new(UsageWatchRegistry::default());
let now = Instant::now();
let old = query(&registry, &request("bucket"), now);
let mutation = registry.begin(None);
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("bucket"), now), None);
drop(mutation);
let new = query(&registry, &request("bucket"), now);
assert_ne!(new.watch_id, old.watch_id);
registry.invalidate();
let replayed = query(&registry, &request("bucket"), now);
assert_ne!(replayed.watch_id, new.watch_id);
assert!(replayed.coverage_epoch > new.coverage_epoch);
}
#[test]
fn revision_exhaustion_invalidates_the_token_without_aba() {
let registry = Arc::new(UsageWatchRegistry::default());
let now = Instant::now();
let old = query(&registry, &request("bucket"), now);
registry
.state
.lock()
.expect("registry state")
.watches
.get_mut("bucket")
.expect("installed watch")
.revision = u64::MAX - 1;
let mutation = registry.begin(Some("bucket"));
let refresh = ScannerUsageWatchRequest {
watch_id: Some(old.watch_id),
..request("bucket")
};
assert_eq!(registry.observe_at(Uuid::from_u128(1), &refresh, now), None);
let during = query(&registry, &request("bucket"), now);
assert_ne!(during.watch_id, old.watch_id);
assert_eq!(during.inflight, 1);
drop(mutation);
assert_eq!(query(&registry, &request("bucket"), now).revision, during.revision + 1);
}
#[test]
fn coverage_epoch_exhaustion_stays_unknown_without_wrapping() {
let registry = UsageWatchRegistry::default();
let now = Instant::now();
query(&registry, &request("bucket"), now);
registry.state.lock().expect("registry state").coverage_epoch = u64::MAX - 1;
registry.invalidate();
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("bucket"), now), None);
registry.invalidate();
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("bucket"), now), None);
let state = registry.state.lock().expect("registry state");
assert!(state.exhausted);
assert!(state.watches.is_empty());
assert_eq!(state.coverage_epoch, u64::MAX - 1);
}
#[test]
fn proof_comparison_rejects_every_changed_identity_and_busy_endpoint() {
let registry = UsageWatchRegistry::default();
let original = query(&registry, &request("bucket"), Instant::now());
for field in 0..10 {
let mut changed = original.clone();
match field {
0 => changed.owner_id = Uuid::new_v4(),
1 => changed.session_id = Uuid::new_v4(),
2 => changed.watch_id = Uuid::new_v4(),
3 => changed.bucket_incarnation = Uuid::new_v4(),
4 => changed.revision += 1,
5 => changed.inflight = 1,
6 => changed.coverage_version += 1,
7 => changed.coverage_epoch += 1,
8 => changed.lease_remaining_ms = 0,
_ => {
changed.revision = 0;
changed.watch_id = Uuid::nil();
}
}
assert!(!changed.unchanged_since(&original));
}
}
#[test]
fn watch_contention_cost_preserves_all_mutation_boundaries() {
let registry = Arc::new(UsageWatchRegistry::default());
let before = query(&registry, &request("hot"), Instant::now());
let threads = 8_u64;
let writes_per_thread = 10_000_u64;
let started = Instant::now();
std::thread::scope(|scope| {
for _ in 0..threads {
let registry = Arc::clone(&registry);
scope.spawn(move || {
for _ in 0..writes_per_thread {
drop(registry.begin(Some("hot")));
}
});
}
});
let elapsed = started.elapsed();
let after = query(&registry, &request("hot"), Instant::now());
let writes = threads * writes_per_thread;
assert_eq!(after.revision, before.revision + 2 * writes);
assert_eq!(after.inflight, 0);
assert!(registry.state.lock().expect("drained registry").active.is_empty());
eprintln!(
"usage_watch_cost threads={threads} writes={writes} elapsed_us={} wall_ns_per_begin_end={}",
elapsed.as_micros(),
elapsed.as_nanos() / u128::from(writes)
);
}
#[test]
fn exact_release_preserves_other_sessions_tokens_and_active_mutations() {
let registry = Arc::new(UsageWatchRegistry::default());
let owner = Uuid::from_u128(1);
let now = Instant::now();
let before = query(&registry, &request("bucket"), now);
let release = ScannerUsageWatchRequest {
watch_id: Some(before.watch_id),
session_id: Some(before.session_id),
release: true,
..request("bucket")
};
for case in 0..5 {
let mut wrong = release.clone();
let mut wrong_owner = owner;
match case {
0 => wrong.watch_id = Some(Uuid::new_v4()),
1 => wrong.session_id = Some(Uuid::new_v4()),
2 => wrong.bucket_incarnation = Uuid::new_v4(),
3 => wrong.watch_id = None,
_ => wrong_owner = Uuid::new_v4(),
}
assert_eq!(registry.observe_at(wrong_owner, &wrong, now), None);
assert!(query(&registry, &request("bucket"), now).unchanged_since(&before));
}
let mutation = registry.begin(Some("bucket"));
assert_eq!(registry.observe_at(owner, &release, now), None);
assert!(registry.state.lock().expect("registry state").watches.is_empty());
let replacement = query(&registry, &request("bucket"), now);
assert_ne!(replacement.watch_id, before.watch_id);
assert_eq!(replacement.inflight, 1, "release cannot discard physical mutation ownership");
assert_eq!(registry.observe_at(owner, &release, now), None);
assert_eq!(query(&registry, &request("bucket"), now).watch_id, replacement.watch_id);
drop(mutation);
let after = query(&registry, &request("bucket"), now);
assert_eq!(after.inflight, 0);
assert!(after.revision > replacement.revision);
}
#[test]
fn completed_watch_release_admits_later_buckets_before_lease_expiry() {
let registry = UsageWatchRegistry::default();
let now = Instant::now();
let mut completed = Vec::new();
for index in 0..SCANNER_USAGE_WATCH_MAX_ENTRIES {
let req = request(&format!("bucket-{index:03}"));
let known = query(&registry, &req, now);
completed.push(ScannerUsageWatchRequest {
watch_id: Some(known.watch_id),
session_id: Some(known.session_id),
release: true,
..req
});
}
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("later-bucket"), now), None);
for release in completed {
assert_eq!(registry.observe_at(Uuid::from_u128(1), &release, now), None);
}
let state = registry.state.lock().expect("registry state");
assert_eq!(state.watches.len(), 0);
assert_eq!(state.watch_name_bytes, 0);
drop(state);
assert!(query(&registry, &request("later-bucket"), now).is_well_formed());
}
@@ -107,7 +107,7 @@ pub(super) async fn run(
result.state = ItemState::Prepared;
result.detail = None;
etag = checkpoint(api, job, &etag, guard).await?;
let mut file = match stage(api, job, index, &info, &opts, guard).await {
let staged = match verify_source(api, job, index, &info, &opts, guard).await {
Ok(file) => file,
Err(IntegrityError::Conflict) => {
job.results[index].state = ItemState::Stale;
@@ -125,6 +125,7 @@ pub(super) async fn run(
if job.request.mode == JobMode::Audit {
job.results[index].state = ItemState::Verified;
} else {
let mut file = staged.ok_or(IntegrityError::Invalid("missing migration payload"))?;
let snapshot = validate_destination(api, job).await?;
file.rewind().await?;
let raw = sha256_bytes(&expected)?;
@@ -231,30 +232,29 @@ async fn ensure_running(api: &Arc<ECStore>, job: &Job, guard: &rustfs_lock::Name
Ok(())
}
async fn stage(
async fn verify_source(
api: &Arc<ECStore>,
job: &Job,
index: usize,
info: &ObjectInfo,
opts: &ObjectOptions,
guard: &rustfs_lock::NamespaceLockGuard,
) -> Result<tokio::fs::File> {
) -> Result<Option<tokio::fs::File>> {
let mut reader = api
.get_object_reader(&job.bucket, &job.request.items[index].key, None, Default::default(), opts)
.await?;
if fingerprint(&reader.object_info) != fingerprint(info) {
return Err(IntegrityError::Conflict);
}
let std_file = tokio::task::spawn_blocking(tempfile::tempfile)
.await
.map_err(|_| IntegrityError::Invalid("staging task failed"))??;
let mut file = tokio::fs::File::from_std(std_file);
let mut file = staging_file(job.request.mode, tempfile::tempfile).await?;
let expected = job.results[index]
.expected_sha256
.as_deref()
.ok_or(IntegrityError::Invalid("missing expected digest"))?;
copy_verified(api, job, &mut reader.stream, Some(&mut file), info.size, expected, guard).await?;
file.flush().await?;
copy_verified(api, job, &mut reader.stream, file.as_mut(), info.size, expected, guard).await?;
if let Some(file) = file.as_mut() {
file.flush().await?;
}
let after = api.get_object_info(&job.bucket, &job.request.items[index].key, opts).await?;
if fingerprint(&after) != fingerprint(info) {
return Err(IntegrityError::Conflict);
@@ -262,6 +262,20 @@ async fn stage(
Ok(file)
}
async fn staging_file(
mode: JobMode,
create: impl FnOnce() -> std::io::Result<std::fs::File> + Send + 'static,
) -> Result<Option<tokio::fs::File>> {
// Migration must publish the exact bytes verified here. Audit has no second consumer.
if mode == JobMode::Audit {
return Ok(None);
}
let file = tokio::task::spawn_blocking(create)
.await
.map_err(|_| IntegrityError::Invalid("staging task failed"))??;
Ok(Some(tokio::fs::File::from_std(file)))
}
async fn copy_verified(
api: &Arc<ECStore>,
job: &Job,
@@ -466,3 +480,31 @@ impl tokio::io::AsyncRead for ThrottledFile {
}
}
}
#[cfg(test)]
mod staging_tests {
use super::*;
#[tokio::test]
async fn audit_does_not_need_a_writable_payload_staging_volume() {
let result = staging_file(JobMode::Audit, || Err(std::io::Error::other("staging volume full")))
.await
.expect("audit must not open staging storage");
assert!(result.is_none());
}
#[tokio::test]
async fn migration_requires_writable_payload_staging() {
assert!(
staging_file(JobMode::Migrate, || { Err(std::io::Error::other("staging volume full")) })
.await
.is_err()
);
let mut file = staging_file(JobMode::Migrate, tempfile::tempfile).await.unwrap().unwrap();
file.write_all(b"verified bytes").await.unwrap();
file.rewind().await.unwrap();
let mut bytes = Vec::new();
file.read_to_end(&mut bytes).await.unwrap();
assert_eq!(bytes, b"verified bytes");
}
}
@@ -3038,7 +3038,7 @@ impl NotificationSys {
return Err(Error::other("scanner scoped dirty usage capability requires scoped acknowledgements"));
}
};
if by_host.insert(host.clone(), acknowledgement).is_some() {
if by_host.insert(host, acknowledgement).is_some() {
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
}
}
@@ -3081,6 +3081,42 @@ impl NotificationSys {
Ok(true)
}
/// Query exactly the requested peers and buckets. Missing peers and old
/// protocol endpoints fail the query instead of fabricating clean entries.
pub async fn scanner_usage_watches(
&self,
requests: Vec<(String, uuid::Uuid, Vec<crate::runtime::usage_watch::ScannerUsageWatchRequest>)>,
) -> Result<Vec<(String, Vec<crate::runtime::usage_watch::ScannerUsageWatchObservation>)>> {
let clients = self
.peer_clients
.iter()
.flatten()
.map(|client| (client.grid_host.clone(), client.clone()))
.collect::<HashMap<_, _>>();
let mut seen = std::collections::HashSet::new();
let mut futures = Vec::with_capacity(requests.len());
for (host, owner, entries) in requests {
if !seen.insert(host.clone()) {
return Err(Error::other("duplicate usage watch peer"));
}
let client = clients
.get(&host)
.cloned()
.ok_or_else(|| Error::other("usage watch peer is not reachable"))?;
futures.push(async move {
let observations = timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_usage_watch(owner, entries))
.await
.map_err(|_| Error::other("usage watch peer timed out"))??;
Ok::<_, Error>((host, observations))
});
}
let mut results = Vec::with_capacity(futures.len());
for result in join_all(futures).await {
results.push(result?);
}
Ok(results)
}
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<ScannerDirtyUsageAcknowledgement>) -> Result<bool> {
let mut by_host = HashMap::with_capacity(acknowledgements.len());
for acknowledgement in acknowledgements {
@@ -3088,7 +3124,7 @@ impl NotificationSys {
ScannerDirtyUsageAcknowledgement::Generation { host, .. }
| ScannerDirtyUsageAcknowledgement::Scoped { host, .. } => host.clone(),
};
if by_host.insert(host.clone(), acknowledgement).is_some() {
if by_host.insert(host, acknowledgement).is_some() {
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
}
}
@@ -802,7 +802,6 @@ impl ECStore {
let set = set.clone();
let rx = rx.clone();
let bucket = bucket.clone();
let entry_tasks = entry_tasks.clone();
let job = tokio::spawn(async move {
let list_rx = rx.clone();
@@ -8144,9 +8144,10 @@ mod tests {
assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 4);
assert_eq!(setup.available_shards(), 4);
assert_eq!(setup.deferred_shards(), 4);
for index in 4..8 {
assert!(setup.attempted.len() >= 8);
for (index, attempted) in setup.attempted.iter().enumerate().take(8).skip(4) {
assert_eq!(calls.for_disk(disk_call_counters::KIND_DISK_SHARD_OPEN, index), 0);
assert!(!setup.attempted[index], "parity must stay unopened until needed");
assert!(!attempted, "parity must stay unopened until needed");
}
let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
assert!(error.is_none(), "healthy decode failed: {error:?}");
+18 -8
View File
@@ -5611,6 +5611,20 @@ impl SetDisks {
metadata_sys::find_bucket_incarnation_id_in(&self.ctx, bucket).await
}
/// Confirm deletion of this exact generation using immutable retirement evidence.
/// Missing directories or unavailable bucket metadata do not prove retirement.
///
/// # Errors
/// Returns an error when the instance owner or retirement record is unavailable,
/// malformed, or bound to a different deployment or generation.
pub async fn is_bucket_generation_retired(&self, bucket: &str, incarnation: Uuid) -> Result<bool> {
let store = metadata_sys::object_store_in(&self.ctx).await?;
if !Arc::ptr_eq(&store.ctx, &self.ctx) {
return Err(Error::other("bucket retirement metadata belongs to another instance"));
}
crate::bucket::retirement::is_retired(store, bucket, incarnation).await
}
/// Admit one short scanner cache publication under this set's instance
/// movement fence. The caller must hold the returned guard through its
/// final conditional cache write; no scan-round work belongs under it.
@@ -6045,14 +6059,10 @@ fn should_use_inline_fast_path_with_plan(
// a stale/corrupt marker cannot route an oversized payload into the
// in-memory decoder. The independent direct-memory reader remains capped
// at 128 KiB below.
if !plan.is_plain()
|| shape.object_size > usize::try_from(INLINE_FAST_PATH_MAX_OBJECT_SIZE).expect("inline fast path limit fits usize")
|| fi.erasure.data_blocks == 0
|| fi.erasure.block_size == 0
{
return false;
}
true
plan.is_plain()
&& shape.object_size <= usize::try_from(INLINE_FAST_PATH_MAX_OBJECT_SIZE).expect("inline fast path limit fits usize")
&& fi.erasure.data_blocks != 0
&& fi.erasure.block_size != 0
}
enum SmallWritePath {
@@ -515,7 +515,7 @@ impl SetDisks {
let owner = context
.namespace_owner
.take()
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
.unwrap_or_else(|| self.ctx.begin_namespace_commit_for_bucket(bucket));
context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
let can_commit = context.can_commit;
let can_try_local_fast_path = disks.len() == self.set_drive_count
@@ -791,7 +791,7 @@ impl SetDisks {
let namespace_owner = context
.namespace_owner
.clone()
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
.unwrap_or_else(|| self.ctx.begin_namespace_commit_for_bucket(bucket));
context.namespace_owner = Some(Arc::new((namespace_owner, guard.clone())));
}
let can_commit = context.can_commit;
@@ -821,7 +821,7 @@ impl SetDisks {
let owner = context
.namespace_owner
.take()
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
.unwrap_or_else(|| self.ctx.begin_namespace_commit_for_bucket(bucket));
context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
}
let DeleteRecoveryContext {
@@ -1025,7 +1025,7 @@ impl SetDisks {
let namespace_owner = if dry_run {
None
} else {
Some(namespace_owner.unwrap_or_else(|| self.ctx.begin_namespace_commit()))
Some(namespace_owner.unwrap_or_else(|| self.ctx.begin_namespace_commit_for_bucket(bucket)))
};
if !dry_run && metadata_may_change {
self.invalidate_get_object_metadata_cache(bucket, object).await;
+6 -6
View File
@@ -1285,7 +1285,7 @@ impl SetDisks {
result.after.drives.push(HealDriveInfo {
uuid: "".to_string(),
endpoint: self.set_endpoints[index].to_string(),
state: drive_state.to_string(),
state: drive_state,
});
}
@@ -2060,7 +2060,7 @@ impl SetDisks {
_write_lock_guard.clone(),
_read_repair_write_lock_guard.clone(),
bucket_heal_scope.clone(),
self.ctx.begin_namespace_commit(),
self.ctx.begin_namespace_commit_for_bucket(bucket),
))
});
match disk
@@ -3102,7 +3102,7 @@ impl SetDisks {
result.after.drives.push(HealDriveInfo {
uuid: "".to_string(),
endpoint,
state: drive_state.to_string(),
state: drive_state,
});
}
@@ -3147,7 +3147,7 @@ impl SetDisks {
_ => DriveState::Corrupt.to_string(),
},
};
result.after.drives[index].state = drive_state.to_string();
result.after.drives[index].state = drive_state;
}
}
@@ -3761,7 +3761,7 @@ impl SetDisks {
namespace_owner: (!opts.dry_run).then(|| {
Arc::new((
scope.clone(),
self.ctx.begin_namespace_commit(),
self.ctx.begin_namespace_commit_for_bucket(bucket),
_write_lock_guard.clone(),
transaction_guard.clone(),
)) as Arc<dyn Send + Sync>
@@ -10994,7 +10994,7 @@ mod heal_result_report_tests {
set.heal_object(bucket, object, "", &heal_opts),
);
let put_info = put_res.expect("overwrite must succeed under lock serialization");
last_etag = put_info.etag.clone().unwrap_or_default();
last_etag = put_info.etag.unwrap_or_default();
// Heal outcome is unconstrained (may hit the tail race and report a
// retryable error); the invariant is checked on the end state.
drop(heal_res);
+7 -3
View File
@@ -610,6 +610,9 @@ impl SetDisks {
new_disk.enable_health_check();
if new_disk.is_local() {
if let crate::disk::Disk::Local(wrapper) = new_disk.as_ref() {
wrapper.get_disk().usage_watch_binding.bind(&self.ctx.usage_watches);
}
let local_disk_map = runtime_sources::local_disk_map_handle();
let mut global_local_disk_map = local_disk_map.write().await;
let path = new_disk.endpoint().to_string();
@@ -690,9 +693,10 @@ impl SetDisks {
return Err(Error::other("DriveID: offline"));
}
for i in 0..self.format.erasure.sets.len() {
for j in 0..self.format.erasure.sets[0].len() {
if fm.erasure.this == self.format.erasure.sets[i][j] {
let set_drive_count = self.format.erasure.sets[0].len();
for (i, set) in self.format.erasure.sets.iter().enumerate() {
for (j, drive) in set.iter().enumerate().take(set_drive_count) {
if fm.erasure.this == *drive {
return Ok((i, j));
}
}
+2 -2
View File
@@ -4688,7 +4688,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
.with_multipart_commit_owner(multipart_commit_owner)
.with_namespace_commit_guard(
(!crate::bucket::utils::is_meta_bucketname(&commit_bucket))
.then(|| commit_set.ctx.begin_namespace_commit()),
.then(|| commit_set.ctx.begin_namespace_commit_for_bucket(&commit_bucket)),
),
)
.await;
@@ -11546,7 +11546,7 @@ mod tests {
break;
}
key_marker = page.next_key_marker.clone();
upload_id_marker = page.next_upload_id_marker.clone();
upload_id_marker = page.next_upload_id_marker;
assert!(key_marker.is_some(), "a truncated page must carry a next key marker");
assert!(upload_id_marker.is_some(), "a truncated upload page must carry a next upload marker");
}
+7 -6
View File
@@ -3918,7 +3918,7 @@ impl SetDisks {
explicit_delete: true,
transaction_guard: transaction_guard.cloned(),
namespace_owner: Some(Arc::new((
self.ctx.begin_namespace_commit(),
self.ctx.begin_namespace_commit_for_bucket(bucket),
lock_guard.cloned(),
opts.object_lock_config_snapshot.clone(),
opts.scanner_publication_commit_scope.clone(),
@@ -5661,7 +5661,8 @@ impl SetDisks {
.with_publication_scope(commit_scanner_publication_scope.clone())
.with_rollback_receipt(commit_rollback_receipt.clone())
.with_namespace_commit_guard(
(!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
(!is_meta_bucketname(&commit_bucket))
.then(|| commit_set.ctx.begin_namespace_commit_for_bucket(&commit_bucket)),
),
)
.await;
@@ -9063,7 +9064,7 @@ impl SetDisks {
let disks = self.disk_inventory().await;
let namespace_owner = (!is_meta_bucketname(bucket)).then(|| {
Arc::new((
self.ctx.begin_namespace_commit(),
self.ctx.begin_namespace_commit_for_bucket(bucket),
commit_context.guard.clone(),
commit_context.transaction_guard.clone(),
commit_context
@@ -10469,7 +10470,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let rollback_dir = Uuid::new_v4();
let namespace_owner = (!is_meta_bucketname(bucket)).then(|| {
Arc::new((
self.ctx.begin_namespace_commit(),
self.ctx.begin_namespace_commit_for_bucket(bucket),
Arc::clone(&transaction_guards),
opts.delete_mutation_owner.clone(),
opts.object_lock_config_snapshot.clone(),
@@ -10973,7 +10974,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
work_budget: opts.delete_recovery_work_budget.clone(),
explicit_delete,
namespace_owner: Some(Arc::new((
self.ctx.begin_namespace_commit(),
self.ctx.begin_namespace_commit_for_bucket(bucket),
opts.delete_mutation_owner.clone(),
opts.object_lock_config_snapshot.clone(),
))),
@@ -11329,7 +11330,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
work_budget: opts.delete_recovery_work_budget.clone(),
transaction_guard: transaction_guard.clone(),
namespace_owner: Some(Arc::new((
self.ctx.begin_namespace_commit(),
self.ctx.begin_namespace_commit_for_bucket(bucket),
_lock_guard.clone(),
opts.object_lock_config_snapshot.clone(),
opts.scanner_publication_commit_scope.clone(),
@@ -25,14 +25,15 @@ pub(crate) mod heal {
pub(crate) mod internode {
pub(crate) use rustfs_storage_api::{
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CHECKPOINT_COVERAGE_QUERY,
NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
NS_SCANNER_COMPLETE_OBSERVATION_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_DURABLE_PROGRESS_QUERY,
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY,
NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse,
PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN,
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY,
PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse,
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilities,
NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN,
PUT_FILE_AUTH_TRAILER_MAC_LEN, PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
PutFileCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
};
+1 -1
View File
@@ -385,7 +385,7 @@ impl ECStore {
dry_run: opts.dry_run,
explicit_delete,
namespace_owner: (!opts.dry_run)
.then(|| Arc::new((scope.clone(), store.ctx.begin_namespace_commit())) as Arc<dyn Send + Sync>),
.then(|| Arc::new((scope.clone(), store.ctx.begin_namespace_commit_for_bucket(&bucket))) as Arc<dyn Send + Sync>),
..crate::set_disk::DeleteRecoveryContext::new(Some(scope.incarnation), &can_commit)
};
let mut summary = crate::set_disk::DeleteRecoverySummary::default();
+34 -35
View File
@@ -17932,10 +17932,10 @@ mod tests {
.expect("shared-tuple recursive prefix delete should commit");
assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
for pool_idx in 0..2 {
assert!(store.pools.len() >= 2);
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
assert!(
store.pools[pool_idx]
.get_disks_by_key(object)
pool.get_disks_by_key(object)
.load_file_info_versions_exact(bucket, object)
.await
.expect("shared-tuple post-delete metadata should decode")
@@ -19768,8 +19768,9 @@ mod tests {
}
wait_for_expiry_workers_idle(&store).await;
let mut free_version_ids = Vec::new();
for pool_idx in 0..2 {
let retained = store.pools[pool_idx]
assert!(store.pools.len() >= 2);
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
let retained = pool
.get_disks_by_key(object)
.load_file_info_versions_exact(&bucket, object)
.await
@@ -19805,10 +19806,10 @@ mod tests {
);
wait_for_expiry_workers_idle(&store).await;
assert_eq!(backend.remove_count().await, 1, "shared remote tuple should be deleted exactly once");
for pool_idx in 0..2 {
assert!(store.pools.len() >= 2);
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
assert!(
store.pools[pool_idx]
.get_disks_by_key(object)
pool.get_disks_by_key(object)
.load_file_info_versions_exact(&bucket, object)
.await
.expect("post-delete metadata should decode")
@@ -19858,9 +19859,10 @@ mod tests {
.expect("multi-pool source bucket should be created");
let mut remote_tuples = Vec::new();
for pool_idx in 0..2 {
assert!(store.pools.len() >= 2);
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
let mut reader = PutObjReader::from_vec(vec![b'a' + pool_idx as u8; 1024 * 1024]);
let source = store.pools[pool_idx]
let source = pool
.put_object(
bucket,
object,
@@ -19872,31 +19874,27 @@ mod tests {
)
.await
.expect("pool-local source should be written");
store.pools[pool_idx]
.transition_object(
bucket,
object,
&ObjectOptions {
transition: TransitionOptions {
status: TRANSITION_PENDING.to_string(),
tier: tier_name.to_string(),
etag: source.etag.clone().expect("pool-local source should have an etag"),
..Default::default()
},
mod_time: source.mod_time,
pool.transition_object(
bucket,
object,
&ObjectOptions {
transition: TransitionOptions {
status: TRANSITION_PENDING.to_string(),
tier: tier_name.to_string(),
etag: source.etag.clone().expect("pool-local source should have an etag"),
..Default::default()
},
)
.await
.expect("pool-local source should transition");
let transitioned = store.pools[pool_idx]
mod_time: source.mod_time,
..Default::default()
},
)
.await
.expect("pool-local source should transition");
let transitioned = pool
.get_object_info(bucket, object, &ObjectOptions::default())
.await
.expect("pool-local transitioned source should be readable");
remote_tuples.push((
transitioned.transitioned_object.name.clone(),
transitioned.transitioned_object.version_id.clone(),
));
remote_tuples.push((transitioned.transitioned_object.name.clone(), transitioned.transitioned_object.version_id));
}
assert_ne!(remote_tuples[0], remote_tuples[1], "fixture must use distinct remote cleanup tuples");
assert_eq!(backend.object_count().await, 2);
@@ -19906,8 +19904,9 @@ mod tests {
.delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
.await
.expect("store-level delete should remove both pool copies");
for pool_idx in 0..2 {
let retained = store.pools[pool_idx]
assert!(store.pools.len() >= 2);
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
let retained = pool
.get_disks_by_key(object)
.load_file_info_versions_exact(bucket, object)
.await
@@ -19946,10 +19945,10 @@ mod tests {
removed.sort();
remote_tuples.sort();
assert_eq!(removed, remote_tuples, "recovery must use each pool's exact remote tuple");
for pool_idx in 0..2 {
assert!(store.pools.len() >= 2);
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
assert!(
store.pools[pool_idx]
.get_disks_by_key(object)
pool.get_disks_by_key(object)
.load_file_info_versions_exact(bucket, object)
.await
.expect("post-recovery pool metadata should decode")
+10 -10
View File
@@ -8504,7 +8504,7 @@ mod test {
let mut results = Vec::new();
while let Some(entry) = out_rx.recv().await {
results.push(entry.name.clone());
results.push(entry.name);
}
handle.await.unwrap().unwrap();
@@ -8533,7 +8533,7 @@ mod test {
let mut results = Vec::new();
while let Some(entry) = out_rx.recv().await {
results.push(entry.name.clone());
results.push(entry.name);
}
handle.await.unwrap().unwrap();
@@ -9177,7 +9177,7 @@ mod test {
let mut results = Vec::new();
while let Some(entry) = out_rx.recv().await {
results.push(entry.name.clone());
results.push(entry.name);
}
handle.await.unwrap().unwrap();
@@ -9265,7 +9265,7 @@ mod test {
let mut results = Vec::new();
while let Some(entry) = out_rx.recv().await {
results.push(entry.name.clone());
results.push(entry.name);
}
handle.await.unwrap().unwrap();
@@ -9295,7 +9295,7 @@ mod test {
let mut results = Vec::new();
while let Some(entry) = out_rx.recv().await {
results.push(entry.name.clone());
results.push(entry.name);
}
handle.await.unwrap().unwrap();
@@ -9367,7 +9367,7 @@ mod test {
let mut results = Vec::new();
while let Some(entry) = out_rx.recv().await {
results.push(entry.name.clone());
results.push(entry.name);
}
handle.await.unwrap().unwrap();
@@ -9401,7 +9401,7 @@ mod test {
let mut results = Vec::new();
while let Some(entry) = out_rx.recv().await {
results.push(entry.name.clone());
results.push(entry.name);
}
handle.await.unwrap().unwrap();
@@ -9491,7 +9491,7 @@ mod test {
let mut results = Vec::new();
while let Some(entry) = out_rx.recv().await {
assert!(entry.is_object(), "the directory-marker object must shadow the same-named prefix dir");
results.push(entry.name.clone());
results.push(entry.name);
}
handle.await.unwrap().unwrap();
@@ -9514,7 +9514,7 @@ mod test {
let mut results = Vec::new();
while let Some(entry) = out_rx.recv().await {
results.push(entry.name.clone());
results.push(entry.name);
}
handle.await.unwrap().unwrap();
@@ -9538,7 +9538,7 @@ mod test {
let mut results = Vec::new();
while let Some(entry) = out_rx.recv().await {
results.push(entry.name.clone());
results.push(entry.name);
}
handle.await.unwrap().unwrap();
+75
View File
@@ -451,6 +451,8 @@ pub use object::{
mod peer;
mod rebalance;
#[cfg(test)]
mod usage_watch_tests;
#[cfg(test)]
pub(crate) use rebalance::PoolSpaceForTest;
pub(crate) mod utils;
@@ -829,6 +831,13 @@ impl ECStore {
runtime_sources::bucket_metadata_sys()
}
/// Recover this instance's weak metadata owner without accepting a
/// process-global store belonging to another scanner or test fixture.
pub async fn scanner_usage_watch_store_handle(&self) -> Option<Arc<Self>> {
let owner = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await?;
std::ptr::eq(self, owner.as_ref()).then_some(owner)
}
/// Get the global endpoints
pub fn endpoints(&self) -> EndpointServerPools {
runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into())
@@ -906,6 +915,72 @@ impl ECStore {
list_objects::scanner_namespace_mutation_generation().saturating_add(self.ctx.namespace_commit_generation())
}
/// Query named buckets against their current storage-owned incarnation.
/// Unknown identities, expired tokens and resource pressure remain explicit
/// Unknown entries; absence from a dirty list never creates a clean proof.
pub async fn scanner_usage_watch(
&self,
requests: &[crate::runtime::usage_watch::ScannerUsageWatchRequest],
) -> Vec<crate::runtime::usage_watch::ScannerUsageWatchObservation> {
use crate::runtime::usage_watch::ScannerUsageWatchObservation;
let mut observations = Vec::with_capacity(requests.len().min(32));
if requests.len() > 32 {
return observations;
}
let local_map = self.ctx.local_disk_map();
let local_disks = local_map.read().await;
let bindings_complete = !local_disks.is_empty()
&& local_disks.values().all(|disk| {
disk.as_ref().is_some_and(|disk| match disk.as_ref() {
crate::disk::Disk::Local(wrapper) => wrapper.get_disk().usage_watch_binding.matches(&self.ctx.usage_watches),
crate::disk::Disk::Remote(_) => false,
})
});
drop(local_disks);
for request in requests {
if request.release {
// Retirement needs the exact saved token, not current bucket
// metadata. Deleted buckets must not retain scarce watch slots.
self.ctx.usage_watches.observe(self.id, request);
observations.push(ScannerUsageWatchObservation {
bucket: request.bucket.clone(),
known: None,
});
continue;
}
let current = self.bucket_incarnation_id_from_disk(&request.bucket).await;
let known = match current {
Ok(incarnation) if bindings_complete && incarnation == request.bucket_incarnation && !incarnation.is_nil() => {
self.ctx.usage_watches.observe(self.id, request)
}
_ => None,
};
observations.push(ScannerUsageWatchObservation {
bucket: request.bucket.clone(),
known,
});
}
observations
}
/// Invalidate session proofs after replay or an unsupported producer mode
/// transition; this does not change durable dirty ACK responsibility.
pub fn invalidate_scanner_usage_watches(&self) {
self.ctx.usage_watches.invalidate();
}
pub fn scanner_usage_watch_session_id(&self) -> Uuid {
self.ctx.usage_watches.session_id()
}
/// Revoke an exact local token without awaiting storage or peer I/O. A
/// failed keepalive must not let a later successful query revive its proof.
pub fn scanner_release_usage_watch(&self, request: &crate::runtime::usage_watch::ScannerUsageWatchRequest) {
if request.release {
self.ctx.usage_watches.observe(self.id, request);
}
}
pub async fn scanner_data_movement_active(&self) -> bool {
let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
decommission || rebalance
+24 -9
View File
@@ -88,7 +88,7 @@ impl ECStore {
/// Resolve a quota lease against this request's captured storage instance.
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
let _owner = admit_local_disk(&self.ctx, &disk, id, false, None).await?;
Ok(disk)
}
@@ -106,7 +106,7 @@ impl ECStore {
let path = target.1.to_owned();
self.run_bucket_heal_at_incarnation(target.0, expected, &Default::default(), move |store, volume, _| async move {
let (disk, id) = local_disk_candidate(&store.ctx, &disk_ref).await?;
let _owner = admit_local_disk(&store.ctx, &disk, id, true).await?;
let _owner = admit_local_disk(&store.ctx, &disk, id, true, Some(&volume)).await?;
disk.write_metadata("", &volume, &path, fi).await.map_err(Into::into)
})
.await
@@ -127,7 +127,7 @@ impl ECStore {
let path = target.1.to_owned();
self.run_bucket_heal_at_incarnation(target.0, expected, &Default::default(), move |store, volume, _| async move {
let (disk, id) = local_disk_candidate(&store.ctx, &disk_ref).await?;
let owner = admit_local_disk(&store.ctx, &disk, id, true).await?;
let owner = admit_local_disk(&store.ctx, &disk, id, true, Some(&volume)).await?;
let scope = super::bucket_heal_scope(&volume).ok_or_else(|| StorageError::other("missing bucket heal scope"))?;
scope.check()?;
disk.delete_with_namespace_owner(&volume, &path, opts, Some(Arc::new((scope, owner))))
@@ -159,7 +159,7 @@ impl ECStore {
.map_err(|error| DiskError::other(error.to_string()))?,
);
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let owner = admit_local_disk(&self.ctx, &disk, id, true).await?;
let owner = admit_local_disk(&self.ctx, &disk, id, true, Some(target.0)).await?;
if fence.is_lock_lost() {
return Err(DiskError::other("bucket heal incarnation fence was lost"));
}
@@ -283,7 +283,7 @@ impl ECStore {
) -> DiskResult<crate::disk::multipart_source::MultipartSourceResponse> {
request.validate()?;
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
let owner = admit_local_disk(&self.ctx, &disk, id, request.operation.is_mutation()).await?;
let owner = admit_local_disk(&self.ctx, &disk, id, request.operation.is_mutation(), None).await?;
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
};
@@ -356,6 +356,7 @@ async fn admit_local_disk(
disk: &DiskStore,
disk_id: Option<Uuid>,
mutates_namespace: bool,
bucket: Option<&str>,
) -> DiskResult<Option<Arc<NamespaceCommitGuard>>> {
if !matches!(disk.as_ref(), Disk::Local(_)) {
return Err(DiskError::DiskNotFound);
@@ -378,7 +379,10 @@ async fn admit_local_disk(
}
// Admission linearizes under the registry read: replacement/quarantine
// before this point rejects; later changes do not revoke physical I/O.
Ok(mutates_namespace.then(|| ctx.begin_namespace_commit()))
Ok(mutates_namespace.then(|| match bucket {
Some(bucket) => ctx.begin_namespace_commit_for_bucket(bucket),
None => ctx.begin_namespace_commit(),
}))
}
async fn rename_local_data_with_ctx(
@@ -398,7 +402,18 @@ async fn rename_local_data_with_ctx(
}
let mutates_namespace = !is_meta_bucketname(source.0) || !is_meta_bucketname(destination.0);
let admit_started = rustfs_io_metrics::put_stage_timer();
let owner = admit_local_disk(ctx, &disk, disk_id, mutates_namespace).await?;
let owner = admit_local_disk(
ctx,
&disk,
disk_id,
mutates_namespace,
Some(if !is_meta_bucketname(destination.0) {
destination.0
} else {
source.0
}),
)
.await?;
if source.0 == crate::disk::RUSTFS_META_MULTIPART_BUCKET {
let Disk::Local(wrapper) = disk.as_ref() else {
return Err(DiskError::DiskNotFound);
@@ -444,7 +459,7 @@ async fn undo_local_write_with_ctx(
return Err(DiskError::other("target undo requires undo_write"));
}
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
let owner = admit_local_disk(ctx, &disk, disk_id, !is_meta_bucketname(volume)).await?;
let owner = admit_local_disk(ctx, &disk, disk_id, !is_meta_bucketname(volume), Some(volume)).await?;
let physical_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
let result = disk
.undo_write_with_namespace_owner(volume, path, fi, opts, physical_owner)
@@ -874,7 +889,7 @@ mod tests {
}
drop(entries);
assert!(
matches!(admit_local_disk(&ctx, &disk, None, true).await, Err(DiskError::DiskNotFound)),
matches!(admit_local_disk(&ctx, &disk, None, true, None).await, Err(DiskError::DiskNotFound)),
"{state}"
);
assert!(!ctx.namespace_commits_pending());
+13 -13
View File
@@ -1722,20 +1722,20 @@ mod tests {
.await
.expect("bucket should be created in both pools");
for pool_index in 0..2 {
assert!(store.pools.len() >= 2);
for (pool_index, pool) in store.pools.iter().enumerate().take(2) {
let mut reader = PutObjReader::from_vec(format!("pool-{pool_index}-history").into_bytes());
store.pools[pool_index]
.put_object(
&bucket,
object,
&mut reader,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("historical version should be stored");
pool.put_object(
&bucket,
object,
&mut reader,
&ObjectOptions {
versioned: true,
..Default::default()
},
)
.await
.expect("historical version should be stored");
}
let marker = store.pools[0]
.delete_object(
@@ -0,0 +1,92 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::runtime::usage_watch::{ScannerUsageWatchKnown, ScannerUsageWatchRequest};
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
async fn known(store: &ECStore, request: &ScannerUsageWatchRequest) -> ScannerUsageWatchKnown {
tokio::time::timeout(Duration::from_secs(10), async {
loop {
let response = store.scanner_usage_watch(std::slice::from_ref(request)).await;
assert_eq!(response.len(), 1);
assert_eq!(response[0].bucket, request.bucket);
let proof = response[0].known.clone().expect("storage-backed watch proof");
if proof.inflight == 0 {
return proof;
}
// API quorum completion can precede a remaining physical tail.
tokio::time::sleep(Duration::from_millis(1)).await;
}
})
.await
.expect("all counted physical tails settle")
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn usage_watch_api_tracks_real_put_delete_and_bucket_recreation() {
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "watch-real-store";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create watched bucket");
let incarnation = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("bucket incarnation");
let mut request = ScannerUsageWatchRequest {
bucket: bucket.into(),
bucket_incarnation: incarnation,
watch_id: None,
release: false,
session_id: None,
};
let first = known(&store, &request).await;
assert!(first.is_well_formed());
assert_eq!(first.inflight, 0);
request.watch_id = Some(first.watch_id);
assert!(known(&store, &request).await.unchanged_since(&first));
let reader =
rustfs_rio::HashReader::from_stream(std::io::Cursor::new(b"payload"), 7, 7, None, None, false).expect("PUT body reader");
store
.put_object(bucket, "object", &mut PutObjReader::new(reader), &ObjectOptions::default())
.await
.expect("real PUT");
let written = known(&store, &request).await;
assert_eq!(written.watch_id, first.watch_id);
assert!(written.revision > first.revision);
assert_eq!(written.inflight, 0);
store
.delete_object(bucket, "object", ObjectOptions::default())
.await
.expect("real object delete");
let deleted = known(&store, &request).await;
assert_eq!(deleted.watch_id, first.watch_id);
assert!(deleted.revision > written.revision);
assert_eq!(deleted.inflight, 0);
store
.delete_bucket(bucket, &DeleteBucketOptions::default())
.await
.expect("delete empty bucket");
assert_eq!(store.scanner_usage_watch(std::slice::from_ref(&request)).await[0].known, None);
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("recreate watched bucket");
let recreated_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation");
assert_ne!(recreated_incarnation, incarnation);
assert_eq!(store.scanner_usage_watch(std::slice::from_ref(&request)).await[0].known, None);
request.bucket_incarnation = recreated_incarnation;
assert_eq!(store.scanner_usage_watch(std::slice::from_ref(&request)).await[0].known, None);
request.watch_id = None;
let recreated = known(&store, &request).await;
assert_ne!(recreated.watch_id, first.watch_id);
assert!(!recreated.unchanged_since(&first));
assert_eq!(recreated.inflight, 0);
}
+1 -1
View File
@@ -905,7 +905,7 @@ mod tests {
assert_indexed_matches_scalar(&original, selected);
}
if inline_mode > 0 {
let mut legacy_null = original.clone();
let mut legacy_null = original;
let payload = legacy_null
.data
.find(NULL_VERSION_ID)
+1 -5
View File
@@ -113,11 +113,7 @@ impl FileMetaVersionHeader {
}
// Check erasure coding parameters
if self.has_ec() && (self.ec_n == 0 || self.ec_m == 0 || self.ec_m < self.ec_n) {
return false;
}
true
!(self.has_ec() && (self.ec_n == 0 || self.ec_m == 0 || self.ec_m < self.ec_n))
}
}
+4 -7
View File
@@ -4223,9 +4223,10 @@ mod tests {
o_hdr.ec_m = o_m;
o_hdr.ec_n = o_n;
let actual = FileMeta::headers_match_strict(&s_hdr, &o_hdr);
let expected = {
let mut s_cmp = s_hdr.clone();
let mut o_cmp = o_hdr.clone();
let mut s_cmp = s_hdr;
let mut o_cmp = o_hdr;
if s_cmp.has_ec() != o_cmp.has_ec() {
s_cmp.ec_m = 0;
s_cmp.ec_n = 0;
@@ -4234,11 +4235,7 @@ mod tests {
}
s_cmp == o_cmp
};
assert_eq!(
FileMeta::headers_match_strict(&s_hdr, &o_hdr),
expected,
"mismatch for s=({s_m},{s_n}) vs o=({o_m},{o_n})"
);
assert_eq!(actual, expected, "mismatch for s=({s_m},{s_n}) vs o=({o_m},{o_n})");
}
}
}
+2 -2
View File
@@ -570,7 +570,7 @@ mod tests {
tonic::Status::internal("operation was canceled"),
] {
assert!(
!Error::Storage(EcstoreError::from(status).clone()).is_recoverable_heal(),
!Error::Storage(EcstoreError::from(status)).is_recoverable_heal(),
"cancellation text alone must not change application error classification"
);
}
@@ -718,7 +718,7 @@ mod tests {
for error in [
Error::Disk(DiskError::from(status.clone())),
Error::Storage(EcstoreError::from(status.clone())),
Error::Io(std::io::Error::from(DiskError::from(status.clone()))),
Error::Io(std::io::Error::from(DiskError::from(status))),
] {
assert!(!error.is_recoverable_heal(), "application RPC errors must stay terminal: {error:?}");
}
+1 -1
View File
@@ -543,7 +543,7 @@ impl HealManager {
.filter_map(|target| local_endpoints.get(target).cloned())
.collect::<Vec<_>>();
if targets.len() != state.replacement_targets.len() {
deferred_recovery_sets.insert(state.set_disk_id.clone());
deferred_recovery_sets.insert(state.set_disk_id);
continue;
}
// A pending handoff may resolve the predecessor to
+28 -12
View File
@@ -257,8 +257,6 @@ impl HealManager {
let notify_clone = notify.clone();
let root_recovery_clone = root_recovery.clone();
let manager_cancel_token = cancel_token.clone();
let task_type_label_for_spawn = task_type_label.clone();
let task_set_label_for_spawn = task_set_label.clone();
let config_for_spawn = config.clone();
// start heal task
@@ -270,8 +268,8 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_MANAGER,
task_id,
priority = ?task_priority,
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
heal_type = %task_type_label,
set = %task_set_label,
state = "task_started",
"Heal scheduler task started"
);
@@ -286,8 +284,8 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
task_id,
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
heal_type = %task_type_label,
set = %task_set_label,
state = "task_completed",
"Heal scheduler task completed"
);
@@ -300,8 +298,8 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "verification_unavailable",
task_id,
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
heal_type = %task_type_label,
set = %task_set_label,
detail = %e,
"Heal attempt completed without independent verification"
);
@@ -314,8 +312,8 @@ impl HealManager {
component = LOG_COMPONENT_HEAL,
subsystem = LOG_SUBSYSTEM_MANAGER,
task_id,
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
heal_type = %task_type_label,
set = %task_set_label,
state = "task_retrying",
retry_attempt = task.retry_attempts.saturating_add(1),
error = %e,
@@ -328,8 +326,8 @@ impl HealManager {
subsystem = LOG_SUBSYSTEM_MANAGER,
state = "task_failed",
task_id,
heal_type = %task_type_label_for_spawn,
set = %task_set_label_for_spawn,
heal_type = %task_type_label,
set = %task_set_label,
error = %e,
"Heal scheduler task failed"
});
@@ -494,11 +492,29 @@ impl HealManager {
}
publish_verified_mrf_repair_events(&notice_targets, &completed_status_for_verified_events);
publish_unverified_legacy_mrf_events(&notice_targets, &completed_status_for_verified_events, &result);
// Single-object MRF attempts retain one storage result.
// Use the original result even if the report was size
// limited, without certifying independent integrity.
let unverified_repair_bytes = if completed_status == HealTaskStatus::Completed
&& matches!(&result, Err(Error::LegacyMrfUnverified { .. }))
{
task.result_items
.read()
.await
.iter()
.find(|(_, item)| item.drives_healed().is_some_and(|healed| healed > 0))
.map(|(_, item)| u64::try_from(item.object_size).unwrap_or(u64::MAX))
} else {
None
};
// update statistics
let mut stats = statistics_clone.write().await;
match completed_status {
HealTaskStatus::Completed if matches!(&result, Err(Error::LegacyMrfUnverified { .. })) => {
stats.update_unverified_task_completion();
if let Some(bytes) = unverified_repair_bytes {
stats.add_unverified_repair(bytes);
}
}
HealTaskStatus::Completed => {
stats.update_task_completion(true);
+52 -4
View File
@@ -1106,7 +1106,10 @@ impl HealStorageAPI for MockStorage {
if bucket.starts_with("legacy-ambiguity-scheduler-") {
return Err(Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous));
}
if bucket == "legacy-held-pending-scheduler" {
if matches!(
bucket,
"legacy-held-pending-scheduler" | "legacy-held-repaired-scheduler" | "legacy-held-repaired-oversized-scheduler"
) {
let drives = vec![rustfs_madmin::heal_commands::HealDriveInfo {
state: "ok".to_owned(),
..Default::default()
@@ -1114,8 +1117,25 @@ impl HealStorageAPI for MockStorage {
return Ok((
HealResultItem {
legacy_identity_unverified: true,
detail: rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned(),
before: rustfs_madmin::heal_commands::Infos { drives: drives.clone() },
resolved_version_id: Some([0; 16]),
disk_count: 1,
object_size: 8192,
detail: if bucket == "legacy-held-repaired-oversized-scheduler" {
"x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES)
} else {
rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned()
},
before: rustfs_madmin::heal_commands::Infos {
drives: vec![rustfs_madmin::heal_commands::HealDriveInfo {
state: if bucket.starts_with("legacy-held-repaired-") {
"missing"
} else {
"ok"
}
.to_owned(),
..Default::default()
}],
},
after: rustfs_madmin::heal_commands::Infos { drives },
..Default::default()
},
@@ -2171,11 +2191,32 @@ fn only_typed_mrf_result_can_pause_unverified_legacy() {
#[tokio::test(flavor = "current_thread")]
async fn mrf_legacy_completed_attempt_retains_responsibility_without_failed_statistics() {
mrf_legacy_completed_attempt_scenario(false, false).await;
}
#[tokio::test(flavor = "current_thread")]
async fn mrf_legacy_repaired_attempt_retains_responsibility_and_physical_statistics() {
mrf_legacy_completed_attempt_scenario(true, false).await;
}
#[tokio::test(flavor = "current_thread")]
async fn mrf_legacy_repair_statistics_survive_completed_result_truncation() {
mrf_legacy_completed_attempt_scenario(true, true).await;
}
async fn mrf_legacy_completed_attempt_scenario(repaired: bool, oversized_result: bool) {
use crate::heal::failure_logging::test_support::FailureLogCapture;
let capture = FailureLogCapture::default();
let _subscriber = tracing::subscriber::set_default(capture.clone());
let (target, _) = unverified_legacy_fixture("legacy-held-pending-scheduler");
let bucket = if oversized_result {
"legacy-held-repaired-oversized-scheduler"
} else if repaired {
"legacy-held-repaired-scheduler"
} else {
"legacy-held-pending-scheduler"
};
let (target, _) = unverified_legacy_fixture(bucket);
let bucket = target.bucket.clone();
let anchor = target.durable_anchor.clone().expect("durable MRF anchor");
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
@@ -2202,8 +2243,15 @@ async fn mrf_legacy_completed_attempt_retains_responsibility_without_failed_stat
assert_eq!(stats.failed_tasks, 0);
assert_eq!(stats.successful_tasks, 0);
assert_eq!(stats.total_objects_healed, 0);
assert_eq!(stats.total_bytes_healed, 0);
assert_eq!(stats.total_unverified_objects_repaired, u64::from(repaired));
assert_eq!(stats.total_unverified_bytes_repaired, if repaired { 8192 } else { 0 });
let report = manager.get_task_report(&receipt.task_id).await.expect("retained task report");
assert_eq!(report.status, HealTaskStatus::Completed);
if oversized_result {
assert!(report.result_items_truncated);
assert!(report.result_items.is_empty());
}
let outcome = report.outcome.expect("retained unknown outcome");
assert_eq!(outcome.counters.unknown, 1);
assert_eq!(outcome.counters.failed, 0);
+48 -7
View File
@@ -50,6 +50,7 @@ use uuid::Uuid;
use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
pub mod management;
mod partial_write;
pub use partial_write::{MrfLegacyResponsibility, MrfOperatorAcceptance};
use partial_write::{PartialWrites, ResponsibilityCheckpoint};
@@ -64,6 +65,8 @@ static GLOBAL_MRF_LIFECYCLE_SENDER: std::sync::OnceLock<mpsc::Sender<MrfLifecycl
pub enum MrfLifecycleControlError {
#[error("MRF lifecycle controller is unavailable")]
Unavailable,
#[error("MRF read query budget is exhausted; retry later")]
Busy,
#[error("the MRF checkpoint is unreadable; retain its files and restore a compatible reader")]
CheckpointUnreadable,
#[error("the requested responsibility changed or does not exist")]
@@ -94,7 +97,7 @@ pub struct MrfLegacyRiskAcceptanceRequest {
pub request_id: Uuid,
}
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MrfLegacyResponsibilitySnapshot {
pub contract_version: u32,
@@ -119,14 +122,19 @@ pub async fn list_legacy_responsibilities(
"cursor must be non-nil and limit must be between 1 and 256".to_string(),
));
}
let permit = management::read_permit()?;
let sender = GLOBAL_MRF_LIFECYCLE_SENDER
.get()
.ok_or(MrfLifecycleControlError::Unavailable)?;
let (response, receiver) = tokio::sync::oneshot::channel();
sender
.send(MrfLifecycleCommand::List { after, limit, response })
.await
.map_err(|_| MrfLifecycleControlError::Unavailable)?;
.try_send(MrfLifecycleCommand::List {
after,
limit,
response,
_permit: permit,
})
.map_err(management::queue_error)?;
receiver.await.map_err(|_| MrfLifecycleControlError::Unavailable)?
}
@@ -181,7 +189,13 @@ pub async fn accept_unverified_legacy_risk(request: MrfLegacyRiskAcceptanceReque
}
enum MrfLifecycleCommand {
Get {
id: Uuid,
_permit: tokio::sync::OwnedSemaphorePermit,
response: tokio::sync::oneshot::Sender<Result<management::MrfResponsibilityObservation, MrfLifecycleControlError>>,
},
List {
_permit: tokio::sync::OwnedSemaphorePermit,
after: Option<Uuid>,
limit: usize,
response: tokio::sync::oneshot::Sender<Result<MrfLegacyResponsibilitySnapshot, MrfLifecycleControlError>>,
@@ -1325,8 +1339,31 @@ impl MrfRuntime {
async fn handle_lifecycle_command(&mut self, manager: &HealManager, command: MrfLifecycleCommand) {
match command {
MrfLifecycleCommand::List { after, limit, response } => {
let _ = response.send(self.list_unverified_legacy(after, limit));
MrfLifecycleCommand::List {
after,
limit,
response,
_permit,
} => {
if !response.is_closed() {
let _ = response.send(self.list_unverified_legacy(after, limit));
}
return;
}
MrfLifecycleCommand::Get { id, response, _permit } => {
if !response.is_closed() {
let result = if self.replay_readable {
Ok(management::MrfResponsibilityObservation {
contract_version: 1,
observed_at_ms: management::now_ms(),
responsibility: self.partial_writes.responsibility(id),
})
} else {
Err(MrfLifecycleControlError::CheckpointUnreadable)
};
let _ = response.send(result);
}
return;
}
MrfLifecycleCommand::Recheck {
responsibility_id,
@@ -1826,6 +1863,7 @@ impl MrfRuntime {
gauge!("rustfs_heal_mrf_legacy_generation_unknown").set(metric_f64(lifecycle.legacy_generation_unknown_count));
gauge!("rustfs_heal_mrf_bucket_incarnation_changed").set(metric_f64(lifecycle.bucket_incarnation_changed_count));
gauge!("rustfs_heal_mrf_operator_accepted_unverified").set(metric_f64(lifecycle.operator_accepted_count));
management::publish_summary(self, &lifecycle);
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX))
@@ -2199,7 +2237,7 @@ async fn replay_into(
// runtime dispatches it after publishing the successor, avoiding
// a duplicate task in the startup-replay and steady-state paths.
if let Ok(Some(anchor)) = manager.durable_mrf_repair_anchor(&intent).await {
durable_replay_anchors.push(anchor.clone());
durable_replay_anchors.push(anchor);
}
let restored = intent_digest(&intent).and_then(|digest| take_lifecycle_record(&mut lifecycle, digest));
partial_writes.push((intent, restored));
@@ -2499,6 +2537,9 @@ async fn finalize_mrf_consumer(
lifecycle_receiver.close();
while let Some(command) = lifecycle_receiver.recv().await {
match command {
MrfLifecycleCommand::Get { response, .. } => {
let _ = response.send(Err(MrfLifecycleControlError::Unavailable));
}
MrfLifecycleCommand::List { response, .. } => {
let _ = response.send(Err(MrfLifecycleControlError::Unavailable));
}
@@ -0,0 +1,315 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
//! Bounded runtime observations. These values never authorize lifecycle changes or prove retirement.
use super::{GLOBAL_MRF_LIFECYCLE_SENDER, MrfLegacyResponsibility, MrfLifecycleCommand, MrfLifecycleControlError, MrfRuntime};
use serde::{Deserialize, Serialize};
use std::sync::{Arc, LazyLock, RwLock};
use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot};
use uuid::Uuid;
const READ_CAPACITY: usize = 8;
static READ_BUDGET: LazyLock<Arc<Semaphore>> = LazyLock::new(|| Arc::new(Semaphore::new(READ_CAPACITY)));
static SUMMARY: RwLock<Option<MrfLifecycleSummary>> = RwLock::new(None);
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MrfLifecycleSummary {
pub contract_version: u32,
pub sampled_at_ms: u64,
pub checkpoint_readable: bool,
pub checkpoint_owner: Option<Uuid>,
pub checkpoint_sequence: Option<u64>,
pub active_responsibility_count: usize,
pub held_count: usize,
pub legacy_generation_unknown_count: usize,
pub bucket_incarnation_changed_count: usize,
pub operator_accepted_count: usize,
pub best_effort_queue_depth: usize,
pub responsibility_bytes: usize,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MrfResponsibilityObservation {
pub contract_version: u32,
pub observed_at_ms: u64,
/// Absence means not observed in this runtime, never verified or discharged.
pub responsibility: Option<MrfLegacyResponsibility>,
}
/// Returns the last published sample without scanning or queueing work.
///
/// # Errors
/// Returns unavailable until the consumer publishes a sample, or after it stops.
pub fn lifecycle_summary() -> Result<MrfLifecycleSummary, MrfLifecycleControlError> {
if GLOBAL_MRF_LIFECYCLE_SENDER.get().is_none_or(mpsc::Sender::is_closed) {
return Err(MrfLifecycleControlError::Unavailable);
}
SUMMARY
.read()
.map_err(|_| MrfLifecycleControlError::Unavailable)?
.clone()
.ok_or(MrfLifecycleControlError::Unavailable)
}
/// Observes one current responsibility, including Active entries excluded from legacy lists.
/// Cancellation skips work that has not started; observations are not durable success receipts.
///
/// # Errors
/// Rejects nil IDs, an exhausted read budget, or an unavailable/unreadable controller.
pub async fn responsibility(id: Uuid) -> Result<MrfResponsibilityObservation, MrfLifecycleControlError> {
if id.is_nil() {
return Err(MrfLifecycleControlError::InvalidAction("responsibility ID must be non-nil".into()));
}
let permit = read_permit()?;
let sender = GLOBAL_MRF_LIFECYCLE_SENDER
.get()
.ok_or(MrfLifecycleControlError::Unavailable)?;
let (response, receiver) = oneshot::channel();
sender
.try_send(MrfLifecycleCommand::Get {
id,
response,
_permit: permit,
})
.map_err(queue_error)?;
receiver.await.map_err(|_| MrfLifecycleControlError::Unavailable)?
}
pub(super) fn read_permit() -> Result<OwnedSemaphorePermit, MrfLifecycleControlError> {
READ_BUDGET
.clone()
.try_acquire_owned()
.map_err(|_| MrfLifecycleControlError::Busy)
}
pub(super) fn queue_error<T>(error: mpsc::error::TrySendError<T>) -> MrfLifecycleControlError {
match error {
mpsc::error::TrySendError::Full(_) => MrfLifecycleControlError::Busy,
mpsc::error::TrySendError::Closed(_) => MrfLifecycleControlError::Unavailable,
}
}
pub(super) fn now_ms() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |time| u64::try_from(time.as_millis()).unwrap_or(u64::MAX))
}
pub(super) fn publish_summary(runtime: &MrfRuntime, counts: &super::partial_write::PartialWriteLifecycleSummary) {
let unverified = counts.held_count
+ counts.legacy_generation_unknown_count
+ counts.bucket_incarnation_changed_count
+ counts.operator_accepted_count;
let summary = MrfLifecycleSummary {
contract_version: 1,
sampled_at_ms: now_ms(),
checkpoint_readable: runtime.replay_readable,
checkpoint_owner: runtime.runtime_checkpoint.map(|(owner, _)| owner),
checkpoint_sequence: runtime.runtime_checkpoint.map(|(_, sequence)| sequence),
active_responsibility_count: runtime.partial_writes.depth().saturating_sub(unverified),
held_count: counts.held_count,
legacy_generation_unknown_count: counts.legacy_generation_unknown_count,
bucket_incarnation_changed_count: counts.bucket_incarnation_changed_count,
operator_accepted_count: counts.operator_accepted_count,
best_effort_queue_depth: runtime.queue.depth(),
responsibility_bytes: runtime.partial_writes.bytes(),
};
if let Ok(mut published) = SUMMARY.write() {
*published = Some(summary);
}
}
#[cfg(test)]
mod tests {
use super::super::{MRF_LIFECYCLE_FORMAT_VERSION, MrfConsumerConfig, MrfQueue, PartialWrites, intent_digest};
use super::*;
use crate::heal::manager::HealManager;
use crate::heal::storage::ECStoreHealStorage;
use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope};
#[test]
fn cancelled_read_holds_budget_until_dequeued() {
let budget = Arc::new(Semaphore::new(1));
let (response, receiver) = oneshot::channel();
let command = MrfLifecycleCommand::Get {
id: Uuid::new_v4(),
response,
_permit: budget.clone().try_acquire_owned().unwrap(),
};
drop(receiver);
assert!(budget.clone().try_acquire_owned().is_err());
drop(command);
assert!(budget.try_acquire_owned().is_ok());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn management_queries_preserve_active_records_and_unreadable_checkpoint() {
let env = rustfs_test_utils::TestECStoreEnv::builder()
.prefix("mrf_management_queries")
.build()
.await;
let manager =
HealManager::new_without_root_recovery_for_test(Arc::new(ECStoreHealStorage::new(env.ecstore.clone())), None);
let mut runtime = MrfRuntime {
queue: MrfQueue::new(4, 8192),
partial_writes: PartialWrites::default(),
config: MrfConsumerConfig::default(),
checkpoint_owner: Uuid::new_v4(),
next_checkpoint_sequence: 2,
lifecycle_format_version: MRF_LIFECYCLE_FORMAT_VERSION,
new_since_flush: 0,
dirty: false,
journal_on_disk: true,
retain_replay_journal: true,
replay_readable: true,
durable_replay_anchors: Vec::new(),
replay_cleanup: None,
runtime_checkpoint: Some((Uuid::new_v4(), 1)),
backoff_until: None,
};
for (key, source) in [("active", Some(Uuid::new_v4())), ("legacy", None)] {
runtime
.admit_partial_write(
MrfIntent {
bucket: Arc::from("management"),
object: Arc::from(key),
version_id: None,
kind: MrfKind::PartialWrite,
delete_marker_purge: None,
scope: Some(MrfScope {
pool_index: 0,
set_index: 0,
}),
lease: None,
enqueued_at_ms: 1,
attempts: 0,
},
source,
)
.expect("retain responsibility fixture");
}
let active = runtime
.partial_writes
.checkpoint_records(intent_digest)
.into_iter()
.find(|row| row.source_bucket_incarnation_id.is_some())
.expect("source-bound responsibility")
.responsibility_id;
let before = runtime.snapshot();
let (sender, mut receiver) = mpsc::channel(16);
assert!(GLOBAL_MRF_LIFECYCLE_SENDER.set(sender).is_ok(), "query test requires an isolated process");
assert!(matches!(lifecycle_summary(), Err(MrfLifecycleControlError::Unavailable)));
publish_summary(&runtime, &runtime.partial_writes.lifecycle_summary());
let mut sample = lifecycle_summary().expect("published summary");
assert_eq!(sample.active_responsibility_count, 1);
assert_eq!(sample.legacy_generation_unknown_count, 1);
assert!(sample.checkpoint_readable);
sample.sampled_at_ms = 0;
*SUMMARY.write().expect("summary fixture") = Some(sample.clone());
let permits: Vec<_> = (0..READ_CAPACITY).map(|_| read_permit().expect("read slot")).collect();
assert!(matches!(responsibility(active).await, Err(MrfLifecycleControlError::Busy)));
assert!(matches!(
super::super::list_legacy_responsibilities(None, 1).await,
Err(MrfLifecycleControlError::Busy)
));
drop(permits);
let (observed, ()) = tokio::join!(responsibility(active), async {
let command = receiver.recv().await.expect("queued Get");
runtime.handle_lifecycle_command(&manager, command).await;
});
let row = observed
.expect("Get succeeds")
.responsibility
.expect("Active responsibility remains observable");
assert_eq!(row.responsibility_id, active);
assert_eq!(row.status, "active");
let (listed, ()) = tokio::join!(super::super::list_legacy_responsibilities(None, 2), async {
let command = receiver.recv().await.expect("queued List");
runtime.handle_lifecycle_command(&manager, command).await;
});
let listed = listed.expect("legacy list succeeds");
assert_eq!(listed.responsibilities.len(), 1);
assert_ne!(listed.responsibilities[0].responsibility_id, active);
assert_eq!(runtime.snapshot(), before, "observations must preserve the retained payloads");
assert_eq!(
lifecycle_summary().expect("cached summary").sampled_at_ms,
sample.sampled_at_ms,
"read commands must not republish summary metrics"
);
for list in [false, true] {
let cancelled = tokio::spawn(async move {
if list {
super::super::list_legacy_responsibilities(None, 1).await.map(|_| ())
} else {
responsibility(active).await.map(|_| ())
}
});
let command = tokio::time::timeout(std::time::Duration::from_secs(5), receiver.recv())
.await
.expect("query is enqueued promptly")
.expect("queued cancellable query");
cancelled.abort();
assert!(cancelled.await.expect_err("query task cancelled").is_cancelled());
assert_eq!(
READ_BUDGET.available_permits(),
READ_CAPACITY - 1,
"queued work retains its slot after client cancellation"
);
runtime.handle_lifecycle_command(&manager, command).await;
assert_eq!(READ_BUDGET.available_permits(), READ_CAPACITY);
assert_eq!(runtime.snapshot(), before);
}
runtime.replay_readable = false;
publish_summary(&runtime, &runtime.partial_writes.lifecycle_summary());
assert!(
!lifecycle_summary()
.expect("unreadable summary remains observable")
.checkpoint_readable
);
let (observed, ()) = tokio::join!(responsibility(active), async {
let command = receiver.recv().await.expect("Get against unreadable checkpoint");
runtime.handle_lifecycle_command(&manager, command).await;
});
assert!(matches!(observed, Err(MrfLifecycleControlError::CheckpointUnreadable)));
let (listed, ()) = tokio::join!(super::super::list_legacy_responsibilities(None, 1), async {
let command = receiver.recv().await.expect("List against unreadable checkpoint");
runtime.handle_lifecycle_command(&manager, command).await;
});
assert!(matches!(listed, Err(MrfLifecycleControlError::CheckpointUnreadable)));
assert_eq!(runtime.snapshot(), before);
receiver.close();
assert!(matches!(responsibility(active).await, Err(MrfLifecycleControlError::Unavailable)));
assert!(
matches!(lifecycle_summary(), Err(MrfLifecycleControlError::Unavailable)),
"a stale sample cannot hide shutdown"
);
assert_eq!(READ_BUDGET.available_permits(), READ_CAPACITY);
}
#[tokio::test]
async fn management_query_bounds_and_queue_errors_are_explicit() {
assert!(matches!(
responsibility(Uuid::nil()).await,
Err(MrfLifecycleControlError::InvalidAction(_))
));
let (sender, receiver) = mpsc::channel(1);
sender.try_send(()).expect("fill queue");
assert!(matches!(
queue_error(sender.try_send(()).expect_err("full queue")),
MrfLifecycleControlError::Busy
));
drop(receiver);
assert!(matches!(
queue_error(sender.try_send(()).expect_err("closed queue")),
MrfLifecycleControlError::Unavailable
));
assert_eq!(READ_BUDGET.available_permits(), READ_CAPACITY);
}
}
+131 -53
View File
@@ -776,29 +776,9 @@ impl PartialWrites {
) -> (Vec<MrfLegacyResponsibility>, Option<Uuid>) {
let mut selected = BTreeMap::new();
for entry in self.entries.values() {
let (status, bucket_incarnation_id, held_since_ms) = match &entry.state {
ResponsibilityState::Active {} if entry.generation_unobserved() => ("legacy_generation_unobserved", None, None),
ResponsibilityState::Active {} => continue,
ResponsibilityState::SourceAndCurrentGenerationUnknown { since_ms, .. } => {
("source_and_current_generation_unknown", None, Some(*since_ms))
}
ResponsibilityState::HeldUnverifiedLegacy {
bucket_incarnation_id,
since_ms,
} => ("held_unverified_legacy", Some(*bucket_incarnation_id), Some(*since_ms)),
ResponsibilityState::LegacyGenerationUnknown {
observed_bucket_incarnation_id,
detected_at_ms,
} => ("legacy_generation_unknown", Some(*observed_bucket_incarnation_id), Some(*detected_at_ms)),
ResponsibilityState::BucketIncarnationChanged {
observed_bucket_incarnation_id,
detected_at_ms,
..
} => ("bucket_incarnation_changed", Some(*observed_bucket_incarnation_id), Some(*detected_at_ms)),
ResponsibilityState::OperatorAcceptedUnverified {
bucket_incarnation_id, ..
} => ("operator_accepted_unverified", Some(*bucket_incarnation_id), None),
};
if !entry.is_parked() {
continue;
}
if after.is_some_and(|after| entry.responsibility_id <= after) {
continue;
}
@@ -809,33 +789,7 @@ impl PartialWrites {
{
continue;
}
let row = MrfLegacyResponsibility {
responsibility_id: entry.responsibility_id,
bucket: entry.intent.bucket.to_string(),
object: entry.intent.object.to_string(),
source_bucket_incarnation_id: entry.source_bucket_incarnation_id,
version_id: entry
.intent
.version_id
.and_then(|bytes| Uuid::from_slice(&bytes).ok())
.filter(|version| !version.is_nil())
.map(|version| version.to_string()),
pool_index: entry.intent.scope.map(|scope| scope.pool_index),
set_index: entry.intent.scope.map(|scope| scope.set_index),
bucket_incarnation_id,
enqueued_at_ms: entry.intent.enqueued_at_ms,
status,
held_since_ms,
accepted: entry.last_operator_acceptance.clone(),
last_identity_probe_result: entry.last_identity_probe_result,
last_identity_probe_at_ms: entry.last_identity_probe_at_ms,
next_identity_probe_at_ms: entry.generation_unobserved().then(|| {
unix_now_ms().saturating_add(
u64::try_from(entry.next_attempt.saturating_duration_since(Instant::now()).as_millis())
.unwrap_or(u64::MAX),
)
}),
};
let row = Self::responsibility_row(entry);
selected.insert(entry.responsibility_id, row);
if selected.len() > limit + 1 {
selected.pop_last();
@@ -852,6 +806,65 @@ impl PartialWrites {
(selected.into_values().collect(), next_cursor)
}
pub(super) fn responsibility(&self, id: Uuid) -> Option<MrfLegacyResponsibility> {
self.entries
.values()
.find(|entry| entry.responsibility_id == id)
.map(Self::responsibility_row)
}
fn responsibility_row(entry: &Responsibility) -> MrfLegacyResponsibility {
let (status, bucket_incarnation_id, held_since_ms) = match &entry.state {
ResponsibilityState::Active {} if entry.generation_unobserved() => ("legacy_generation_unobserved", None, None),
ResponsibilityState::Active {} => ("active", None, None),
ResponsibilityState::SourceAndCurrentGenerationUnknown { since_ms, .. } => {
("source_and_current_generation_unknown", None, Some(*since_ms))
}
ResponsibilityState::HeldUnverifiedLegacy {
bucket_incarnation_id,
since_ms,
} => ("held_unverified_legacy", Some(*bucket_incarnation_id), Some(*since_ms)),
ResponsibilityState::LegacyGenerationUnknown {
observed_bucket_incarnation_id,
detected_at_ms,
} => ("legacy_generation_unknown", Some(*observed_bucket_incarnation_id), Some(*detected_at_ms)),
ResponsibilityState::BucketIncarnationChanged {
observed_bucket_incarnation_id,
detected_at_ms,
..
} => ("bucket_incarnation_changed", Some(*observed_bucket_incarnation_id), Some(*detected_at_ms)),
ResponsibilityState::OperatorAcceptedUnverified {
bucket_incarnation_id, ..
} => ("operator_accepted_unverified", Some(*bucket_incarnation_id), None),
};
MrfLegacyResponsibility {
responsibility_id: entry.responsibility_id,
bucket: entry.intent.bucket.to_string(),
object: entry.intent.object.to_string(),
source_bucket_incarnation_id: entry.source_bucket_incarnation_id,
version_id: entry
.intent
.version_id
.and_then(|bytes| Uuid::from_slice(&bytes).ok())
.filter(|version| !version.is_nil())
.map(|version| version.to_string()),
pool_index: entry.intent.scope.map(|scope| scope.pool_index),
set_index: entry.intent.scope.map(|scope| scope.set_index),
bucket_incarnation_id,
enqueued_at_ms: entry.intent.enqueued_at_ms,
status: status.to_string(),
held_since_ms,
accepted: entry.last_operator_acceptance.clone(),
last_identity_probe_result: entry.last_identity_probe_result.map(str::to_string),
last_identity_probe_at_ms: entry.last_identity_probe_at_ms,
next_identity_probe_at_ms: entry.generation_unobserved().then(|| {
unix_now_ms().saturating_add(
u64::try_from(entry.next_attempt.saturating_duration_since(Instant::now()).as_millis()).unwrap_or(u64::MAX),
)
}),
}
}
pub(super) fn replace_observed_incarnation(
&mut self,
responsibility_id: Uuid,
@@ -1154,7 +1167,7 @@ fn validate_operator_audit_fields(actor: &str, reason: &str, reference: &str, re
Ok(())
}
#[derive(Clone, Debug, Serialize)]
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct MrfLegacyResponsibility {
pub responsibility_id: Uuid,
@@ -1166,11 +1179,11 @@ pub struct MrfLegacyResponsibility {
pub set_index: Option<u32>,
pub bucket_incarnation_id: Option<Uuid>,
pub enqueued_at_ms: u64,
pub status: &'static str,
pub status: String,
pub held_since_ms: Option<u64>,
pub accepted: Option<MrfOperatorAcceptance>,
#[serde(skip_serializing_if = "Option::is_none")]
pub last_identity_probe_result: Option<&'static str>,
pub last_identity_probe_result: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub last_identity_probe_at_ms: Option<u64>,
/// Process-local schedule; replay restarts the conservative probe delay.
@@ -1230,6 +1243,71 @@ mod tests {
intent
}
#[test]
fn management_query_scale_baseline_and_active_lookup() {
for budget in [8 * 1024 * 1024, 128 * 1024 * 1024] {
let mut writes = PartialWrites::default();
let source = Uuid::new_v4();
let started = std::time::Instant::now();
for index in 0..100_000 {
let item = MrfIntent {
bucket: Arc::from("management-scale"),
object: Arc::from(format!("key-{index:06}")),
version_id: None,
kind: MrfKind::PartialWrite,
scope: Some(MrfScope {
pool_index: 0,
set_index: 0,
}),
lease: None,
enqueued_at_ms: 1,
attempts: 0,
delete_marker_purge: None,
};
if writes
.admit_with_source_incarnation(item, if index % 10 == 0 { Some(source) } else { None }, 100_000, budget)
.is_err()
{
break;
}
}
let admission_ms = started.elapsed().as_millis();
assert!(writes.depth() > 1000);
let active = writes
.entries
.values()
.find(|entry| !entry.is_parked())
.unwrap()
.responsibility_id;
assert_eq!(writes.responsibility(active).unwrap().status, "active");
assert!(writes.responsibility(Uuid::new_v4()).is_none());
let mut elapsed = Vec::new();
let mut after = None;
for _ in 0..32 {
let start = std::time::Instant::now();
let (rows, next) = writes.list_unverified_legacy(after, 256);
let _ = writes.lifecycle_summary();
assert!(writes.responsibility(active).is_some());
elapsed.push(start.elapsed().as_micros());
assert!(rows.len() <= 256);
assert!(
rows.iter()
.all(|row| row.responsibility_id != active && after.is_none_or(|id| row.responsibility_id > id))
);
after = next;
}
elapsed.sort_unstable();
eprintln!(
"MRF query baseline: entries={} bytes={} admission_ms={} page_summary_id_p50_us={} p99_us={}",
writes.depth(),
writes.bytes(),
admission_ms,
elapsed[16],
elapsed[31]
);
}
}
fn risk_acceptance(
responsibility_id: Uuid,
expected_bucket_incarnation_id: Uuid,
@@ -179,7 +179,7 @@ async fn unknown_source_deleted_bucket_probes_preserve_errors_and_bound_retries(
assert_eq!(row.bucket_incarnation_id, None);
assert_eq!(row.source_bucket_incarnation_id, None);
assert_eq!(
row.last_identity_probe_result,
row.last_identity_probe_result.as_deref(),
Some(if matches!(outcome, Lookup::Identity(None)) {
"unresolved"
} else {
@@ -395,6 +395,77 @@ fn empty_events_keep_dirty_flush_failure_retry_and_cleanup_gates() {
});
}
#[test]
fn legacy_park_replay_requires_committed_checkpoint() {
if isolated("legacy_park_replay_requires_committed_checkpoint") {
return;
}
tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("runtime")
.block_on(async {
let (env, manager, mut runtime, anchor) = bound_runtime().await;
let original = runtime.partial_writes.checkpoint_records(intent_digest);
assert!(runtime.flush().await, "the active responsibility must be durable first");
// Losing the completed-task notice leaves the original active
// owner replayable. No physical result alone can discharge it.
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
let replay = replay_into(&manager, &mut queue, &mut None).await;
assert_eq!(replay.partial_writes.len(), original.len());
assert!(replay.partial_writes.iter().all(|(_, record)| {
!matches!(
record.as_ref().expect("active companion").state,
partial_write::ResponsibilityState::HeldUnverifiedLegacy { .. }
)
}));
note_mrf_unverified_legacy(MrfUnverifiedLegacyEvent { anchor: anchor.clone() });
runtime.discharge_durable_replay_anchors();
assert_eq!(runtime.partial_writes.lifecycle_summary().held_count, 1);
let paths = env
.disk_paths
.iter()
.map(|disk| (disk.join(".rustfs.sys"), disk.join(".mrf-park-sys-backup")))
.collect::<Vec<_>>();
for (path, backup) in &paths {
fs::rename(path, backup).expect("preserve committed active checkpoint");
fs::write(path, b"block held checkpoint publication").expect("inject publication failure");
}
let persisted = runtime.flush().await;
for (path, backup) in &paths {
fs::remove_file(path).expect("remove publication blocker");
fs::rename(backup, path).expect("restore committed active checkpoint");
}
assert!(!persisted);
assert!(runtime.dirty);
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
let replay = replay_into(&manager, &mut queue, &mut None).await;
let mut restored = replay
.partial_writes
.into_iter()
.map(|(_, record)| record.expect("previous active companion"))
.collect::<Vec<_>>();
let mut original = original;
restored.sort_by_key(|record| record.responsibility_id);
original.sort_by_key(|record| record.responsibility_id);
assert_eq!(restored, original, "a crash before commit may conservatively repeat Deep");
assert!(runtime.flush().await, "commit the held state after storage recovers");
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
let replay = replay_into(&manager, &mut queue, &mut None).await;
assert_eq!(replay.partial_writes.len(), original.len());
assert!(replay.partial_writes.iter().any(|(_, record)| {
matches!(
record.as_ref().expect("committed held companion").state,
partial_write::ResponsibilityState::HeldUnverifiedLegacy { .. }
)
}));
assert!(runtime.retained_replay_journal(), "parking never releases responsibility");
});
}
#[test]
fn empty_events_real_consumer_stop_retains_unresolved_journal() {
if isolated("empty_events_real_consumer_stop_retains_unresolved_journal") {
@@ -421,6 +492,7 @@ fn empty_events_real_consumer_stop_retains_unresolved_journal() {
after: None,
limit: 10,
response,
_permit: crate::heal::mrf_queue::management::read_permit().expect("MRF management read permit"),
})
.await
.expect("query the actual consumer");
@@ -230,6 +230,7 @@ async fn assert_unreadable_lifecycle_is_retained(corrupt: fn(&mut Value)) {
after: None,
limit: 10,
response,
_permit: crate::heal::mrf_queue::management::read_permit().expect("MRF management read permit"),
})
.await
.expect("query the actual consumer");
@@ -234,6 +234,7 @@ async fn check_reader_restart(version: u8) {
after: None,
limit: 3,
response,
_permit: crate::heal::mrf_queue::management::read_permit().expect("MRF management read permit"),
})
.await
.expect("query the real consumer after its startup replay");
@@ -72,6 +72,7 @@ async fn owned_mrf_shutdown_drains_and_persists_before_ack_and_join() {
after: None,
limit: 1,
response,
_permit: crate::heal::mrf_queue::management::read_permit().expect("MRF management read permit"),
})
.await
.expect("queued lifecycle query");
+25 -1
View File
@@ -509,6 +509,12 @@ pub struct HealStatistics {
/// Completed attempts without independent verification; not the current durable MRF backlog.
#[serde(default)]
pub unverified_tasks: u64,
/// Physical repair attempts for unverified legacy objects, not verified outcomes or unique objects.
#[serde(default)]
pub total_unverified_objects_repaired: u64,
/// Logical object bytes in those repair attempts, not physical disk or network bytes.
#[serde(default)]
pub total_unverified_bytes_repaired: u64,
/// Running tasks
pub running_tasks: u64,
/// Total healed objects
@@ -532,6 +538,8 @@ impl HealStatistics {
successful_tasks: 0,
failed_tasks: 0,
unverified_tasks: 0,
total_unverified_objects_repaired: 0,
total_unverified_bytes_repaired: 0,
running_tasks: 0,
total_objects_healed: 0,
total_bytes_healed: 0,
@@ -558,6 +566,11 @@ impl HealStatistics {
self.last_update_time = SystemTime::now();
}
pub(crate) fn add_unverified_repair(&mut self, bytes: u64) {
self.total_unverified_objects_repaired = self.total_unverified_objects_repaired.saturating_add(1);
self.total_unverified_bytes_repaired = self.total_unverified_bytes_repaired.saturating_add(bytes);
}
pub fn add_healed_objects(&mut self, count: u64, bytes: u64) {
self.total_objects_healed += count;
self.total_bytes_healed += bytes;
@@ -928,15 +941,26 @@ mod tests {
fn unverified_statistics_preserve_old_snapshot_compatibility() {
let mut stats = HealStatistics::new();
stats.update_unverified_task_completion();
stats.add_unverified_repair(8192);
assert_eq!(stats.unverified_tasks, 1);
assert_eq!(stats.successful_tasks, 0);
assert_eq!(stats.failed_tasks, 0);
let mut wire = serde_json::to_value(&stats).expect("serialize task statistics");
let current: HealStatistics = serde_json::from_value(wire.clone()).expect("read new statistics");
assert_eq!(current.unverified_tasks, 1);
wire.as_object_mut().expect("statistics object").remove("unverified_tasks");
assert_eq!(current.total_unverified_objects_repaired, 1);
assert_eq!(current.total_unverified_bytes_repaired, 8192);
for field in [
"unverified_tasks",
"total_unverified_objects_repaired",
"total_unverified_bytes_repaired",
] {
wire.as_object_mut().expect("statistics object").remove(field);
}
let old: HealStatistics = serde_json::from_value(wire).expect("read statistics without new field");
assert_eq!(old.unverified_tasks, 0);
assert_eq!(old.total_unverified_objects_repaired, 0);
assert_eq!(old.total_unverified_bytes_repaired, 0);
}
#[test]
@@ -348,10 +348,7 @@ mod tests {
let enabled = std::env::var(ENABLE_ENV)
.ok()
.is_some_and(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES"));
if !enabled {
return Ok(false);
}
Ok(true)
Ok(enabled)
}
fn run_current_test_in_mount_namespace() -> Result<(), Box<dyn Error + Send + Sync>> {
+24 -5
View File
@@ -514,17 +514,36 @@ impl HealTask {
let receipt_missing = storage_result.receipt.is_none();
if !self.record_verified_storage_receipt(&expected, storage_result.receipt).await {
let ok_drive_state = DriveState::Ok.to_string();
let healthy_legacy_without_repair = receipt_missing
let selected_version_matches = storage_result.item.resolved_version_id.is_some_and(|resolved| {
let requested = version_id.filter(|version| !version.is_empty());
match requested {
Some(requested) => Uuid::parse_str(requested).is_ok_and(|version| version.as_bytes() == &resolved),
None => resolved == [0; 16],
}
});
// A completed legacy repair can restore redundancy without proving
// original content. Keep the same durable responsibility as an
// already-healthy Deep result, including when this attempt repaired
// a shard. Storage errors and invalid receipts never qualify.
let healthy_legacy = receipt_missing
&& storage_result.item.legacy_identity_unverified
&& storage_result.item.drives_healed() == Some(0)
&& !storage_result.item.after.drives.is_empty()
&& selected_version_matches
&& storage_result.item.drives_healed().is_some()
&& storage_result.item.after.drives.len() == storage_result.item.disk_count
&& storage_result
.item
.before
.drives
.iter()
.zip(&storage_result.item.after.drives)
.all(|(before, after)| before.endpoint == after.endpoint)
&& storage_result
.item
.after
.drives
.iter()
.all(|drive| drive.state == ok_drive_state);
if healthy_legacy_without_repair {
if healthy_legacy {
self.outcome.write().await.record(HealObjectOutcome {
deep_check_observed: false,
quorum_selected_data_dir: None,
@@ -534,7 +553,7 @@ impl HealTask {
});
}
self.record_result_item(storage_result.item).await;
return Err(if healthy_legacy_without_repair {
return Err(if healthy_legacy {
Error::LegacyMrfUnverified {
target: format!("{bucket}/{object}"),
}
+54 -3
View File
@@ -3227,6 +3227,8 @@ fn healthy_legacy_heal_item() -> HealResultItem {
}];
HealResultItem {
legacy_identity_unverified: true,
resolved_version_id: Some([0; 16]),
disk_count: 1,
detail: rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned(),
before: Infos { drives: drives.clone() },
after: Infos { drives },
@@ -3234,6 +3236,34 @@ fn healthy_legacy_heal_item() -> HealResultItem {
}
}
#[tokio::test]
async fn mrf_legacy_repair_completes_unverified_without_second_deep() {
let mut item = healthy_legacy_heal_item();
item.before.drives[0].state = "missing".to_owned();
item.object_size = 8192;
let (task, storage) = mrf_task_for_item("legacy-repaired", item);
let error = task
.execute()
.await
.expect_err("physical repair cannot certify legacy identity");
assert!(matches!(error, Error::LegacyMrfUnverified { .. }), "{error:?}");
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
let outcome = task.get_outcome().await;
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
assert_eq!(outcome.counters.unknown, 1);
assert_eq!(outcome.counters.healed, 0);
assert_eq!(outcome.counters.failed, 0);
assert!(matches!(outcome.objects[0].disposition, HealObjectDisposition::Unknown));
let items = task.get_seqed_result_items().await;
assert_eq!(items.len(), 1);
assert_eq!(items[0].1.drives_healed(), Some(1));
assert_eq!(items[0].1.object_size, 8192);
let opts = storage.object_heal_opts.lock().expect("physical repair options");
assert_eq!(opts.len(), 1, "the first completed repair must supply the terminal classification");
assert_eq!(opts[0].scan_mode, HealScanMode::Deep);
}
fn mrf_task_for_item(object: &str, item: HealResultItem) -> (HealTask, Arc<MockStorage>) {
let incarnation = Uuid::new_v4();
let storage = Arc::new(MockStorage {
@@ -3343,11 +3373,18 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
"empty",
"unhealthy",
"unaligned",
"wrong_endpoint",
"wrong_disk_count",
"wrong_version",
"unknown_version",
"versioned",
"versioned_mismatch",
"receipt",
"storage_error",
"dry_run",
] {
let mut item = healthy_legacy_heal_item();
let requested_version = Uuid::new_v4();
match case {
"qualified" => item.detail = "updated diagnostic wording".to_owned(),
"text_only" => item.legacy_identity_unverified = false,
@@ -3361,9 +3398,20 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
item.after.drives[0].state = "missing".to_owned();
}
"unaligned" => item.before.drives.clear(),
"wrong_endpoint" => item.before.drives[0].endpoint = "another-disk".to_owned(),
"wrong_disk_count" => item.disk_count = 4,
"wrong_version" => item.resolved_version_id = Some(*Uuid::new_v4().as_bytes()),
"unknown_version" => item.resolved_version_id = None,
"versioned" => item.resolved_version_id = Some(*requested_version.as_bytes()),
"versioned_mismatch" => item.resolved_version_id = Some(*Uuid::new_v4().as_bytes()),
_ => {}
}
let (mut task, storage) = mrf_task_for_item("object", item);
if matches!(case, "versioned" | "versioned_mismatch")
&& let HealType::Object { version_id, .. } = &mut task.heal_type
{
*version_id = Some(requested_version.to_string());
}
if case == "receipt" {
storage.heal_object_receipts.lock().expect("receipt fixture").insert(
"object".to_owned(),
@@ -3383,13 +3431,16 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
.execute()
.await
.expect_err("legacy result cannot discharge a durable responsibility");
assert_eq!(matches!(error, Error::LegacyMrfUnverified { .. }), case == "qualified", "case={case}");
let qualifies = matches!(case, "qualified" | "repaired" | "versioned");
assert_eq!(matches!(error, Error::LegacyMrfUnverified { .. }), qualifies, "case={case}");
let outcome = task.get_outcome().await;
assert_eq!(outcome.counters.healed, 0);
if case == "qualified" {
if qualifies {
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
assert_eq!(outcome.counters.unknown, 1);
assert_eq!(outcome.objects[0].detail.as_deref(), Some("updated diagnostic wording"));
if case == "qualified" {
assert_eq!(outcome.objects[0].detail.as_deref(), Some("updated diagnostic wording"));
}
let opts = storage.object_heal_opts.lock().expect("storage options");
assert_eq!(opts.len(), 1);
assert_eq!(opts[0].scan_mode, HealScanMode::Deep);
@@ -1004,6 +1004,15 @@ async fn partial_write_sigkill_replay_scenario(protected: bool) {
.await,
"legacy replay attempts must finish"
);
let stats = manager.get_statistics().await;
assert_eq!(stats.failed_tasks, 0, "the first physical legacy repair must not be a failed attempt");
assert_eq!(
stats.unverified_tasks, 1,
"no second Deep is needed to classify the repaired legacy object"
);
assert_eq!(stats.total_unverified_objects_repaired, 1);
assert_eq!(stats.total_unverified_bytes_repaired, 36);
assert_eq!(stats.total_objects_healed, 0, "physical repair does not prove legacy identity");
// Cover two MRF admission backoff periods so a missed hold notice
// cannot pass merely because its task finished between polls.
let retry_window = tokio::time::Instant::now() + Duration::from_secs(12);
+2 -2
View File
@@ -379,8 +379,8 @@ mod tests {
let mut stored = generated.stored;
assert_eq!(remaining(&stored), RECOVERY_CODE_COUNT as u32);
for index in 0..3 {
consume(&mut stored, &generated.plaintext[index], now());
for code in generated.plaintext.iter().take(3) {
consume(&mut stored, code, now());
}
assert_eq!(remaining(&stored), (RECOVERY_CODE_COUNT - 3) as u32);
}
+1 -1
View File
@@ -181,7 +181,7 @@ where
);
let body: BoxBody = Full::new(Bytes::from(error_xml))
.map_err(|e| -> BoxError { Box::new(e) })
.map_err(|never| match never {})
.boxed_unsync();
let response = Response::builder()
+1 -1
View File
@@ -1347,7 +1347,7 @@ mod tests {
let summary = KmsConfigSummary::from(&config);
let response = KmsStatusResponse {
status: KmsServiceStatus::Configured,
backend_type: Some(config.backend.clone()),
backend_type: Some(config.backend),
healthy: None,
config_summary: Some(summary),
};
+1 -1
View File
@@ -977,7 +977,7 @@ async fn enumerate_target_conflicts(target: &Path, manifest: &BackupManifest) ->
(stem.to_string(), format!("target already has key file {entry:?}"))
} else if entry == LOCAL_KMS_MASTER_KEY_SALT_FILE {
(
entry.clone(),
entry,
"target holds an orphan master-key KDF salt; it would poison every key created after the restore".to_string(),
)
} else {
+1 -1
View File
@@ -692,7 +692,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
let predicted = Event {
action: IlmAction::DeleteAction,
rule_id: rule.id.clone().unwrap_or_default(),
rule_id: rule.id.unwrap_or_default(),
due: Some(due),
noncurrent_days: 0,
newer_noncurrent_versions: 0,
+1 -1
View File
@@ -487,7 +487,7 @@ mod fast_lock_tests {
// Waiter: starts a touch later so the holder wins the lock first, then
// must survive the whole hold and acquire promptly after the release.
let waiter_mgr = manager.clone();
let waiter_key = key.clone();
let waiter_key = key;
handles.push(tokio::spawn(async move {
sleep(Duration::from_millis(10)).await;
let request = ObjectLockRequest::new_write(waiter_key, "waiter").with_acquire_timeout(Duration::from_secs(5));
+1 -1
View File
@@ -1537,7 +1537,7 @@ impl RealtimeMetrics {
self.hosts.sort();
for (k, v) in other.by_disk.into_iter() {
self.by_disk.entry(k.to_string()).and_modify(|h| *h = v.clone()).or_insert(v);
self.by_disk.entry(k).and_modify(|h| *h = v.clone()).or_insert(v);
}
}
}
@@ -120,8 +120,8 @@ pub fn collect_erasure_set_metrics(stats: &[ErasureSetStats]) -> Vec<PrometheusM
);
metrics.push(
PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_HEALTH_MD, stat.write_health as f64)
.with_label_owned(POOL_ID_L, pool_id_label.clone())
.with_label_owned(SET_ID_L, set_id_label.clone()),
.with_label_owned(POOL_ID_L, pool_id_label)
.with_label_owned(SET_ID_L, set_id_label),
);
}
+1 -1
View File
@@ -102,7 +102,7 @@ fn check() -> Result<(), OpaConfigError> {
let prefix = format!("{ENV_PREFIX}{POLICY_PLUGIN_SUB_SYS}").to_uppercase();
for (key, value) in env_list {
if key.starts_with(&prefix) {
candidate.insert(key.to_string(), value);
candidate.insert(key, value);
}
}

Some files were not shown because too many files have changed in this diff Show More