mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-16 21:14:45 +00:00
Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| de9009d882 | |||
| 94d54f8f55 | |||
| 01edeb0eab | |||
| dd2713687c | |||
| 4304e1b3ed | |||
| 5b331fad73 | |||
| 130a19de8f | |||
| 3e882ea0cf | |||
| 517ea37eb2 | |||
| 6d0832b992 | |||
| 34e6449ed9 |
@@ -1,77 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"plan_id": "rustfs-release-validation",
|
||||
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"targets": [
|
||||
{"os": "linux", "arch": "x86_64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
|
||||
{"os": "macos", "arch": "aarch64", "abi": "none"},
|
||||
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
|
||||
],
|
||||
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
|
||||
"features": ["pyroscope"],
|
||||
"trusted_workflows": [
|
||||
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
|
||||
],
|
||||
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
|
||||
"lanes": [
|
||||
{
|
||||
"lane": "identity",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
|
||||
"timeout_seconds": 1800,
|
||||
"environment_capabilities": ["process-exec"],
|
||||
"cases": ["artifact-identity", "startup"],
|
||||
"applies_to": {
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"platforms": "all"
|
||||
},
|
||||
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
|
||||
},
|
||||
{
|
||||
"lane": "upgrade",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
|
||||
"timeout_seconds": 5400,
|
||||
"environment_capabilities": ["process-exec", "previous-release-artifact"],
|
||||
"cases": [
|
||||
"direct-upgrade",
|
||||
"mixed-version-upgrade",
|
||||
"bucket-config-upgrade",
|
||||
"bucket-config-rollback",
|
||||
"multipart-layout-upgrade",
|
||||
"multipart-layout-baseline"
|
||||
],
|
||||
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
|
||||
"applies_to": {
|
||||
"release_types": ["prerelease", "stable"],
|
||||
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
|
||||
},
|
||||
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
|
||||
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
|
||||
},
|
||||
{
|
||||
"version": "1.0.0-rc.5",
|
||||
"role": "N-2",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
|
||||
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
+6
-312
@@ -193,7 +193,7 @@ jobs:
|
||||
{"target_id":"linux-aarch64-musl","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-musl","cross":false,"platform":"linux","rustflags":"","features":"","glibc":""},
|
||||
{"target_id":"linux-x86_64-gnu","os":"sm-standard-4","target":"x86_64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"linux-aarch64-gnu","os":"ubicloud-standard-16-arm","target":"aarch64-unknown-linux-gnu","cross":true,"platform":"linux","rustflags":"","features":"pyroscope","glibc":"2.31"},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"macos-aarch64","os":"macos-latest-xlarge","target":"aarch64-apple-darwin","cross":false,"platform":"macos","rustflags":"","features":"pyroscope","glibc":""},
|
||||
{"target_id":"windows-x86_64","os":"windows-latest","target":"x86_64-pc-windows-msvc","cross":false,"platform":"windows","rustflags":"","features":"","glibc":""}
|
||||
]}'
|
||||
|
||||
@@ -814,18 +814,13 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Determine upload path based on build type. Release and prerelease
|
||||
# builds only stage their bytes here: the official
|
||||
# artifacts/rustfs/release/ path is written solely by the
|
||||
# certification-gated promote-release-artifacts job, from these same
|
||||
# staged objects. Development builds keep the uncertified dev channel
|
||||
# and never touch official or latest channels.
|
||||
# Determine upload path based on build type
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
|
||||
echo "📤 Uploading development build to R2 dev directory"
|
||||
else
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
echo "📤 Staging release build for certification-gated promotion"
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
echo "📤 Uploading release build to R2 release directory"
|
||||
fi
|
||||
|
||||
# Upload all rustfs zip files to R2 using S3-compatible API
|
||||
@@ -874,221 +869,6 @@ jobs:
|
||||
echo "This run does not trigger Docker publishing."
|
||||
fi
|
||||
|
||||
# Aggregate the parent candidate batch. This job always collects results and
|
||||
# retains its evidence, including on failure, and every publication job
|
||||
# below requires its success. Missing, stale, substituted or non-PASS
|
||||
# evidence fails closed here: only the trusted verification sources
|
||||
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
|
||||
# it, so a PR-uploaded JSON set never gains release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download all build artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Write candidate and collect trusted validation results
|
||||
env:
|
||||
# Cross-repo read of the trusted verification run. Falls back to the
|
||||
# workflow token, which cannot read rustfs/s3chaos; a missing token
|
||||
# therefore fail-closes instead of certifying.
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
PRODUCT_COMMIT: ${{ github.sha }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
|
||||
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p results release-gate-output
|
||||
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
|
||||
|
||||
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
|
||||
import hashlib, json, os, sys, zipfile
|
||||
|
||||
plan_sha256 = sys.argv[1]
|
||||
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
|
||||
if profile is None:
|
||||
raise SystemExit("release certification only covers preview, prerelease and stable builds")
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
deliverables = []
|
||||
for name in sorted(os.listdir("artifacts")):
|
||||
if not name.startswith("rustfs-") or not name.endswith(".zip"):
|
||||
continue
|
||||
path = os.path.join("artifacts", name)
|
||||
members = zipfile.ZipFile(path).namelist()
|
||||
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
binary_bytes = archive.read(binary_name)
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile() as handle:
|
||||
handle.write(binary_bytes)
|
||||
handle.flush()
|
||||
binary_sha = sha256(handle.name)
|
||||
deliverables.append({
|
||||
"kind": "zip",
|
||||
"name": name,
|
||||
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
|
||||
"sha256": sha256(path),
|
||||
"size": os.path.getsize(path),
|
||||
"binary_sha256": binary_sha,
|
||||
})
|
||||
if not deliverables:
|
||||
raise SystemExit("no built deliverables found to certify")
|
||||
candidate = {
|
||||
"schema_version": 1,
|
||||
"product_commit": os.environ["PRODUCT_COMMIT"],
|
||||
"version": os.environ["VERSION"],
|
||||
"release_profile": profile,
|
||||
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
|
||||
"plan_sha256": plan_sha256,
|
||||
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
|
||||
"deliverables": deliverables,
|
||||
}
|
||||
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
|
||||
print()
|
||||
PY
|
||||
|
||||
# Trusted verification evidence only: the release-gate envelopes of
|
||||
# rustfs/s3chaos for this exact product commit. Absence blocks below.
|
||||
TRUSTED_REPO="rustfs/s3chaos"
|
||||
TRUSTED_WORKFLOW="release-gate.yml"
|
||||
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
|
||||
if [[ -z "$TRUSTED_RUN" ]]; then
|
||||
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
|
||||
else
|
||||
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
|
||||
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
|
||||
fi
|
||||
|
||||
python3 scripts/release/release_gate.py validate \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate candidate.json \
|
||||
--results results \
|
||||
--output release-gate-output
|
||||
|
||||
- name: Upload certification evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: |
|
||||
candidate.json
|
||||
results/
|
||||
release-gate-output/
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
# The only writer of the official R2 release path. Promotion re-checks the
|
||||
# exact bytes about to be published against the passed certification list
|
||||
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
|
||||
promote-release-artifacts:
|
||||
name: Promote Release Artifacts
|
||||
needs: [ build-check, build-rustfs, release-certification ]
|
||||
# Target-channel lock for the official R2 release path.
|
||||
concurrency:
|
||||
group: release-channel-r2-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download build and certification artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged release artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$name"
|
||||
done
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged object ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
|
||||
# Create GitHub Release for every valid release tag, including previews
|
||||
create-release:
|
||||
name: Create GitHub Release
|
||||
@@ -1248,10 +1028,6 @@ jobs:
|
||||
update-latest-version:
|
||||
name: Update Latest Version
|
||||
needs: [ build-check, publish-release ]
|
||||
# Target-channel lock for the latest.json version pointer.
|
||||
concurrency:
|
||||
group: release-channel-oss-latest
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
@@ -1292,45 +1068,6 @@ jobs:
|
||||
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
|
||||
chmod +x "$OSSUTIL_BIN"
|
||||
|
||||
# Candidate/pointer ordering: an older candidate must never overwrite
|
||||
# a newer pointer, even if its promotion job finishes later.
|
||||
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
|
||||
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
|
||||
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
def key(version):
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
|
||||
if not match:
|
||||
raise SystemExit(3)
|
||||
core = tuple(int(part) for part in match.groups()[:3])
|
||||
prerelease = match.group(4)
|
||||
if prerelease is None:
|
||||
return core + (1,)
|
||||
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
|
||||
|
||||
try:
|
||||
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
|
||||
proposed = sys.argv[2]
|
||||
if current and key(current) > key(proposed):
|
||||
raise SystemExit(1)
|
||||
except SystemExit:
|
||||
raise
|
||||
except (OSError, ValueError, TypeError):
|
||||
raise SystemExit(3)
|
||||
PY
|
||||
guard_status=$?
|
||||
if [[ "$guard_status" -eq 1 ]]; then
|
||||
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
|
||||
exit 0
|
||||
elif [[ "$guard_status" -ne 0 ]]; then
|
||||
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create latest.json
|
||||
cat > latest.json << EOF
|
||||
{
|
||||
@@ -1347,59 +1084,16 @@ jobs:
|
||||
|
||||
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
|
||||
|
||||
# Publish release (remove draft status). Requires the release-certification
|
||||
# aggregate: without a complete legal success there is no public write.
|
||||
# Publish release (remove draft status)
|
||||
publish-release:
|
||||
name: Publish Release
|
||||
needs: [ build-check, create-release, upload-release-assets, release-certification ]
|
||||
# Target-channel lock: concurrent releases serialize on this channel so an
|
||||
# older candidate cannot interleave its publish with a newer one.
|
||||
concurrency:
|
||||
group: release-channel-github-release
|
||||
cancel-in-progress: false
|
||||
needs: [ build-check, create-release, upload-release-assets ]
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify release assets against certification
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${{ needs.build-check.outputs.version }}"
|
||||
mkdir -p "$RUNNER_TEMP/promotion-assets"
|
||||
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
|
||||
found=0
|
||||
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
found=1
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$(basename "$zip_file")"
|
||||
done
|
||||
if [[ "$found" -ne 1 ]]; then
|
||||
echo "No release zip assets to verify against the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Publish release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -329,75 +329,13 @@ jobs:
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Consume the trusted release certification for the exact version this
|
||||
# workflow wraps and verify the certified input digests. A manual dispatch
|
||||
# without that certification fails here and pushes no official tag; only
|
||||
# results from the trusted verification sources carry release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$run_id" ]]; then
|
||||
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$run_id" -n release-certification -D certification; then
|
||||
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Input digest binding: every zip this image wraps must be the
|
||||
# certified byte-for-byte content, never a mutable latest pointer.
|
||||
while IFS=$'\t' read -r name location expected; do
|
||||
echo "Verifying certified input ${name}"
|
||||
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "Certified input ${name} does not match its recorded digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
# Supports both release and dev channel binaries based on build context
|
||||
# Only runs when should_build is true (which includes workflow success check)
|
||||
build-docker:
|
||||
name: Build Docker Images
|
||||
needs: [ build-check, release-certification ]
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -505,14 +443,6 @@ jobs:
|
||||
# Output tags
|
||||
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Staging identity for this run's promotion: official tags are only
|
||||
# created after the staged image verifies against the certification,
|
||||
# and they point at this same digest (no rebuild-after-verify).
|
||||
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
|
||||
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
|
||||
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
|
||||
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Generate labels
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
@@ -537,9 +467,7 @@ jobs:
|
||||
file: ${{ matrix.file }}
|
||||
platforms: ${{ env.DOCKER_PLATFORMS }}
|
||||
push: ${{ needs.build-check.outputs.should_push == 'true' }}
|
||||
# Push to staging tags only: official tags are created by the
|
||||
# certification-gated promotion step below from this same digest.
|
||||
tags: ${{ steps.meta.outputs.staging_tags }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# No layer cache. This build compiles nothing — it downloads a
|
||||
# release zip and runs apk/apt — so the cache could only save the
|
||||
@@ -565,114 +493,6 @@ jobs:
|
||||
no-cache: false
|
||||
pull: true
|
||||
|
||||
# A later-built wrapper: the staged image must complete this platform's
|
||||
# own binary identity verification before any official tag is written.
|
||||
# It may not inherit an unverified package's conclusion.
|
||||
- name: Verify staged image against certification
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
case "${{ matrix.variant }}" in
|
||||
musl) libc="musl" ;;
|
||||
glibc) libc="gnu" ;;
|
||||
esac
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
case "$platform" in
|
||||
linux/amd64) arch_sub="x86_64" ;;
|
||||
linux/arm64) arch_sub="aarch64" ;;
|
||||
esac
|
||||
docker pull --platform "$platform" "$STAGING_REF"
|
||||
container="$(docker create --platform "$platform" "$STAGING_REF")"
|
||||
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
|
||||
docker rm "$container" >/dev/null
|
||||
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
|
||||
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
|
||||
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/staged-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Promote staged image to official tags
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
|
||||
for tag in "${official_tags[@]}"; do
|
||||
docker buildx imagetools create -t "$tag" "$STAGING_REF"
|
||||
echo "Promoted $tag -> $STAGING_REF"
|
||||
done
|
||||
|
||||
# Downstream consumers (Helm) verify this record instead of inheriting
|
||||
# an unverified wrapper's conclusion.
|
||||
- name: Record image promotion
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
jq -n \
|
||||
--arg version "$VERSION" \
|
||||
--arg digest "$digest" \
|
||||
--arg staging_ref "$STAGING_REF" \
|
||||
--arg candidate_sha256 "$candidate_sha" \
|
||||
--arg tags "${{ steps.meta.outputs.tags }}" \
|
||||
--slurpfile candidate certification/candidate.json \
|
||||
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
|
||||
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
|
||||
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
|
||||
> image-promotion-record.json
|
||||
|
||||
- name: Upload image promotion record
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: image-promotion-record${{ matrix.suffix }}
|
||||
path: image-promotion-record.json
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
# Note: Manifest creation is no longer needed as we only build one variant
|
||||
# Multi-arch manifests are automatically created by docker/build-push-action
|
||||
|
||||
|
||||
@@ -30,102 +30,9 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Consume the same trusted release certification as every other publication
|
||||
# exit and verify the image promotion record of the Docker workflow. The
|
||||
# chart wraps no binary of its own, so it may only reference an image whose
|
||||
# platform verification already passed; it must not inherit an unverified
|
||||
# package's conclusion. A run without that evidence fails closed here and
|
||||
# never reaches the chart repository.
|
||||
release-certification:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.raw_tag }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Normalize release version
|
||||
id: version
|
||||
env:
|
||||
RAW_INPUT: ${{ github.event.inputs.version }}
|
||||
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
RAW="$RAW_INPUT"
|
||||
else
|
||||
RAW="$RAW_BRANCH"
|
||||
fi
|
||||
case "$RAW" in
|
||||
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
|
||||
*) RAW_TAG="$RAW" ;;
|
||||
esac
|
||||
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download and verify release certification and image record
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification image-record
|
||||
|
||||
build_run="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$build_run" ]]; then
|
||||
build_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$build_run" ]]; then
|
||||
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$build_run" -n release-certification -D certification; then
|
||||
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
docker_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
|
||||
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
|
||||
exit 1
|
||||
fi
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
|
||||
if [[ "$record_sha" != "$candidate_sha" ]]; then
|
||||
echo "Image promotion record does not belong to this certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
|
||||
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
|
||||
image-record/image-promotion-record.json)"
|
||||
if [[ "$overlap" -ne 0 ]]; then
|
||||
echo "Image promotion record carries binary digests outside the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build-helm-package:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ release-certification ]
|
||||
if: |
|
||||
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
|
||||
(
|
||||
|
||||
+14
-210
@@ -225,57 +225,10 @@ jobs:
|
||||
echo " Build run ID: $BUILD_RUN_ID"
|
||||
echo " Build run number: $RUN_NUMBER"
|
||||
|
||||
# Consume the trusted release certification of the resolved build run and
|
||||
# verify the certified input digests before any package is wrapped. A run
|
||||
# without that certification fails closed here; packages built later must
|
||||
# complete their own verification and cannot inherit an unverified
|
||||
# conclusion.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: resolve
|
||||
if: needs.resolve.result == 'success'
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
|
||||
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
# Development artifacts are explicitly uncertified and never touch
|
||||
# official or latest channels; only the staging/dev paths remain.
|
||||
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
|
||||
> certification/development-uncertified.json
|
||||
exit 0
|
||||
fi
|
||||
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
|
||||
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Build DEB and RPM packages for each architecture and libc variant
|
||||
package:
|
||||
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
|
||||
needs: [ resolve, release-certification ]
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
@@ -541,55 +494,6 @@ jobs:
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
- name: Download certification evidence
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
# Later-built wrapper verification: the binary inside each package must
|
||||
# match the certified binary digest for this platform before anything is
|
||||
# uploaded, and the package itself stays on staging until promotion.
|
||||
- name: Verify wrapped binary against certification
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
shell: bash
|
||||
env:
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
ARTIFACT_NAME: ${{ matrix.artifact_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" ]]; then
|
||||
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y cpio
|
||||
fi
|
||||
for package_file in "$DEB_FILE" "$RPM_FILE"; do
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Upload packages to artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
@@ -627,14 +531,10 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Build-phase writes stay on staging for release types: the official
|
||||
# packages/release and packages/latest paths are written only by the
|
||||
# certification-gated promote-packages job, from these same staged
|
||||
# objects. Development packages keep the uncertified dev channel.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PREFIX="artifacts/rustfs/packages/dev"
|
||||
else
|
||||
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
|
||||
R2_PREFIX="artifacts/rustfs/packages/release"
|
||||
fi
|
||||
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
|
||||
|
||||
@@ -649,6 +549,18 @@ jobs:
|
||||
|
||||
echo "✅ Upload complete"
|
||||
|
||||
# Also upload as latest for release/prerelease
|
||||
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading latest: $(basename "$f")"
|
||||
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
@@ -702,114 +614,6 @@ jobs:
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# The only writer of the official packages/release and packages/latest R2
|
||||
# paths. Promotion re-checks each package's wrapped binary against the
|
||||
# passed certification list and copies the staged objects byte-for-byte:
|
||||
# no rebuild-after-verify, no mutable latest resolution downstream.
|
||||
promote-packages:
|
||||
name: Promote Packages
|
||||
needs: [ resolve, package, release-certification ]
|
||||
# Target-channel lock for the official package paths.
|
||||
concurrency:
|
||||
group: release-channel-r2-packages
|
||||
cancel-in-progress: false
|
||||
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download packages and certification
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./packages
|
||||
pattern: packages-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged packages
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y cpio
|
||||
fi
|
||||
|
||||
promoted=0
|
||||
for package_file in packages/*.deb packages/*.rpm; do
|
||||
[[ -f "$package_file" ]] || continue
|
||||
name="$(basename "$package_file")"
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged package ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
promoted=$((promoted + 1))
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
if [[ "$promoted" -eq 0 ]]; then
|
||||
echo "No staged packages found to promote" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
|
||||
Generated
+53
-52
@@ -4136,7 +4136,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"async-compression",
|
||||
@@ -9520,7 +9520,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9680,7 +9680,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"futures",
|
||||
@@ -9702,7 +9702,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9718,7 +9718,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"hotpath",
|
||||
@@ -9733,7 +9733,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
@@ -9746,7 +9746,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"hotpath",
|
||||
@@ -9756,7 +9756,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-connect-symbol-catalog"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"hex-simd",
|
||||
"object 0.40.0",
|
||||
@@ -9768,7 +9768,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9782,7 +9782,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9803,17 +9803,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rmp-serde",
|
||||
"rustfs-filemeta",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -9951,7 +9952,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -9961,7 +9962,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"base64-simd",
|
||||
@@ -9990,7 +9991,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -10031,7 +10032,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal-contracts"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -10041,7 +10042,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -10091,7 +10092,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
@@ -10104,7 +10105,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"hotpath",
|
||||
@@ -10168,7 +10169,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -10194,7 +10195,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -10244,14 +10245,14 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-license"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10273,7 +10274,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
@@ -10297,7 +10298,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-log-analyzer"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"flate2",
|
||||
@@ -10316,7 +10317,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
@@ -10354,7 +10355,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -10389,7 +10390,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -10408,7 +10409,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -10425,7 +10426,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -10488,7 +10489,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -10519,7 +10520,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"async-trait",
|
||||
@@ -10582,7 +10583,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
@@ -10607,7 +10608,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -10625,7 +10626,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -10671,7 +10672,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -10694,7 +10695,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-client"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10738,7 +10739,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-s3-types",
|
||||
@@ -10746,7 +10747,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -10755,7 +10756,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-compression",
|
||||
@@ -10796,7 +10797,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10818,7 +10819,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10869,7 +10870,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner-metrics"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"jiff",
|
||||
@@ -10884,7 +10885,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"thiserror 2.0.21",
|
||||
@@ -10892,7 +10893,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10911,7 +10912,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10926,7 +10927,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -10980,7 +10981,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-data-usage",
|
||||
@@ -10996,7 +10997,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"hotpath",
|
||||
@@ -11035,7 +11036,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -11072,7 +11073,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"base64-simd",
|
||||
@@ -11115,7 +11116,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"futures",
|
||||
|
||||
+51
-51
@@ -75,7 +75,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.99.0"
|
||||
version = "1.0.5"
|
||||
version = "1.0.6"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -92,56 +92,56 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.5" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.5" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.5" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.5" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.5" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.5" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.5" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.5" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.5" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.5" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.5" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.5" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.5" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.5" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.5" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.5" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.5" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.5" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.5" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.5" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.5" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.5" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.5" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.5" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.5" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.5", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.5" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.5" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.5" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.5" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.5" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.5" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.5" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.5" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.5" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.5" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.5" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.5" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.5" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.5" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.5" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.5" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.5" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.5" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.5" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.5" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.5" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.5" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.5" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.6" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.6" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.6" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.6" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.6" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.6" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.6" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.6" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.6" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.6" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.6" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.6" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.6" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.6" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.6" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.6" }
|
||||
rustfs-license = { path = "crates/license", version = "1.0.6" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.6" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.6" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.6" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.6" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.6" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.6" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.6" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.6" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.6", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.6" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.6" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.6" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.6" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.6" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.6" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.6" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.6" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.6" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.6" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.6" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.6" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.6" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.6" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.6" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.6" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.6" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.6" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.6" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.6" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.6" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.6" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.6" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.6" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
|
||||
@@ -121,6 +121,11 @@ when restoring `/rustfs/console` for a custom OEM build.
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_OBJECTS` (canonical)
|
||||
- `RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES` (canonical)
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_WALKERS_PER_DISK` (canonical)
|
||||
@@ -130,6 +135,12 @@ Scanner cycle budget controls:
|
||||
- When `RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS` is unset, the finite default is 1800 seconds (30 minutes), matching the scanner benchmark guidance.
|
||||
- An explicit `0` preserves the compatibility behavior of an unbounded runtime budget. Object and directory budgets likewise remain unbounded when explicitly set to `0`.
|
||||
|
||||
Scanner bucket quantum controls:
|
||||
|
||||
- `RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS`, `RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS`, `RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES`, `RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES`, and `RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES` cap one bucket admission. Their persistent keys are `scanner.bucket_quantum_duration`, `scanner.bucket_quantum_objects`, `scanner.bucket_quantum_directories`, `scanner.bucket_quantum_entries`, and `scanner.bucket_quantum_name_bytes`; duration is in whole seconds and the remaining keys are work counts or bytes.
|
||||
- All five defaults are `0` for controlled rollout. A positive value enables that dimension; `0` disables that dimension, and all five zero values disable the bucket quantum. Parent cycle limits remain in force. Invalid unsigned integers and durations outside the platform timer range are rejected; environment overrides take precedence but cannot hide invalid persisted values. Admin scanner status reports each effective value and source under `runtime_config.bucket_quantum_*`, using `bucket_quantum_duration_seconds` for duration.
|
||||
- Disabled defaults are an initial rollout state, not completion of usage-convergence acceptance. Production defaults require matched-topology A/B measurements and the remote progress/fairness checks in [Scanner runtime controls](../../docs/operations/scanner-runtime-controls.md#bucket-quantum-rollout-and-acceptance). These controls do not certify partial usage or acknowledge pending dirty work.
|
||||
|
||||
Scanner recovery retry controls:
|
||||
|
||||
- `RUSTFS_SCANNER_RECOVERY_BACKOFF_CAP_SECS` caps the wait between automatic startup recovery retries. The persistent admin key is `scanner.recovery_backoff_cap`; the environment takes precedence over the persistent value, and the unset default remains 600 seconds (10 minutes). Values must be positive integer seconds within the platform timer range; `0`, malformed values, and overflow are rejected. A valid environment override does not hide an invalid persistent value from `admin config set` validation.
|
||||
|
||||
@@ -44,6 +44,21 @@ pub const SCANNER_CYCLE_MAX_OBJECTS: &str = "cycle_max_objects";
|
||||
/// Scanner config key capping directories entered by one cycle.
|
||||
pub const SCANNER_CYCLE_MAX_DIRECTORIES: &str = "cycle_max_directories";
|
||||
|
||||
/// Per-bucket admission limit in seconds; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_DURATION: &str = "bucket_quantum_duration";
|
||||
|
||||
/// Per-bucket admission limit in objects; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_OBJECTS: &str = "bucket_quantum_objects";
|
||||
|
||||
/// Per-bucket admission limit in directories; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_DIRECTORIES: &str = "bucket_quantum_directories";
|
||||
|
||||
/// Per-bucket admission limit in raw entries; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_ENTRIES: &str = "bucket_quantum_entries";
|
||||
|
||||
/// Per-bucket admission limit in raw entry-name bytes; zero disables this dimension.
|
||||
pub const SCANNER_BUCKET_QUANTUM_NAME_BYTES: &str = "bucket_quantum_name_bytes";
|
||||
|
||||
/// Scanner config key setting the periodic bitrot scan cycle in seconds.
|
||||
pub const SCANNER_BITROT_CYCLE: &str = "bitrot_cycle";
|
||||
|
||||
@@ -93,6 +108,11 @@ pub const SCANNER_KEYS: &[&str] = &[
|
||||
SCANNER_CYCLE_MAX_DURATION,
|
||||
SCANNER_CYCLE_MAX_OBJECTS,
|
||||
SCANNER_CYCLE_MAX_DIRECTORIES,
|
||||
SCANNER_BUCKET_QUANTUM_DURATION,
|
||||
SCANNER_BUCKET_QUANTUM_OBJECTS,
|
||||
SCANNER_BUCKET_QUANTUM_DIRECTORIES,
|
||||
SCANNER_BUCKET_QUANTUM_ENTRIES,
|
||||
SCANNER_BUCKET_QUANTUM_NAME_BYTES,
|
||||
SCANNER_BITROT_CYCLE,
|
||||
SCANNER_IDLE_MODE,
|
||||
SCANNER_CACHE_SAVE_TIMEOUT,
|
||||
@@ -141,6 +161,36 @@ pub const ENV_SCANNER_CYCLE_MAX_OBJECTS: &str = "RUSTFS_SCANNER_CYCLE_MAX_OBJECT
|
||||
/// - Example: `export RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES=100000`
|
||||
pub const ENV_SCANNER_CYCLE_MAX_DIRECTORIES: &str = "RUSTFS_SCANNER_CYCLE_MAX_DIRECTORIES";
|
||||
|
||||
/// Per-bucket admission limit in seconds; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_DURATION_SECS: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_DURATION_SECS";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_DURATION_SECS: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in objects; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_OBJECTS: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_OBJECTS";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_OBJECTS: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in directories; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_DIRECTORIES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_DIRECTORIES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_DIRECTORIES: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in raw entries; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_ENTRIES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_ENTRIES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_ENTRIES: u64 = 0;
|
||||
|
||||
/// Per-bucket admission limit in raw entry-name bytes; zero disables this dimension.
|
||||
pub const ENV_SCANNER_BUCKET_QUANTUM_NAME_BYTES: &str = "RUSTFS_SCANNER_BUCKET_QUANTUM_NAME_BYTES";
|
||||
|
||||
/// Disabled until controlled-fleet acceptance selects production limits.
|
||||
pub const DEFAULT_SCANNER_BUCKET_QUANTUM_NAME_BYTES: u64 = 0;
|
||||
|
||||
/// Environment variable that selects the scanner speed preset.
|
||||
/// Valid values: `fastest`, `fast`, `default`, `slow`, `slowest`.
|
||||
/// Controls the sleep factor, maximum sleep duration, and cycle interval.
|
||||
|
||||
@@ -39,5 +39,8 @@ serde = { workspace = true, features = ["derive"] }
|
||||
rmp-serde = { workspace = true }
|
||||
rustfs-filemeta = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
serde_json.workspace = true
|
||||
|
||||
[lib]
|
||||
doctest = false
|
||||
|
||||
@@ -474,6 +474,12 @@ pub struct BucketTargetUsageInfo {
|
||||
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct BucketUsageInfo {
|
||||
pub size: u64,
|
||||
/// Provenance of this value. Missing and unknown states remain unproven.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_value_state: Option<crate::BucketUsageValueState>,
|
||||
/// Certification of this exact value, independent of aggregation time.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub usage_certification: Option<crate::BucketUsageCertification>,
|
||||
/// Bucket identity certified by every set contributing to this snapshot.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bucket_incarnation: Option<String>,
|
||||
@@ -580,6 +586,18 @@ pub struct DataUsageInfo {
|
||||
/// which last-known-good generation contributed to the view.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub usage_snapshot_set_states: Vec<DataUsageSnapshotSetState>,
|
||||
/// Complete mixed observations for admin display only. Legacy bucket maps
|
||||
/// and quota consumers continue to use the canonical values above.
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "HashMap::is_empty",
|
||||
deserialize_with = "crate::usage_observation::deserialize_bucket_observations"
|
||||
)]
|
||||
pub usage_snapshot_bucket_observations: HashMap<String, crate::DataUsageBucketObservation>,
|
||||
/// Response-only marker: a projected admin view must never be persisted or
|
||||
/// admitted as a quota/cache baseline, including after serialization.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub usage_snapshot_admin_projection: bool,
|
||||
/// Buckets whose dirty-usage generation moved while this complete
|
||||
/// snapshot was being walked. Their `buckets_usage` values are the walk
|
||||
/// result, which the cycle could not certify as current; the next cycle
|
||||
@@ -709,7 +727,8 @@ pub fn observed_data_usage_is_newer(observed: &DataUsageInfo, authoritative: &Da
|
||||
/// This applies the structural, exact-baseline, and generation checks of
|
||||
/// [`observed_data_usage_is_newer`] without retaining the baseline's bucket maps.
|
||||
pub fn observed_data_usage_is_newer_than_identity(observed: &DataUsageInfo, authoritative: DataUsageSnapshotIdentity) -> bool {
|
||||
observed.usage_snapshot_converged == Some(false)
|
||||
!observed.usage_snapshot_admin_projection
|
||||
&& observed.usage_snapshot_converged == Some(false)
|
||||
&& (observed.is_complete_bucket_usage_snapshot() || observed.is_valid_partial_snapshot())
|
||||
&& observed.usage_snapshot_authoritative_baseline == Some(authoritative)
|
||||
&& data_usage_identity_is_newer(observed.snapshot_identity(), authoritative)
|
||||
@@ -1764,12 +1783,7 @@ impl DataUsageCache {
|
||||
let mut found = HashSet::new();
|
||||
found.insert(top);
|
||||
mark(self, &top_e, &mut found);
|
||||
self.cache.retain(|k, _| {
|
||||
if !found.contains(k) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
});
|
||||
self.cache.retain(|k, _| found.contains(k));
|
||||
}
|
||||
|
||||
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
|
||||
@@ -2029,7 +2043,8 @@ impl DataUsageInfo {
|
||||
|
||||
/// Whether this snapshot authoritatively covers every reported bucket.
|
||||
pub fn is_complete_bucket_usage_snapshot(&self) -> bool {
|
||||
!self.usage_snapshot_bootstrap_pending
|
||||
!self.usage_snapshot_admin_projection
|
||||
&& !self.usage_snapshot_bootstrap_pending
|
||||
&& self.usage_snapshot_complete
|
||||
&& self.last_update.is_some()
|
||||
&& u64::try_from(self.buckets_usage.len()).ok() == Some(self.buckets_count)
|
||||
@@ -2039,7 +2054,8 @@ impl DataUsageInfo {
|
||||
/// admin display. Partial data is accepted only with unique set states,
|
||||
/// a plan digest for every state, and at least one usable generation.
|
||||
pub fn is_valid_partial_snapshot(&self) -> bool {
|
||||
if self.usage_snapshot_bootstrap_pending
|
||||
if self.usage_snapshot_admin_projection
|
||||
|| self.usage_snapshot_bootstrap_pending
|
||||
|| !self.usage_snapshot_partial
|
||||
|| self.usage_snapshot_converged != Some(false)
|
||||
|| self.last_update.is_none()
|
||||
@@ -2077,9 +2093,11 @@ impl DataUsageInfo {
|
||||
Ok(name) => name,
|
||||
Err(_) => return,
|
||||
};
|
||||
self.usage_snapshot_bucket_observations.remove(&bucket_name);
|
||||
|
||||
// Update bucket statistics
|
||||
if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
|
||||
bucket_usage.mark_observed();
|
||||
bucket_usage.size += meta_object.size as u64;
|
||||
bucket_usage.objects_count += 1;
|
||||
bucket_usage.versions_count += 1; // Simplified: assume 1 version per object
|
||||
@@ -2132,6 +2150,7 @@ impl DataUsageInfo {
|
||||
Ok(name) => name,
|
||||
Err(_) => return,
|
||||
};
|
||||
self.usage_snapshot_bucket_observations.remove(&bucket_name);
|
||||
|
||||
// Calculate accurate statistics from all versions
|
||||
let mut total_size = 0u64;
|
||||
@@ -2160,6 +2179,7 @@ impl DataUsageInfo {
|
||||
|
||||
// Update bucket statistics
|
||||
if let Some(bucket_usage) = self.buckets_usage.get_mut(&bucket_name) {
|
||||
bucket_usage.mark_observed();
|
||||
bucket_usage.size += total_size;
|
||||
bucket_usage.objects_count += 1;
|
||||
bucket_usage.versions_count += versions_count;
|
||||
@@ -2340,6 +2360,9 @@ impl BucketUsageInfo {
|
||||
/// Add size summary to this bucket usage
|
||||
/// Merge another BucketUsageInfo into this one
|
||||
pub fn merge(&mut self, other: &BucketUsageInfo) {
|
||||
// Adding contributions produces a new value, not either input's
|
||||
// whole-bucket certificate. The scanner may certify the final sum.
|
||||
self.mark_observed();
|
||||
if self.bucket_incarnation != other.bucket_incarnation {
|
||||
self.bucket_incarnation = None;
|
||||
}
|
||||
|
||||
@@ -13,5 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod data_usage;
|
||||
pub mod usage_observation;
|
||||
|
||||
pub use data_usage::*;
|
||||
pub use usage_observation::*;
|
||||
|
||||
@@ -0,0 +1,265 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Diagnostic provenance and response-only projections of complete bucket observations.
|
||||
|
||||
use crate::{BucketUsageInfo, DataUsageInfo, DataUsageSnapshotIdentity};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{collections::HashMap, time::SystemTime};
|
||||
|
||||
pub const DATA_USAGE_BUCKET_OBSERVATION_VERSION: u16 = 1;
|
||||
|
||||
/// Provenance of the returned bucket value, independent of quota authority.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum BucketUsageValueState {
|
||||
Certified,
|
||||
Carried,
|
||||
Observed,
|
||||
#[serde(other)]
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// A certification of this exact bucket value, never its publication timestamp.
|
||||
#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct BucketUsageCertification {
|
||||
pub epoch: u64,
|
||||
pub cycle: u64,
|
||||
pub certified_at: SystemTime,
|
||||
}
|
||||
|
||||
/// Immutable provenance of one complete traversal contribution.
|
||||
///
|
||||
/// The scanner validates its storage-owned sweep identity before constructing
|
||||
/// this public projection. Equal start/request plans are allowed when a fresh
|
||||
/// set contributes alongside another set's mixed traversal.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketObservationSource {
|
||||
pub pool_index: u64,
|
||||
pub set_index: u64,
|
||||
pub bucket_incarnation: String,
|
||||
pub scanner_epoch: u64,
|
||||
pub scanner_cycle: u64,
|
||||
pub scan_plan_digest: [u8; 32],
|
||||
pub started_plan: [u8; 32],
|
||||
pub requested_plan: [u8; 32],
|
||||
pub completed_at: SystemTime,
|
||||
}
|
||||
|
||||
/// An additive companion; legacy bucket maps never contain this mixed value.
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
|
||||
pub struct DataUsageBucketObservation {
|
||||
pub version: u16,
|
||||
pub usage: BucketUsageInfo,
|
||||
/// Exact enclosing candidate identity. Rewrapping preserves source times.
|
||||
pub snapshot_identity: DataUsageSnapshotIdentity,
|
||||
/// Oldest contribution time, so a newer set cannot hide an older value.
|
||||
pub completed_at: SystemTime,
|
||||
/// Strictly ordered, complete coverage of the enclosing live set inventory.
|
||||
pub sources: Vec<DataUsageBucketObservationSource>,
|
||||
}
|
||||
|
||||
/// Companion evolution must not make an otherwise usable canonical snapshot
|
||||
/// unreadable. Unsupported or malformed diagnostic entries never grant proof.
|
||||
pub(crate) fn deserialize_bucket_observations<'de, D>(
|
||||
deserializer: D,
|
||||
) -> Result<HashMap<String, DataUsageBucketObservation>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum Entry {
|
||||
Observation(Box<DataUsageBucketObservation>),
|
||||
Ignored(serde::de::IgnoredAny),
|
||||
}
|
||||
Ok(HashMap::<String, Entry>::deserialize(deserializer)?
|
||||
.into_iter()
|
||||
.filter_map(|(bucket, entry)| match entry {
|
||||
Entry::Observation(observation) if observation.version == DATA_USAGE_BUCKET_OBSERVATION_VERSION => {
|
||||
Some((bucket, *observation))
|
||||
}
|
||||
Entry::Observation(_) | Entry::Ignored(_) => None,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
impl BucketUsageInfo {
|
||||
/// Attach a certificate only after the caller proves complete bucket coverage.
|
||||
pub fn certify(&mut self, certification: BucketUsageCertification) {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Certified);
|
||||
self.usage_certification = Some(certification);
|
||||
}
|
||||
|
||||
/// Preserve the certificate when carrying the exact prior bucket value.
|
||||
pub fn mark_carried(&mut self) {
|
||||
self.normalize_usage_provenance();
|
||||
self.usage_value_state = Some(BucketUsageValueState::Carried);
|
||||
}
|
||||
|
||||
/// The value changed without a complete bucket certificate.
|
||||
pub fn mark_observed(&mut self) {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Observed);
|
||||
self.usage_certification = None;
|
||||
}
|
||||
|
||||
/// Future or legacy provenance cannot authorize a retained certificate.
|
||||
pub fn normalize_usage_provenance(&mut self) {
|
||||
if !matches!(
|
||||
self.usage_value_state,
|
||||
Some(BucketUsageValueState::Certified | BucketUsageValueState::Carried)
|
||||
) {
|
||||
self.usage_certification = None;
|
||||
}
|
||||
if self.usage_value_state == Some(BucketUsageValueState::Certified) && self.usage_certification.is_none() {
|
||||
self.usage_value_state = Some(BucketUsageValueState::Unknown);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageBucketObservation {
|
||||
/// Verify the public provenance against the enclosing candidate and current
|
||||
/// storage-owned incarnation. This never grants publication or ACK authority.
|
||||
pub fn matches_snapshot(&self, snapshot: &DataUsageInfo, incarnation: &str) -> bool {
|
||||
let Some(root_generation) = snapshot.scanner_epoch.zip(snapshot.scanner_cycle) else {
|
||||
return false;
|
||||
};
|
||||
if self.version != DATA_USAGE_BUCKET_OBSERVATION_VERSION
|
||||
|| snapshot.usage_snapshot_admin_projection
|
||||
|| self.snapshot_identity != snapshot.snapshot_identity()
|
||||
|| incarnation.is_empty()
|
||||
|| self.usage.bucket_incarnation.as_deref() != Some(incarnation)
|
||||
|| self.sources.is_empty()
|
||||
|| self.usage.usage_certification.is_some()
|
||||
|| self.usage.usage_value_state != Some(BucketUsageValueState::Observed)
|
||||
|| self.sources.iter().map(|source| source.completed_at).min() != Some(self.completed_at)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut previous = None;
|
||||
let mut plan = None;
|
||||
for state in &snapshot.usage_snapshot_set_states {
|
||||
let key = (state.pool_index, state.set_index);
|
||||
if previous.is_some_and(|previous| previous >= key)
|
||||
|| state.scan_plan_digest.is_none()
|
||||
|| plan.is_some_and(|plan| Some(plan) != state.scan_plan_digest)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
previous = Some(key);
|
||||
plan = state.scan_plan_digest;
|
||||
}
|
||||
let mut sources = self.sources.iter();
|
||||
for state in snapshot.usage_snapshot_set_states.iter().filter(|state| !state.tombstone) {
|
||||
let Some(source) = sources.next() else {
|
||||
return false;
|
||||
};
|
||||
let Some(set_generation) = state.scanner_epoch.zip(state.scanner_cycle) else {
|
||||
return false;
|
||||
};
|
||||
if (source.pool_index, source.set_index) != (state.pool_index, state.set_index)
|
||||
|| source.bucket_incarnation != incarnation
|
||||
|| Some(source.scan_plan_digest) != state.scan_plan_digest
|
||||
|| (source.scanner_epoch, source.scanner_cycle) > set_generation
|
||||
|| set_generation > root_generation
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
sources.next().is_none()
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageInfo {
|
||||
/// Overlay validated companions on an owned admin response only.
|
||||
///
|
||||
/// `incarnations` must come from current storage metadata, not the companion.
|
||||
/// The projection is marked explicitly so persistence and quota boundaries
|
||||
/// can reject accidental reuse. Invalid companions leave canonical values
|
||||
/// unchanged. Counter overflow rejects the whole projection.
|
||||
pub fn project_bucket_observations_for_admin(&mut self, incarnations: &HashMap<String, String>) -> usize {
|
||||
if self.usage_snapshot_admin_projection
|
||||
|| (!self.is_complete_bucket_usage_snapshot() && !self.is_valid_partial_snapshot())
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
let observations =
|
||||
self.usage_snapshot_bucket_observations
|
||||
.iter()
|
||||
.filter(|(bucket, observation)| {
|
||||
if self.usage_snapshot_complete
|
||||
&& (!self.usage_snapshot_pending_buckets.iter().any(|pending| pending == *bucket)
|
||||
|| u64::try_from(self.usage_snapshot_pending_buckets.len()).ok()
|
||||
!= Some(self.usage_snapshot_pending_bucket_count))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if self
|
||||
.buckets_usage
|
||||
.get(*bucket)
|
||||
.and_then(|usage| usage.usage_certification)
|
||||
.is_some_and(|certificate| {
|
||||
observation.sources.iter().all(|source| {
|
||||
(source.scanner_epoch, source.scanner_cycle) <= (certificate.epoch, certificate.cycle)
|
||||
})
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
incarnations.get(*bucket).is_some_and(|incarnation| {
|
||||
self.buckets_usage
|
||||
.get(*bucket)
|
||||
.and_then(|usage| usage.bucket_incarnation.as_ref())
|
||||
.is_none_or(|canonical| canonical == incarnation)
|
||||
&& observation.matches_snapshot(self, incarnation)
|
||||
})
|
||||
})
|
||||
.collect::<HashMap<_, _>>();
|
||||
if observations.is_empty() {
|
||||
return 0;
|
||||
}
|
||||
let mut values = self
|
||||
.buckets_usage
|
||||
.iter()
|
||||
.filter(|(bucket, _)| !observations.contains_key(bucket))
|
||||
.map(|(_, usage)| usage)
|
||||
.chain(observations.values().map(|observation| &observation.usage));
|
||||
let totals = values.try_fold([0_u64; 4], |total, usage| {
|
||||
Some([
|
||||
total[0].checked_add(usage.size)?,
|
||||
total[1].checked_add(usage.objects_count)?,
|
||||
total[2].checked_add(usage.versions_count)?,
|
||||
total[3].checked_add(usage.delete_markers_count)?,
|
||||
])
|
||||
});
|
||||
let Some([size, objects, versions, markers]) = totals else {
|
||||
return 0;
|
||||
};
|
||||
let projected = observations.len();
|
||||
for (bucket, observation) in observations {
|
||||
self.buckets_usage.insert(bucket.clone(), observation.usage.clone());
|
||||
}
|
||||
self.bucket_sizes = self
|
||||
.buckets_usage
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
.collect();
|
||||
self.buckets_count = u64::try_from(self.buckets_usage.len()).unwrap_or(u64::MAX);
|
||||
self.objects_total_size = size;
|
||||
self.objects_total_count = objects;
|
||||
self.versions_total_count = versions;
|
||||
self.delete_markers_total_count = markers;
|
||||
self.replication_info.clear();
|
||||
self.tier_stats = None;
|
||||
self.unknown_tier_stats = None;
|
||||
self.usage_snapshot_complete = false;
|
||||
self.usage_snapshot_partial = true;
|
||||
self.usage_snapshot_converged = Some(false);
|
||||
self.usage_snapshot_observational = true;
|
||||
self.usage_snapshot_admin_projection = true;
|
||||
projected
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,323 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::DataUsageSnapshotSetState;
|
||||
use std::time::Duration;
|
||||
|
||||
const INCARNATION: &str = "11111111-1111-4111-8111-111111111111";
|
||||
|
||||
fn time(seconds: u64) -> SystemTime {
|
||||
SystemTime::UNIX_EPOCH + Duration::from_secs(seconds)
|
||||
}
|
||||
|
||||
fn usage(size: u64) -> BucketUsageInfo {
|
||||
BucketUsageInfo {
|
||||
size,
|
||||
objects_count: size,
|
||||
versions_count: size,
|
||||
bucket_incarnation: Some(INCARNATION.to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot() -> DataUsageInfo {
|
||||
let mut main = usage(10);
|
||||
main.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 3,
|
||||
certified_at: time(30),
|
||||
});
|
||||
main.mark_carried();
|
||||
let mut snapshot = DataUsageInfo {
|
||||
last_update: Some(time(90)),
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(9),
|
||||
buckets_count: 1,
|
||||
buckets_usage: HashMap::from([("bucket".to_string(), main)]),
|
||||
bucket_sizes: HashMap::from([("bucket".to_string(), 10)]),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_pending_buckets: vec!["bucket".to_string()],
|
||||
usage_snapshot_pending_bucket_count: 1,
|
||||
usage_snapshot_set_states: (0..2)
|
||||
.map(|set_index| DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index,
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(9),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: true,
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
snapshot.calculate_totals();
|
||||
let mut observed = usage(25);
|
||||
observed.mark_observed();
|
||||
let observation = DataUsageBucketObservation {
|
||||
version: DATA_USAGE_BUCKET_OBSERVATION_VERSION,
|
||||
usage: observed,
|
||||
snapshot_identity: snapshot.snapshot_identity(),
|
||||
completed_at: time(40),
|
||||
sources: (0..2)
|
||||
.map(|set_index| DataUsageBucketObservationSource {
|
||||
pool_index: 0,
|
||||
set_index,
|
||||
bucket_incarnation: INCARNATION.to_string(),
|
||||
scanner_epoch: 2,
|
||||
scanner_cycle: 4 + set_index,
|
||||
scan_plan_digest: [1; 32],
|
||||
started_plan: [2; 32],
|
||||
requested_plan: [3; 32],
|
||||
completed_at: time(40 + set_index),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
snapshot
|
||||
.usage_snapshot_bucket_observations
|
||||
.insert("bucket".to_string(), observation);
|
||||
snapshot
|
||||
}
|
||||
|
||||
fn incarnations() -> HashMap<String, String> {
|
||||
HashMap::from([("bucket".to_string(), INCARNATION.to_string())])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provenance_is_additive_and_unknown_states_do_not_keep_certificates() {
|
||||
let legacy = usage(10);
|
||||
let bytes = serde_json::to_vec(&legacy).expect("encode a legacy bucket without provenance");
|
||||
let text = std::str::from_utf8(&bytes).expect("JSON is UTF-8");
|
||||
assert!(!text.contains("usage_value_state"));
|
||||
assert!(!text.contains("usage_certification"));
|
||||
let decoded: BucketUsageInfo = serde_json::from_slice(&bytes).expect("read legacy bucket");
|
||||
assert_eq!(decoded.usage_value_state, None);
|
||||
assert_eq!(decoded.usage_certification, None);
|
||||
|
||||
let mut wire = serde_json::to_value(legacy).expect("encode future bucket fixture");
|
||||
wire["usage_value_state"] = serde_json::json!("future_state");
|
||||
wire["usage_certification"] =
|
||||
serde_json::json!({"epoch":2,"cycle":3,"certified_at":{"secs_since_epoch":30,"nanos_since_epoch":0}});
|
||||
let mut future: BucketUsageInfo = serde_json::from_value(wire).expect("unknown diagnostic enum remains readable");
|
||||
future.normalize_usage_provenance();
|
||||
assert_eq!(future.usage_value_state, Some(BucketUsageValueState::Unknown));
|
||||
assert_eq!(future.usage_certification, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn carrying_preserves_time_but_adding_a_source_invalidates_the_certificate() {
|
||||
let mut bucket = snapshot().buckets_usage.remove("bucket").expect("canonical bucket");
|
||||
bucket.mark_carried();
|
||||
assert_eq!(bucket.usage_certification.map(|certification| certification.certified_at), Some(time(30)));
|
||||
bucket.merge(&usage(1));
|
||||
assert_eq!(bucket.size, 11);
|
||||
assert_eq!(bucket.usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(bucket.usage_certification, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn direct_object_accumulation_does_not_reuse_the_old_certificate() {
|
||||
let mut simple = snapshot();
|
||||
simple.add_object("bucket/object", &rustfs_filemeta::MetaObject::default());
|
||||
assert_eq!(simple.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(simple.buckets_usage["bucket"].usage_certification, None);
|
||||
assert!(simple.usage_snapshot_bucket_observations.is_empty());
|
||||
|
||||
let mut versioned = snapshot();
|
||||
versioned.add_object_from_file_meta("bucket/object", &rustfs_filemeta::FileMeta::default());
|
||||
assert_eq!(versioned.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(versioned.buckets_usage["bucket"].usage_certification, None);
|
||||
assert!(versioned.usage_snapshot_bucket_observations.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn future_or_malformed_companions_do_not_break_the_canonical_reader() {
|
||||
let canonical = snapshot();
|
||||
for entry in [
|
||||
serde_json::json!({"version": 2, "future_payload": {"new": true}}),
|
||||
serde_json::json!({"version": 1, "usage": "broken"}),
|
||||
] {
|
||||
let mut wire = serde_json::to_value(&canonical).expect("canonical JSON");
|
||||
wire["usage_snapshot_bucket_observations"]["bucket"] = entry;
|
||||
let decoded: DataUsageInfo = serde_json::from_value(wire).expect("diagnostics cannot break canonical data");
|
||||
assert_eq!(decoded.buckets_usage, canonical.buckets_usage);
|
||||
assert!(decoded.usage_snapshot_bucket_observations.is_empty());
|
||||
assert!(decoded.is_complete_bucket_usage_snapshot());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn complete_companion_projects_only_the_admin_copy_and_keeps_measurement_times() {
|
||||
let canonical = snapshot();
|
||||
let canonical_bytes = serde_json::to_vec(&canonical).expect("encode canonical candidate");
|
||||
let mut response = canonical.clone();
|
||||
assert_eq!(response.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(response.buckets_usage["bucket"].size, 25);
|
||||
assert_eq!(response.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(response.buckets_usage["bucket"].usage_certification, None);
|
||||
assert_eq!(response.objects_total_size, 25);
|
||||
assert_eq!(response.bucket_sizes["bucket"], 25);
|
||||
assert_eq!(response.last_update, canonical.last_update);
|
||||
assert_eq!(response.usage_snapshot_bucket_observations["bucket"].completed_at, time(40));
|
||||
assert!(response.usage_snapshot_admin_projection);
|
||||
assert!(response.usage_snapshot_partial);
|
||||
assert!(response.usage_snapshot_observational);
|
||||
assert_eq!(response.usage_snapshot_converged, Some(false));
|
||||
assert!(!response.is_complete_bucket_usage_snapshot());
|
||||
assert!(
|
||||
!response.is_valid_partial_snapshot(),
|
||||
"response-only values cannot be a storage candidate"
|
||||
);
|
||||
assert_eq!(serde_json::to_vec(&canonical).expect("reencode original"), canonical_bytes);
|
||||
assert_eq!(canonical.buckets_usage["bucket"].size, 10);
|
||||
assert_eq!(
|
||||
canonical.buckets_usage["bucket"]
|
||||
.usage_certification
|
||||
.map(|certificate| certificate.certified_at),
|
||||
Some(time(30))
|
||||
);
|
||||
assert_eq!(response.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_companions_preserve_the_canonical_bucket() {
|
||||
for invalid in 0..11 {
|
||||
let mut candidate = snapshot();
|
||||
let observation = candidate
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("companion");
|
||||
match invalid {
|
||||
0 => observation.version += 1,
|
||||
1 => observation.snapshot_identity.scanner_cycle = Some(8),
|
||||
2 => {
|
||||
observation.sources.pop();
|
||||
}
|
||||
3 => observation.sources[1].set_index = 0,
|
||||
4 => observation.sources[1].scanner_cycle = 10,
|
||||
5 => observation.sources[1].scan_plan_digest = [9; 32],
|
||||
6 => observation.sources[1].bucket_incarnation = "other".to_string(),
|
||||
7 => observation.completed_at = time(41),
|
||||
8 => observation.usage.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 9,
|
||||
certified_at: time(90),
|
||||
}),
|
||||
9 => candidate.usage_snapshot_set_states[1].scanner_cycle = None,
|
||||
10 => {
|
||||
candidate
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("canonical")
|
||||
.bucket_incarnation = Some("recreated".to_string())
|
||||
}
|
||||
_ => unreachable!("bounded fixture variants"),
|
||||
}
|
||||
let before = candidate.clone();
|
||||
assert_eq!(
|
||||
candidate.project_bucket_observations_for_admin(&incarnations()),
|
||||
0,
|
||||
"invalid case {invalid}"
|
||||
);
|
||||
assert_eq!(candidate, before, "invalid case {invalid} changed canonical data");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_fresh_set_can_contribute_without_refreshing_an_older_mixed_set() {
|
||||
let mut candidate = snapshot();
|
||||
let observation = candidate
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("companion");
|
||||
observation.sources[1].started_plan = observation.sources[1].requested_plan;
|
||||
observation.sources[1].completed_at = time(90);
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(candidate.usage_snapshot_bucket_observations["bucket"].completed_at, time(40));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn companion_only_partial_keeps_legacy_maps_empty_until_admin_projection() {
|
||||
let mut candidate = snapshot();
|
||||
candidate.buckets_usage.clear();
|
||||
candidate.bucket_sizes.clear();
|
||||
candidate.buckets_count = 0;
|
||||
candidate.calculate_totals();
|
||||
candidate.usage_snapshot_complete = false;
|
||||
candidate.usage_snapshot_partial = true;
|
||||
candidate.usage_snapshot_observational = true;
|
||||
for state in &mut candidate.usage_snapshot_set_states {
|
||||
state.complete = false;
|
||||
}
|
||||
assert!(candidate.is_valid_partial_snapshot());
|
||||
let wire = serde_json::to_value(&candidate).expect("encode companion-only observation");
|
||||
assert_eq!(wire["buckets_usage"], serde_json::json!({}));
|
||||
assert_eq!(wire["bucket_sizes"], serde_json::json!({}));
|
||||
assert_eq!(wire["objects_total_size"], 0);
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 1);
|
||||
assert_eq!(candidate.objects_total_size, 25);
|
||||
assert!(candidate.usage_snapshot_partial);
|
||||
assert!(!candidate.usage_snapshot_complete);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn projection_rejects_counter_overflow_without_partial_mutation() {
|
||||
let mut candidate = snapshot();
|
||||
candidate.buckets_usage.insert("cold".to_string(), usage(u64::MAX));
|
||||
candidate.buckets_count = 2;
|
||||
let before = candidate.clone();
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
assert_eq!(candidate, before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_newer_canonical_certificate_and_missing_pending_prevent_old_overlay() {
|
||||
for newer_certificate in [false, true] {
|
||||
let mut candidate = snapshot();
|
||||
if newer_certificate {
|
||||
candidate
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("canonical")
|
||||
.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 9,
|
||||
certified_at: time(90),
|
||||
});
|
||||
} else {
|
||||
candidate.usage_snapshot_pending_buckets.clear();
|
||||
candidate.usage_snapshot_pending_bucket_count = 0;
|
||||
}
|
||||
assert_eq!(candidate.project_bucket_observations_for_admin(&incarnations()), 0);
|
||||
assert_eq!(candidate.buckets_usage["bucket"].size, 10);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provenance_and_companions_change_candidate_bytes_and_round_trip_as_maps() {
|
||||
let canonical = snapshot();
|
||||
let original = serde_json::to_vec(&canonical).expect("encode candidate");
|
||||
let mut changed = canonical.clone();
|
||||
changed
|
||||
.buckets_usage
|
||||
.get_mut("bucket")
|
||||
.expect("bucket")
|
||||
.usage_certification
|
||||
.as_mut()
|
||||
.expect("certificate")
|
||||
.certified_at = time(31);
|
||||
assert_ne!(serde_json::to_vec(&changed).expect("encode changed certificate"), original);
|
||||
let mut changed = canonical.clone();
|
||||
changed
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.expect("observation")
|
||||
.sources[0]
|
||||
.started_plan = [8; 32];
|
||||
assert_ne!(serde_json::to_vec(&changed).expect("encode changed source"), original);
|
||||
let named = rmp_serde::to_vec_named(&canonical).expect("encode named map");
|
||||
let restored: DataUsageInfo = rmp_serde::from_slice(&named).expect("read named map with additive fields");
|
||||
assert_eq!(restored, canonical);
|
||||
}
|
||||
@@ -766,9 +766,10 @@ impl Run {
|
||||
|| async {
|
||||
let mut ready = true;
|
||||
let mut observations = Vec::with_capacity(4);
|
||||
for node in 0..4 {
|
||||
assert!(self.dist.cluster.nodes.len() >= 4);
|
||||
for (node, cluster_node) in self.dist.cluster.nodes.iter().enumerate().take(4) {
|
||||
match http
|
||||
.get(format!("{}/health/ready", self.dist.cluster.nodes[node].url))
|
||||
.get(format!("{}/health/ready", cluster_node.url))
|
||||
.timeout(Duration::from_secs(3))
|
||||
.send()
|
||||
.await
|
||||
|
||||
@@ -1332,14 +1332,15 @@ mod tests {
|
||||
.collect();
|
||||
assert_eq!(confirmed.len(), 1);
|
||||
let mut receivers = Vec::new();
|
||||
for drive in 0..phase.disks.get(1).map_or(0, Vec::len) {
|
||||
let receiver_endpoints = phase.disks.get(1).map_or(&[][..], |disks| &phase.endpoints[1][..disks.len()]);
|
||||
for endpoint in receiver_endpoints {
|
||||
for cas in [prepares[1], commits[1]] {
|
||||
let matching: Vec<_> = events[1]
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|event| {
|
||||
event["kind"] == "receiver"
|
||||
&& event["disk"] == phase.endpoints[1][drive]
|
||||
&& event["disk"] == *endpoint
|
||||
&& event["dst_volume"] == ".rustfs.sys"
|
||||
&& event["dst_path"] == "pool.bin"
|
||||
&& event["etag"] == cas["etag"]
|
||||
|
||||
@@ -22,10 +22,11 @@ use rustfs_protos::{
|
||||
models::PingBodyBuilder,
|
||||
proto_gen::node_service::{
|
||||
BatchGenerallyLockRequest, BatchGenerallyLockResponse, BatchReadVersionRequest, BatchReadVersionResponse,
|
||||
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, PingRequest, PingResponse,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseReleaseResponse, ScannerPublicationLeaseRequest,
|
||||
ScannerPublicationLeaseResponse, SnapshotLeaseMutationResponse, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest,
|
||||
SnapshotLeaseRequest, SnapshotLeaseResponse, node_service_server::NodeService,
|
||||
GenerallyLockRequest, GenerallyLockResponse, GenerallyLockResult, MrfLifecycleRequest, MrfLifecycleResponse, PingRequest,
|
||||
PingResponse, ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseReleaseResponse,
|
||||
ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse, SnapshotLeaseMutationResponse,
|
||||
SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest, SnapshotLeaseResponse,
|
||||
node_service_server::NodeService,
|
||||
},
|
||||
};
|
||||
use std::pin::Pin;
|
||||
@@ -995,6 +996,10 @@ impl NodeService for MinimalLockNodeService {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn mrf_lifecycle(&self, _request: Request<MrfLifecycleRequest>) -> Result<Response<MrfLifecycleResponse>, Status> {
|
||||
Err(Status::unimplemented("lock-only test server"))
|
||||
}
|
||||
|
||||
async fn background_heal_status(
|
||||
&self,
|
||||
_request: Request<rustfs_protos::proto_gen::node_service::BackgroundHealStatusRequest>,
|
||||
|
||||
@@ -553,6 +553,10 @@ pub mod runtime {
|
||||
global_lock_client, global_lock_clients, global_tier_config_mgr, local_disk_map_read, object_store_handle, region,
|
||||
rustfs_port, setup_is_dist_erasure, setup_is_erasure, setup_is_erasure_sd, transition_state_handle,
|
||||
};
|
||||
pub use crate::runtime::usage_watch::{
|
||||
SCANNER_USAGE_WATCH_COVERAGE_VERSION, SCANNER_USAGE_WATCH_LEASE_MS, ScannerUsageWatchKnown, ScannerUsageWatchObservation,
|
||||
ScannerUsageWatchRequest,
|
||||
};
|
||||
}
|
||||
|
||||
pub mod layout {
|
||||
@@ -636,6 +640,30 @@ pub mod rio {
|
||||
}
|
||||
|
||||
pub mod rpc {
|
||||
#[cfg(feature = "test-util")]
|
||||
pub use crate::cluster::rpc::internode_data_transport::{
|
||||
InternodeDataTransport, InternodeDataTransportCapabilities, NsScannerCapabilityRequest, NsScannerStreamRequest,
|
||||
ReadStreamRequest, WalkDirStreamRequest, WriteStreamRequest,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
pub mod test_util {
|
||||
/// Build a remote disk using a controlled transport and no health monitor.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns the normal remote-disk construction error for the supplied endpoint.
|
||||
pub async fn remote_disk_with_transport(
|
||||
endpoint: &crate::layout::endpoint::Endpoint,
|
||||
transport: std::sync::Arc<dyn crate::cluster::rpc::internode_data_transport::InternodeDataTransport>,
|
||||
) -> crate::disk::error::Result<crate::disk::DiskStore> {
|
||||
let options = crate::disk::DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
};
|
||||
let disk = crate::cluster::rpc::remote_disk::RemoteDisk::new(endpoint, &options, transport).await?;
|
||||
Ok(std::sync::Arc::new(crate::disk::Disk::Remote(Box::new(disk))))
|
||||
}
|
||||
}
|
||||
|
||||
pub use crate::cluster::rpc::http_auth::with_rpc_background_request;
|
||||
pub use crate::cluster::rpc::{
|
||||
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
|
||||
@@ -648,13 +676,16 @@ pub mod rpc {
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
|
||||
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
|
||||
rpc_auth_is_resource_exhausted, rpc_background_admission, set_tonic_canonical_body_digest, sign_ns_scanner_capability,
|
||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability,
|
||||
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_durable_progress,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_ns_scanner_capability_with_usage_capabilities,
|
||||
sign_ns_scanner_usage_watch_capability, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_ns_scanner_capability_with_usage_capabilities, verify_ns_scanner_usage_watch_capability,
|
||||
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
|
||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -544,8 +544,10 @@ mod tests {
|
||||
] {
|
||||
let first: Error = make_error("buckets/first/.metadata.bin".into()).into();
|
||||
let second: Error = make_error("buckets/second/.metadata.bin".into()).into();
|
||||
let cloned = first.clone();
|
||||
assert_eq!(first, second, "record paths must not fragment error grouping");
|
||||
assert_eq!(first.clone(), second, "cloning must preserve error grouping");
|
||||
assert_eq!(cloned, second, "cloning must preserve error grouping");
|
||||
assert_eq!(std::io::Error::from(cloned).to_string(), message);
|
||||
|
||||
let io_error = std::io::Error::from(first);
|
||||
let detail = io_error
|
||||
|
||||
@@ -20,6 +20,8 @@ use crate::test_metrics::CapturingRecorder;
|
||||
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
|
||||
use std::time::SystemTime;
|
||||
|
||||
mod native_compatibility;
|
||||
|
||||
async fn stop_reporting_fixture(store: &crate::store::ECStore) {
|
||||
if let Some(cancel) = store.ctx.background_cancel_token() {
|
||||
cancel.cancel();
|
||||
@@ -90,6 +92,10 @@ async fn install_reporting_snapshot(
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial]
|
||||
async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger_admission() {
|
||||
assert_durable_quota_reporting_preserves_admission().await;
|
||||
}
|
||||
|
||||
async fn assert_durable_quota_reporting_preserves_admission() {
|
||||
let recorder = CapturingRecorder::default();
|
||||
let _recorder = metrics::set_default_local_recorder(&recorder);
|
||||
for protocol in [1, 2] {
|
||||
@@ -196,6 +202,11 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
|
||||
.expect("fixture cache cleanup");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
stop_reporting_fixture(&store).await;
|
||||
for dir in _dirs {
|
||||
let path = dir.path().to_path_buf();
|
||||
dir.close().expect("remove durable quota reporting fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -203,6 +214,10 @@ async fn durable_quota_reporting_retires_degraded_floors_without_changing_ledger
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn snapshot_quota_admission_retains_degraded_floors_until_migration() {
|
||||
assert_snapshot_quota_preserves_degraded_floors().await;
|
||||
}
|
||||
|
||||
async fn assert_snapshot_quota_preserves_degraded_floors() {
|
||||
for source in ["missing", "legacy", "incomplete", "observed"] {
|
||||
let (_dirs, store, _) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("quota-compat-{}", Uuid::new_v4().simple());
|
||||
@@ -233,12 +248,21 @@ async fn snapshot_quota_admission_retains_degraded_floors_until_migration() {
|
||||
.expect("fixture cache cleanup");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
stop_reporting_fixture(&store).await;
|
||||
for dir in _dirs {
|
||||
let path = dir.path().to_path_buf();
|
||||
dir.close().expect("remove legacy quota reporting fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial]
|
||||
async fn quota_without_limit_keeps_the_zero_usage_io_fast_path() {
|
||||
assert_unlimited_quota_preserves_fast_path().await;
|
||||
}
|
||||
|
||||
async fn assert_unlimited_quota_preserves_fast_path() {
|
||||
let (_dirs, store, sys) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("quota-unlimited-{}", Uuid::new_v4().simple());
|
||||
create_bucket_with_quota_json(
|
||||
@@ -259,6 +283,11 @@ async fn quota_without_limit_keeps_the_zero_usage_io_fast_path() {
|
||||
assert_eq!(recorder.counter_value(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL, &[]), 0);
|
||||
assert_eq!(crate::data_usage::reload_unknown_bucket_usage_calls_for_test(&bucket), 0);
|
||||
stop_reporting_fixture(&store).await;
|
||||
for dir in _dirs {
|
||||
let path = dir.path().to_path_buf();
|
||||
dir.close().expect("remove unlimited quota reporting fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
use rustfs_data_usage::{
|
||||
DATA_USAGE_BUCKET_OBSERVATION_VERSION, DataUsageBucketObservation, DataUsageBucketObservationSource,
|
||||
DataUsageSnapshotSetState,
|
||||
};
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial]
|
||||
async fn usage_quota_modes_preserve_actual_admission_decisions() {
|
||||
let (dirs, store, sys) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("usage-no-quota-{}", Uuid::new_v4().simple());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("unconfigured quota bucket");
|
||||
let check = QuotaChecker::new(sys)
|
||||
.check_quota(&bucket, QuotaOperation::PutObject, u64::MAX)
|
||||
.await
|
||||
.expect("no quota");
|
||||
assert!(check.allowed && check.quota_limit.is_none());
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"allowed",
|
||||
&mut PutObjReader::from_vec(vec![0x5a; 512]),
|
||||
&ObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("real no-quota write");
|
||||
stop_reporting_fixture(&store).await;
|
||||
let paths = dirs.iter().map(|dir| dir.path().to_path_buf()).collect::<Vec<_>>();
|
||||
drop(store);
|
||||
drop(dirs);
|
||||
assert!(paths.iter().all(|path| !path.exists()));
|
||||
|
||||
assert_unlimited_quota_preserves_fast_path().await;
|
||||
assert_snapshot_quota_preserves_degraded_floors().await;
|
||||
assert_durable_quota_reporting_preserves_admission().await;
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-19", "assertion_id": "USAGE-19-A01", "status": "PASS",
|
||||
"evidence": {
|
||||
"quota_modes": ["unconfigured", "unlimited", "legacy-snapshot", "durable-v1", "durable-v2"],
|
||||
"durable_snapshot_sources": ["missing", "legacy", "incomplete", "observed", "complete"],
|
||||
"durable_storage_decisions_per_fixture": ["reject-8192", "accept-512", "accept-3584", "reject-1"],
|
||||
"legacy_missing_baseline_fails_closed": true,
|
||||
"legacy_grounded_baselines": ["legacy", "incomplete", "observed"]
|
||||
}
|
||||
})
|
||||
);
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-19", "status": "PASS", "fixtures_removed": true})
|
||||
);
|
||||
}
|
||||
|
||||
async fn persist_companion_fixture(
|
||||
store: Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
primary: bool,
|
||||
companion_size: u64,
|
||||
) -> (String, Vec<u8>) {
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("real companion bucket identity");
|
||||
let completed_at = SystemTime::UNIX_EPOCH + std::time::Duration::from_secs(20);
|
||||
let mut snapshot = DataUsageInfo {
|
||||
last_update: Some(completed_at),
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(3),
|
||||
usage_snapshot_complete: primary,
|
||||
usage_snapshot_partial: !primary,
|
||||
usage_snapshot_observational: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(DataUsageInfo::default().snapshot_identity()),
|
||||
usage_snapshot_pending_buckets: vec![bucket.to_owned()],
|
||||
usage_snapshot_pending_bucket_count: 1,
|
||||
usage_snapshot_set_states: vec![DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(3),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: primary,
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
if primary {
|
||||
snapshot.buckets_count = 1;
|
||||
snapshot.buckets_usage.insert(
|
||||
bucket.into(),
|
||||
BucketUsageInfo {
|
||||
size: 1000,
|
||||
objects_count: 1,
|
||||
versions_count: 1,
|
||||
bucket_incarnation: Some(incarnation.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
snapshot.bucket_sizes.insert(bucket.into(), 1000);
|
||||
snapshot.calculate_totals();
|
||||
}
|
||||
let mut usage = BucketUsageInfo {
|
||||
size: companion_size,
|
||||
objects_count: 1,
|
||||
versions_count: 1,
|
||||
bucket_incarnation: Some(incarnation.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
usage.mark_observed();
|
||||
snapshot.usage_snapshot_bucket_observations.insert(
|
||||
bucket.into(),
|
||||
DataUsageBucketObservation {
|
||||
version: DATA_USAGE_BUCKET_OBSERVATION_VERSION,
|
||||
usage,
|
||||
snapshot_identity: snapshot.snapshot_identity(),
|
||||
completed_at,
|
||||
sources: vec![DataUsageBucketObservationSource {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
bucket_incarnation: incarnation.to_string(),
|
||||
scanner_epoch: 2,
|
||||
scanner_cycle: 3,
|
||||
scan_plan_digest: [1; 32],
|
||||
started_plan: [2; 32],
|
||||
requested_plan: [3; 32],
|
||||
completed_at,
|
||||
}],
|
||||
},
|
||||
);
|
||||
let mut admin = snapshot.clone();
|
||||
assert_eq!(
|
||||
admin.project_bucket_observations_for_admin(&HashMap::from([(bucket.to_owned(), incarnation.to_string())])),
|
||||
1,
|
||||
"the companion must be valid, not discarded as malformed"
|
||||
);
|
||||
assert_eq!(admin.buckets_usage[bucket].size, companion_size);
|
||||
let path = format!(
|
||||
"{}/{}",
|
||||
crate::disk::BUCKET_META_PREFIX,
|
||||
rustfs_data_usage::DATA_USAGE_OBSERVED_OBJECT_NAME
|
||||
);
|
||||
let bytes = serde_json::to_vec(&snapshot).expect("encode valid companion snapshot");
|
||||
crate::config::com::save_config(store, &path, bytes.clone())
|
||||
.await
|
||||
.expect("persist companion beside canonical fields");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
(path, bytes)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
#[serial]
|
||||
async fn usage_companions_do_not_expand_legacy_fallback_or_durable_admission() {
|
||||
let mut decisions = Vec::new();
|
||||
for protocol in [0, 1, 2] {
|
||||
for (primary, companion_size) in [(true, 0), (true, 1_000_000), (false, 1_000_000)] {
|
||||
let (dirs, store, sys) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("usage-companion-quota-{}", Uuid::new_v4().simple());
|
||||
let quota = match protocol {
|
||||
0 => legacy_quota_json(4096),
|
||||
1 => serde_json::to_vec(&BucketQuota::new(Some(4096))).unwrap(),
|
||||
_ => serde_json::to_vec(&BucketQuota::new_sharded(4096)).unwrap(),
|
||||
};
|
||||
create_bucket_with_quota_json(&store, &bucket, quota).await;
|
||||
let (path, bytes) = persist_companion_fixture(store.clone(), &bucket, primary, companion_size).await;
|
||||
let checker = if protocol == 0 {
|
||||
QuotaChecker::new(Arc::new(RwLock::new(BucketMetadataSys::new(store.clone()))))
|
||||
} else {
|
||||
QuotaChecker::new(sys)
|
||||
};
|
||||
if protocol == 0 {
|
||||
let small = checker.check_quota(&bucket, QuotaOperation::PutObject, 512).await;
|
||||
if primary {
|
||||
let small = small.expect("grounded primary fallback");
|
||||
assert!(small.allowed && !small.uses_durable_reservations);
|
||||
assert_eq!(small.current_usage, Some(1000));
|
||||
assert!(
|
||||
!checker
|
||||
.check_quota(&bucket, QuotaOperation::PutObject, 4096)
|
||||
.await
|
||||
.expect("legacy limit")
|
||||
.allowed
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
matches!(small, Err(QuotaError::UsageUnavailable { .. })),
|
||||
"a companion cannot invent a legacy baseline"
|
||||
);
|
||||
}
|
||||
} else {
|
||||
assert!(
|
||||
checker
|
||||
.check_quota(&bucket, QuotaOperation::PutObject, 512)
|
||||
.await
|
||||
.expect("durable checker")
|
||||
.uses_durable_reservations
|
||||
);
|
||||
if protocol == 1 {
|
||||
assert!(matches!(
|
||||
store
|
||||
.put_object(
|
||||
&bucket,
|
||||
"bootstrap",
|
||||
&mut PutObjReader::from_vec(vec![0x5a; 512]),
|
||||
&ObjectOptions::default()
|
||||
)
|
||||
.await,
|
||||
Err(StorageError::SlowDown)
|
||||
));
|
||||
crate::bucket::quota::reservation::retry_quota_reconciliation_for_test(store.clone(), &bucket)
|
||||
.await
|
||||
.expect("real v1 ledger bootstrap");
|
||||
}
|
||||
store
|
||||
.put_object(&bucket, "fits", &mut PutObjReader::from_vec(vec![0x5a; 4096]), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("ledger admits exactly the physical quota regardless of companion");
|
||||
assert!(
|
||||
matches!(
|
||||
store
|
||||
.put_object(&bucket, "exceeds", &mut PutObjReader::from_vec(vec![0x5a; 1]), &ObjectOptions::default())
|
||||
.await,
|
||||
Err(StorageError::QuotaExceeded { .. })
|
||||
),
|
||||
"a low companion cannot expand the ledger"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
crate::config::com::read_config(store.clone(), &path)
|
||||
.await
|
||||
.expect("retained scanner evidence"),
|
||||
bytes
|
||||
);
|
||||
decisions
|
||||
.push(serde_json::json!({"protocol": protocol, "primary_baseline": primary, "companion_size": companion_size}));
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
|
||||
.await
|
||||
.expect("clear fixture caches");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
stop_reporting_fixture(&store).await;
|
||||
let paths = dirs.iter().map(|dir| dir.path().to_path_buf()).collect::<Vec<_>>();
|
||||
drop(store);
|
||||
drop(dirs);
|
||||
assert!(paths.iter().all(|path| !path.exists()));
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-19", "assertion_id": "USAGE-19-A02", "status": "PASS",
|
||||
"evidence": {"validated_companion_fixtures": decisions, "legacy_primary_bytes": 1000, "durable_limit_bytes": 4096, "companion_only_legacy_admission": "UsageUnavailable"}
|
||||
})
|
||||
);
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-19", "status": "PASS", "fixtures_removed": true})
|
||||
);
|
||||
}
|
||||
@@ -139,6 +139,8 @@ const REPLAY_CACHE_AUTO_MEMORY_PERCENT: u64 = 13;
|
||||
const REPLAY_CACHE_METADATA_BYTES: u64 = 1024 * 1024;
|
||||
const REPLAY_CACHE_TABLE_OVERHEAD_BYTES: u64 = 512;
|
||||
const NS_SCANNER_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-capability-v3";
|
||||
const NS_SCANNER_COMPLETE_OBSERVATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-complete-observation-v1";
|
||||
const NS_SCANNER_DURABLE_PROGRESS_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-durable-progress-v1";
|
||||
const NS_SCANNER_CHECKPOINT_COVERAGE_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-checkpoint-coverage-v1";
|
||||
const NS_SCANNER_TIER_REGISTRY_GENERATION_AUTH_DOMAIN: &[u8] = b"rustfs-ns-scanner-tier-registry-generation-v1";
|
||||
pub const TONIC_RPC_PREFIX: &str = "/node_service.NodeService";
|
||||
@@ -795,6 +797,8 @@ fn update_ns_scanner_capability_mac(
|
||||
server_epoch: Uuid,
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
supports_durable_progress: bool,
|
||||
supports_complete_observation: bool,
|
||||
) {
|
||||
mac.update(NS_SCANNER_CAPABILITY_AUTH_DOMAIN);
|
||||
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
|
||||
@@ -809,14 +813,22 @@ fn update_ns_scanner_capability_mac(
|
||||
if supports_checkpoint_coverage {
|
||||
mac.update(NS_SCANNER_CHECKPOINT_COVERAGE_AUTH_DOMAIN);
|
||||
}
|
||||
if supports_durable_progress {
|
||||
mac.update(NS_SCANNER_DURABLE_PROGRESS_AUTH_DOMAIN);
|
||||
}
|
||||
if supports_complete_observation {
|
||||
mac.update(NS_SCANNER_COMPLETE_OBSERVATION_AUTH_DOMAIN);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_ns_scanner_capability_proof(
|
||||
fn generate_ns_scanner_capability_proof_with_observation(
|
||||
secret: &str,
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
supports_durable_progress: bool,
|
||||
supports_complete_observation: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
if challenge.is_nil() || server_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
|
||||
@@ -829,17 +841,22 @@ fn generate_ns_scanner_capability_proof(
|
||||
server_epoch,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
supports_durable_progress,
|
||||
supports_complete_observation,
|
||||
);
|
||||
Ok(mac.finalize().into_bytes().to_vec())
|
||||
}
|
||||
|
||||
fn verify_ns_scanner_capability_proof(
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn verify_ns_scanner_capability_proof_with_observation(
|
||||
secret: &str,
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
proof: &[u8],
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
supports_durable_progress: bool,
|
||||
supports_complete_observation: bool,
|
||||
) -> std::io::Result<()> {
|
||||
if challenge.is_nil() || server_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid namespace scanner capability scope"));
|
||||
@@ -852,11 +869,35 @@ fn verify_ns_scanner_capability_proof(
|
||||
server_epoch,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
supports_durable_progress,
|
||||
supports_complete_observation,
|
||||
);
|
||||
mac.verify_slice(proof)
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid namespace scanner capability proof"))
|
||||
}
|
||||
|
||||
fn generate_ns_scanner_capability_proof(
|
||||
secret: &str,
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
tier: bool,
|
||||
checkpoint: bool,
|
||||
progress: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
generate_ns_scanner_capability_proof_with_observation(secret, challenge, server_epoch, tier, checkpoint, progress, false)
|
||||
}
|
||||
fn verify_ns_scanner_capability_proof(
|
||||
secret: &str,
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
proof: &[u8],
|
||||
tier: bool,
|
||||
checkpoint: bool,
|
||||
progress: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_ns_scanner_capability_proof_with_observation(secret, challenge, server_epoch, proof, tier, checkpoint, progress, false)
|
||||
}
|
||||
|
||||
pub fn sign_ns_scanner_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
|
||||
sign_ns_scanner_capability_with_tier_registry_generation(challenge, server_epoch, false)
|
||||
}
|
||||
@@ -878,6 +919,22 @@ pub fn sign_ns_scanner_capability_with_checkpoint_coverage(
|
||||
server_epoch: Uuid,
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
sign_ns_scanner_capability_with_durable_progress(
|
||||
challenge,
|
||||
server_epoch,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn sign_ns_scanner_capability_with_durable_progress(
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
supports_durable_progress: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
generate_ns_scanner_capability_proof(
|
||||
&get_shared_secret()?,
|
||||
@@ -885,6 +942,7 @@ pub fn sign_ns_scanner_capability_with_checkpoint_coverage(
|
||||
server_epoch,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
supports_durable_progress,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -909,6 +967,24 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
|
||||
proof: &[u8],
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_ns_scanner_capability_with_durable_progress(
|
||||
challenge,
|
||||
server_epoch,
|
||||
proof,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
false,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn verify_ns_scanner_capability_with_durable_progress(
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
proof: &[u8],
|
||||
supports_tier_registry_generation: bool,
|
||||
supports_checkpoint_coverage: bool,
|
||||
supports_durable_progress: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_ns_scanner_capability_proof(
|
||||
&get_shared_secret()?,
|
||||
@@ -917,6 +993,74 @@ pub fn verify_ns_scanner_capability_with_checkpoint_coverage(
|
||||
proof,
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage,
|
||||
supports_durable_progress,
|
||||
)
|
||||
}
|
||||
|
||||
pub fn sign_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
tier: bool,
|
||||
checkpoint: bool,
|
||||
progress: bool,
|
||||
observation: bool,
|
||||
) -> std::io::Result<Vec<u8>> {
|
||||
generate_ns_scanner_capability_proof_with_observation(
|
||||
&get_shared_secret()?,
|
||||
challenge,
|
||||
server_epoch,
|
||||
tier,
|
||||
checkpoint,
|
||||
progress,
|
||||
observation,
|
||||
)
|
||||
}
|
||||
fn ns_scanner_usage_watch_capability_mac(secret: &str, challenge: Uuid, server_epoch: Uuid) -> std::io::Result<HmacSha256> {
|
||||
if challenge.is_nil() || server_epoch.is_nil() {
|
||||
return Err(std::io::Error::other("Invalid namespace scanner usage watch capability scope"));
|
||||
}
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
mac.update(b"rustfs-ns-scanner-usage-watch-capability-v1\0");
|
||||
mac.update(&NS_SCANNER_PROTOCOL_VERSION.to_be_bytes());
|
||||
mac.update(challenge.as_bytes());
|
||||
mac.update(server_epoch.as_bytes());
|
||||
Ok(mac)
|
||||
}
|
||||
|
||||
/// Sign the independent worker capability to validate bucket watches before
|
||||
/// reusing or durably publishing a complete scanner cache.
|
||||
pub fn sign_ns_scanner_usage_watch_capability(challenge: Uuid, server_epoch: Uuid) -> std::io::Result<Vec<u8>> {
|
||||
Ok(ns_scanner_usage_watch_capability_mac(&get_shared_secret()?, challenge, server_epoch)?
|
||||
.finalize()
|
||||
.into_bytes()
|
||||
.to_vec())
|
||||
}
|
||||
|
||||
pub fn verify_ns_scanner_usage_watch_capability(challenge: Uuid, server_epoch: Uuid, proof: &[u8]) -> std::io::Result<()> {
|
||||
ns_scanner_usage_watch_capability_mac(&get_shared_secret()?, challenge, server_epoch)?
|
||||
.verify_slice(proof)
|
||||
.map_err(|_| std::io::Error::other("Invalid namespace scanner usage watch capability proof"))
|
||||
}
|
||||
|
||||
pub fn verify_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge: Uuid,
|
||||
server_epoch: Uuid,
|
||||
proof: &[u8],
|
||||
tier: bool,
|
||||
checkpoint: bool,
|
||||
progress: bool,
|
||||
observation: bool,
|
||||
) -> std::io::Result<()> {
|
||||
verify_ns_scanner_capability_proof_with_observation(
|
||||
&get_shared_secret()?,
|
||||
challenge,
|
||||
server_epoch,
|
||||
proof,
|
||||
tier,
|
||||
checkpoint,
|
||||
progress,
|
||||
observation,
|
||||
)
|
||||
}
|
||||
|
||||
@@ -2135,13 +2279,119 @@ mod tests {
|
||||
let secret = "test-scanner-capability-secret";
|
||||
let challenge = Uuid::new_v4();
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false)
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false, false)
|
||||
.expect("capability proof should be generated");
|
||||
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false, false).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, false, false, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, Uuid::new_v4(), &proof, false, false, false).is_err());
|
||||
assert!(
|
||||
verify_ns_scanner_capability_proof("different-secret", challenge, server_epoch, &proof, false, false, false).is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_watch_capability_is_independent_and_rejects_replay() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("usage-watch-capability-test-secret".into());
|
||||
let challenge = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
let proof = sign_ns_scanner_usage_watch_capability(challenge, epoch).expect("watch capability");
|
||||
verify_ns_scanner_usage_watch_capability(challenge, epoch, &proof).expect("exact watch capability scope");
|
||||
assert!(verify_ns_scanner_usage_watch_capability(Uuid::new_v4(), epoch, &proof).is_err());
|
||||
assert!(verify_ns_scanner_usage_watch_capability(challenge, Uuid::new_v4(), &proof).is_err());
|
||||
for progress in [false, true] {
|
||||
for observation in [false, true] {
|
||||
let other =
|
||||
sign_ns_scanner_capability_with_usage_capabilities(challenge, epoch, true, true, progress, observation)
|
||||
.expect("independent older capabilities");
|
||||
assert!(verify_ns_scanner_usage_watch_capability(challenge, epoch, &other).is_err());
|
||||
assert!(
|
||||
verify_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge,
|
||||
epoch,
|
||||
&proof,
|
||||
true,
|
||||
true,
|
||||
progress,
|
||||
observation
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_observation_and_progress_scopes_are_independent() {
|
||||
let challenge = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
let secret = "independent-usage-capabilities";
|
||||
for progress in [false, true] {
|
||||
for observation in [false, true] {
|
||||
let proof = generate_ns_scanner_capability_proof_with_observation(
|
||||
secret,
|
||||
challenge,
|
||||
epoch,
|
||||
true,
|
||||
true,
|
||||
progress,
|
||||
observation,
|
||||
)
|
||||
.expect("proof");
|
||||
assert!(
|
||||
verify_ns_scanner_capability_proof_with_observation(
|
||||
secret,
|
||||
challenge,
|
||||
epoch,
|
||||
&proof,
|
||||
true,
|
||||
true,
|
||||
progress,
|
||||
observation
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
verify_ns_scanner_capability_proof_with_observation(
|
||||
secret,
|
||||
challenge,
|
||||
epoch,
|
||||
&proof,
|
||||
true,
|
||||
true,
|
||||
!progress,
|
||||
observation
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
verify_ns_scanner_capability_proof_with_observation(
|
||||
secret,
|
||||
challenge,
|
||||
epoch,
|
||||
&proof,
|
||||
true,
|
||||
true,
|
||||
progress,
|
||||
!observation
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn namespace_scanner_capability_proof_binds_durable_progress_independently() {
|
||||
let challenge = Uuid::new_v4();
|
||||
let epoch = Uuid::new_v4();
|
||||
let secret = "durable-progress-test";
|
||||
for supported in [false, true] {
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, epoch, true, true, supported).expect("proof");
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, epoch, &proof, true, true, supported).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, epoch, &proof, true, true, !supported).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), epoch, &proof, true, true, supported).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2149,16 +2399,16 @@ mod tests {
|
||||
let secret = "checkpoint-capability-test-secret";
|
||||
let challenge = Uuid::new_v4();
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, true)
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, true, false)
|
||||
.expect("checkpoint capability proof");
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, true).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, true).is_err());
|
||||
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false)
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, true, false).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, true, false).is_err());
|
||||
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false, false)
|
||||
.expect("legacy generation capability proof");
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, true).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &[], true, true).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, true, true).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, true, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &[], true, true, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, Uuid::new_v4(), server_epoch, &proof, true, true, false).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2166,14 +2416,14 @@ mod tests {
|
||||
let secret = "test-scanner-capability-secret";
|
||||
let challenge = Uuid::new_v4();
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false)
|
||||
let proof = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, true, false, false)
|
||||
.expect("generation capability proof should be generated");
|
||||
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false).is_err());
|
||||
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false)
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, true, false, false).is_ok());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &proof, false, false, false).is_err());
|
||||
let legacy = generate_ns_scanner_capability_proof(secret, challenge, server_epoch, false, false, false)
|
||||
.expect("legacy capability proof should be generated");
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, false).is_err());
|
||||
assert!(verify_ns_scanner_capability_proof(secret, challenge, server_epoch, &legacy, true, false, false).is_err());
|
||||
}
|
||||
|
||||
/// Security regression for GHSA-r5qv-rc46-hv8q (internode RPC fail-closed,
|
||||
|
||||
@@ -14,14 +14,16 @@
|
||||
|
||||
use crate::cluster::rpc::{
|
||||
build_auth_headers, build_put_file_auth_trailer, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_capability,
|
||||
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_ns_scanner_capability_with_usage_capabilities, verify_put_file_capability,
|
||||
};
|
||||
use crate::disk::error::{Error, Result};
|
||||
use crate::disk::{FileReader, FileWriter};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CHECKPOINT_COVERAGE_QUERY,
|
||||
NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
|
||||
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY,
|
||||
NS_SCANNER_COMPLETE_OBSERVATION_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_DURABLE_PROGRESS_QUERY,
|
||||
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY, NS_SCANNER_REQUEST_ID_QUERY,
|
||||
NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY, NS_SCANNER_SESSION_SEQUENCE_QUERY,
|
||||
NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_V1,
|
||||
PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY,
|
||||
PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
|
||||
@@ -267,6 +269,9 @@ pub struct NsScannerCapabilityRequest {
|
||||
pub endpoint: String,
|
||||
pub supports_tier_registry_generation: bool,
|
||||
pub supports_checkpoint_coverage: bool,
|
||||
pub supports_durable_progress: bool,
|
||||
pub supports_complete_observation: bool,
|
||||
pub supports_usage_watch: bool,
|
||||
}
|
||||
|
||||
/// Data-plane stream opener used by `RemoteDisk`.
|
||||
@@ -307,6 +312,9 @@ pub trait InternodeDataTransport: Send + Sync + std::fmt::Debug {
|
||||
proof: Vec::new(),
|
||||
supports_tier_registry_generation: None,
|
||||
supports_checkpoint_coverage: None,
|
||||
supports_durable_progress: None,
|
||||
supports_complete_observation: None,
|
||||
usage_watch_proof: None,
|
||||
})
|
||||
}
|
||||
// Interface facet nobody calls yet: every transport implements both, but no
|
||||
@@ -401,11 +409,53 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
}
|
||||
|
||||
async fn probe_ns_scanner_capability(&self, request: NsScannerCapabilityRequest) -> Result<NsScannerCapabilityResponse> {
|
||||
let request = if request.supports_usage_watch {
|
||||
match self.probe_ns_scanner_capability_once(&request).await {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
|
||||
supports_usage_watch: false,
|
||||
..request
|
||||
},
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let request = if request.supports_complete_observation {
|
||||
match self.probe_ns_scanner_capability_once(&request).await {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
..request
|
||||
},
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let request = if request.supports_durable_progress {
|
||||
match self.probe_ns_scanner_capability_once(&request).await {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
..request
|
||||
},
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
} else {
|
||||
request
|
||||
};
|
||||
let request = if request.supports_checkpoint_coverage {
|
||||
match self.probe_ns_scanner_capability_once(&request).await {
|
||||
Ok(response) => return Ok(response),
|
||||
Err(error) if ns_scanner_capability_error_allows_legacy(&error) => NsScannerCapabilityRequest {
|
||||
supports_checkpoint_coverage: false,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
..request
|
||||
},
|
||||
Err(error) => return Err(error),
|
||||
@@ -425,6 +475,9 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
endpoint: request.endpoint.clone(),
|
||||
supports_tier_registry_generation: false,
|
||||
supports_checkpoint_coverage: false,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
};
|
||||
match self.probe_ns_scanner_capability_once(&legacy_request).await {
|
||||
Ok(mut response) => {
|
||||
@@ -482,13 +535,62 @@ impl TcpHttpInternodeDataTransport {
|
||||
if response.version != NS_SCANNER_PROTOCOL_VERSION || response.server_epoch.is_nil() {
|
||||
return Err(Error::other("incompatible remote namespace scanner capability response"));
|
||||
}
|
||||
if let Err(err) = verify_ns_scanner_capability_with_checkpoint_coverage(
|
||||
if request.supports_usage_watch {
|
||||
if let Some(proof) = response.usage_watch_proof.as_deref() {
|
||||
crate::cluster::rpc::verify_ns_scanner_usage_watch_capability(challenge, response.server_epoch, proof)
|
||||
.map_err(|_| Error::other("remote namespace scanner usage watch capability authentication failed"))?;
|
||||
}
|
||||
} else {
|
||||
response.usage_watch_proof = None;
|
||||
}
|
||||
if let Err(err) = verify_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge,
|
||||
response.server_epoch,
|
||||
&response.proof,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
request.supports_durable_progress,
|
||||
request.supports_complete_observation,
|
||||
) {
|
||||
if request.supports_complete_observation
|
||||
&& verify_ns_scanner_capability_with_durable_progress(
|
||||
challenge,
|
||||
response.server_epoch,
|
||||
&response.proof,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
request.supports_durable_progress,
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
normalize_ns_scanner_capability_response(
|
||||
&mut response,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
request.supports_durable_progress,
|
||||
false,
|
||||
);
|
||||
return Ok(response);
|
||||
}
|
||||
if request.supports_durable_progress
|
||||
&& verify_ns_scanner_capability_with_checkpoint_coverage(
|
||||
challenge,
|
||||
response.server_epoch,
|
||||
&response.proof,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
normalize_ns_scanner_capability_response(
|
||||
&mut response,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
return Ok(response);
|
||||
}
|
||||
if request.supports_checkpoint_coverage
|
||||
&& verify_ns_scanner_capability_with_tier_registry_generation(
|
||||
challenge,
|
||||
@@ -498,7 +600,13 @@ impl TcpHttpInternodeDataTransport {
|
||||
)
|
||||
.is_ok()
|
||||
{
|
||||
normalize_ns_scanner_capability_response(&mut response, request.supports_tier_registry_generation, false);
|
||||
normalize_ns_scanner_capability_response(
|
||||
&mut response,
|
||||
request.supports_tier_registry_generation,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
);
|
||||
return Ok(response);
|
||||
}
|
||||
// A permissive older peer can ignore the additive marker and
|
||||
@@ -508,6 +616,8 @@ impl TcpHttpInternodeDataTransport {
|
||||
if request.supports_tier_registry_generation && ns_scanner_capability_legacy_proof_is_valid(challenge, &response) {
|
||||
response.supports_tier_registry_generation = None;
|
||||
response.supports_checkpoint_coverage = None;
|
||||
response.supports_durable_progress = None;
|
||||
response.supports_complete_observation = None;
|
||||
return Ok(response);
|
||||
}
|
||||
return Err(Error::other(format!("remote namespace scanner capability authentication failed: {err}")));
|
||||
@@ -521,6 +631,8 @@ impl TcpHttpInternodeDataTransport {
|
||||
&mut response,
|
||||
request.supports_tier_registry_generation,
|
||||
request.supports_checkpoint_coverage,
|
||||
request.supports_durable_progress,
|
||||
request.supports_complete_observation,
|
||||
);
|
||||
Ok(response)
|
||||
}
|
||||
@@ -855,9 +967,13 @@ fn normalize_ns_scanner_capability_response(
|
||||
response: &mut NsScannerCapabilityResponse,
|
||||
requested_generation_support: bool,
|
||||
requested_checkpoint_coverage: bool,
|
||||
requested_durable_progress: bool,
|
||||
requested_complete_observation: bool,
|
||||
) {
|
||||
response.supports_tier_registry_generation = requested_generation_support.then_some(true);
|
||||
response.supports_checkpoint_coverage = requested_checkpoint_coverage.then_some(true);
|
||||
response.supports_durable_progress = requested_durable_progress.then_some(true);
|
||||
response.supports_complete_observation = requested_complete_observation.then_some(true);
|
||||
}
|
||||
|
||||
fn ns_scanner_capability_legacy_proof_is_valid(challenge: Uuid, response: &NsScannerCapabilityResponse) -> bool {
|
||||
@@ -890,7 +1006,7 @@ fn build_ns_scanner_url(request: &NsScannerStreamRequest) -> String {
|
||||
|
||||
fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challenge: Uuid) -> String {
|
||||
format!(
|
||||
"{}{}?{}={}&{}={}{}{}",
|
||||
"{}{}?{}={}&{}={}{}{}{}{}{}",
|
||||
request.endpoint,
|
||||
NS_SCANNER_PATH,
|
||||
NS_SCANNER_PROTOCOL_VERSION_QUERY,
|
||||
@@ -906,6 +1022,21 @@ fn build_ns_scanner_capability_url(request: &NsScannerCapabilityRequest, challen
|
||||
format!("&{}=true", NS_SCANNER_CHECKPOINT_COVERAGE_QUERY)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if request.supports_durable_progress {
|
||||
format!("&{}=true", NS_SCANNER_DURABLE_PROGRESS_QUERY)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if request.supports_complete_observation {
|
||||
format!("&{}=true", NS_SCANNER_COMPLETE_OBSERVATION_QUERY)
|
||||
} else {
|
||||
String::new()
|
||||
},
|
||||
if request.supports_usage_watch {
|
||||
"&ns_scanner_usage_watch=true".to_string()
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
)
|
||||
}
|
||||
@@ -1252,6 +1383,9 @@ mod tests {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
supports_tier_registry_generation: false,
|
||||
supports_checkpoint_coverage: false,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
})
|
||||
.await
|
||||
.expect_err("legacy transport should report namespace scanner as unsupported");
|
||||
@@ -1278,6 +1412,106 @@ mod tests {
|
||||
assert!(matches!(open_err, Error::MethodNotAllowed));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_scanner_watch_capability_preserves_older_capabilities_and_rejects_stale_proof() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
for peer in ["current", "strict-old", "missing-watch", "stale-watch"] {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("watch capability listener");
|
||||
let address = listener.local_addr().expect("listener address");
|
||||
let epoch = Uuid::new_v4();
|
||||
let server = tokio::spawn(async move {
|
||||
let requests = if peer == "strict-old" { 2 } else { 1 };
|
||||
for index in 0..requests {
|
||||
let (mut socket, _) = listener.accept().await.expect("watch capability request");
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let mut chunk = [0; 1024];
|
||||
let read = socket.read(&mut chunk).await.expect("request headers");
|
||||
assert!(read > 0 && bytes.len() < 16 * 1024);
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
let headers = std::str::from_utf8(&bytes).expect("HTTP headers");
|
||||
let target = headers.split_whitespace().nth(1).expect("target");
|
||||
let url = reqwest::Url::parse(&format!("http://{address}{target}")).expect("request URL");
|
||||
let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
|
||||
assert_eq!(query.contains_key("ns_scanner_usage_watch"), index == 0);
|
||||
assert!(query.contains_key(NS_SCANNER_DURABLE_PROGRESS_QUERY));
|
||||
assert!(query.contains_key(NS_SCANNER_COMPLETE_OBSERVATION_QUERY));
|
||||
if peer == "strict-old" && index == 0 {
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("old parser rejects only the new field");
|
||||
continue;
|
||||
}
|
||||
let challenge = query[NS_SCANNER_CAPABILITY_CHALLENGE_QUERY].parse().expect("challenge");
|
||||
let proof = crate::cluster::rpc::sign_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge, epoch, true, true, true, true,
|
||||
)
|
||||
.expect("prior capability proof");
|
||||
let usage_watch_proof = match peer {
|
||||
"current" => Some(
|
||||
crate::cluster::rpc::sign_ns_scanner_usage_watch_capability(challenge, epoch).expect("watch proof"),
|
||||
),
|
||||
"stale-watch" => Some(
|
||||
crate::cluster::rpc::sign_ns_scanner_usage_watch_capability(Uuid::new_v4(), epoch)
|
||||
.expect("stale watch proof"),
|
||||
),
|
||||
_ => None,
|
||||
};
|
||||
let body = rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
|
||||
version: NS_SCANNER_PROTOCOL_VERSION,
|
||||
server_epoch: epoch,
|
||||
proof,
|
||||
supports_tier_registry_generation: Some(true),
|
||||
supports_checkpoint_coverage: Some(true),
|
||||
supports_durable_progress: Some(true),
|
||||
supports_complete_observation: Some(true),
|
||||
usage_watch_proof,
|
||||
})
|
||||
.expect("capability body");
|
||||
socket
|
||||
.write_all(
|
||||
format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", body.len()).as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("response headers");
|
||||
socket.write_all(&body).await.expect("response body");
|
||||
}
|
||||
requests
|
||||
});
|
||||
let response = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
TcpHttpInternodeDataTransport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
|
||||
endpoint: format!("http://{address}"),
|
||||
supports_tier_registry_generation: true,
|
||||
supports_checkpoint_coverage: true,
|
||||
supports_durable_progress: true,
|
||||
supports_complete_observation: true,
|
||||
supports_usage_watch: true,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("bounded probe");
|
||||
if peer == "stale-watch" {
|
||||
assert!(
|
||||
response
|
||||
.expect_err("stale response must fail authentication")
|
||||
.to_string()
|
||||
.contains("authentication failed")
|
||||
);
|
||||
} else {
|
||||
let response = response.expect("verified capability result");
|
||||
assert_eq!(response.usage_watch_proof.is_some(), peer == "current");
|
||||
assert_eq!(response.supports_durable_progress, Some(true));
|
||||
assert_eq!(response.supports_complete_observation, Some(true));
|
||||
}
|
||||
assert_eq!(server.await.expect("capability server"), if peer == "strict-old" { 2 } else { 1 });
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_scanner_checkpoint_capability_authenticates_new_and_legacy_peers() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
@@ -1330,6 +1564,9 @@ mod tests {
|
||||
supports_tier_registry_generation: Some(true),
|
||||
// Strip a supported bit, or forge it for an older peer.
|
||||
supports_checkpoint_coverage: (peer != "current").then_some(true),
|
||||
supports_durable_progress: None,
|
||||
supports_complete_observation: None,
|
||||
usage_watch_proof: None,
|
||||
})
|
||||
.expect("capability response");
|
||||
socket
|
||||
@@ -1348,6 +1585,9 @@ mod tests {
|
||||
endpoint: format!("http://{address}"),
|
||||
supports_tier_registry_generation: true,
|
||||
supports_checkpoint_coverage: true,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
@@ -1369,6 +1609,206 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_scanner_durable_progress_capability_authenticates_new_and_legacy_peers() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
for peer in ["current", "strict-old", "permissive-old", "invalid-proof", "missing-proof"] {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("capability listener");
|
||||
let address = listener.local_addr().expect("capability listener address");
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let server = tokio::spawn(async move {
|
||||
let requests = if peer == "strict-old" { 2 } else { 1 };
|
||||
for index in 0..requests {
|
||||
let (mut socket, _) = listener.accept().await.expect("capability request");
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let mut chunk = [0; 1024];
|
||||
let read = socket.read(&mut chunk).await.expect("capability headers");
|
||||
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete capability headers");
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
let request = std::str::from_utf8(&bytes).expect("HTTP headers");
|
||||
let target = request.split_whitespace().nth(1).expect("HTTP request target");
|
||||
let url = reqwest::Url::parse(&format!("http://{address}{target}")).expect("capability URL");
|
||||
let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
|
||||
assert_eq!(query.get(NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY).map(String::as_str), Some("true"));
|
||||
assert_eq!(query.contains_key(NS_SCANNER_DURABLE_PROGRESS_QUERY), index == 0);
|
||||
if peer == "strict-old" && index == 0 {
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("legacy parser rejection");
|
||||
continue;
|
||||
}
|
||||
let challenge = query[NS_SCANNER_CAPABILITY_CHALLENGE_QUERY].parse().expect("challenge UUID");
|
||||
let proof = match peer {
|
||||
"invalid-proof" => vec![0; 32],
|
||||
"missing-proof" => Vec::new(),
|
||||
_ => crate::cluster::rpc::sign_ns_scanner_capability_with_durable_progress(
|
||||
challenge,
|
||||
server_epoch,
|
||||
true,
|
||||
true,
|
||||
peer == "current",
|
||||
)
|
||||
.expect("server capability proof"),
|
||||
};
|
||||
let body = rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
|
||||
version: NS_SCANNER_PROTOCOL_VERSION,
|
||||
server_epoch,
|
||||
proof,
|
||||
supports_tier_registry_generation: Some(true),
|
||||
// Strip a supported bit, or forge it for an older peer.
|
||||
supports_checkpoint_coverage: Some(true),
|
||||
supports_durable_progress: (peer != "current").then_some(true),
|
||||
supports_complete_observation: None,
|
||||
usage_watch_proof: None,
|
||||
})
|
||||
.expect("capability response");
|
||||
socket
|
||||
.write_all(
|
||||
format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", body.len()).as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("capability response headers");
|
||||
socket.write_all(&body).await.expect("capability response body");
|
||||
}
|
||||
requests
|
||||
});
|
||||
let response = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
TcpHttpInternodeDataTransport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
|
||||
endpoint: format!("http://{address}"),
|
||||
supports_tier_registry_generation: true,
|
||||
supports_checkpoint_coverage: true,
|
||||
supports_durable_progress: true,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("bounded capability probe");
|
||||
if peer.ends_with("proof") {
|
||||
assert!(
|
||||
response
|
||||
.expect_err("unverified capability must fail closed")
|
||||
.to_string()
|
||||
.contains("authentication failed")
|
||||
);
|
||||
} else {
|
||||
let response = response.expect("authenticated capability");
|
||||
assert_eq!(response.server_epoch, server_epoch);
|
||||
assert_eq!(response.supports_tier_registry_generation, Some(true));
|
||||
assert_eq!(response.supports_checkpoint_coverage, Some(true));
|
||||
assert_eq!(response.supports_durable_progress, (peer == "current").then_some(true));
|
||||
}
|
||||
assert_eq!(server.await.expect("capability server"), if peer == "strict-old" { 2 } else { 1 });
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_scanner_complete_observation_capability_authenticates_new_and_legacy_peers() {
|
||||
crate::runtime::sources::ensure_test_rpc_secret();
|
||||
for peer in ["current", "strict-old", "permissive-old", "invalid-proof", "missing-proof"] {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("capability listener");
|
||||
let address = listener.local_addr().expect("capability listener address");
|
||||
let server_epoch = Uuid::new_v4();
|
||||
let server = tokio::spawn(async move {
|
||||
let requests = if peer == "strict-old" { 2 } else { 1 };
|
||||
for index in 0..requests {
|
||||
let (mut socket, _) = listener.accept().await.expect("capability request");
|
||||
let mut bytes = Vec::new();
|
||||
while !bytes.windows(4).any(|window| window == b"\r\n\r\n") {
|
||||
let mut chunk = [0; 1024];
|
||||
let read = socket.read(&mut chunk).await.expect("capability headers");
|
||||
assert!(read > 0 && bytes.len() < 16 * 1024, "bounded complete capability headers");
|
||||
bytes.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
let request = std::str::from_utf8(&bytes).expect("HTTP headers");
|
||||
let target = request.split_whitespace().nth(1).expect("HTTP request target");
|
||||
let url = reqwest::Url::parse(&format!("http://{address}{target}")).expect("capability URL");
|
||||
let query: HashMap<_, _> = url.query_pairs().into_owned().collect();
|
||||
assert_eq!(query.get(NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY).map(String::as_str), Some("true"));
|
||||
assert_eq!(query.contains_key(NS_SCANNER_COMPLETE_OBSERVATION_QUERY), index == 0);
|
||||
if peer == "strict-old" && index == 0 {
|
||||
socket
|
||||
.write_all(b"HTTP/1.1 400 Bad Request\r\ncontent-length: 0\r\nconnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("legacy parser rejection");
|
||||
continue;
|
||||
}
|
||||
let challenge = query[NS_SCANNER_CAPABILITY_CHALLENGE_QUERY].parse().expect("challenge UUID");
|
||||
let proof = match peer {
|
||||
"invalid-proof" => vec![0; 32],
|
||||
"missing-proof" => Vec::new(),
|
||||
_ => crate::cluster::rpc::sign_ns_scanner_capability_with_usage_capabilities(
|
||||
challenge,
|
||||
server_epoch,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
peer == "current",
|
||||
)
|
||||
.expect("server capability proof"),
|
||||
};
|
||||
let body = rmp_serde::to_vec_named(&NsScannerCapabilityResponse {
|
||||
version: NS_SCANNER_PROTOCOL_VERSION,
|
||||
server_epoch,
|
||||
proof,
|
||||
supports_tier_registry_generation: Some(true),
|
||||
// Strip a supported bit, or forge it for an older peer.
|
||||
supports_checkpoint_coverage: Some(true),
|
||||
supports_durable_progress: Some(true),
|
||||
supports_complete_observation: (peer != "current").then_some(true),
|
||||
usage_watch_proof: None,
|
||||
})
|
||||
.expect("capability response");
|
||||
socket
|
||||
.write_all(
|
||||
format!("HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n", body.len()).as_bytes(),
|
||||
)
|
||||
.await
|
||||
.expect("capability response headers");
|
||||
socket.write_all(&body).await.expect("capability response body");
|
||||
}
|
||||
requests
|
||||
});
|
||||
let response = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
TcpHttpInternodeDataTransport.probe_ns_scanner_capability(NsScannerCapabilityRequest {
|
||||
endpoint: format!("http://{address}"),
|
||||
supports_tier_registry_generation: true,
|
||||
supports_checkpoint_coverage: true,
|
||||
supports_durable_progress: true,
|
||||
supports_complete_observation: true,
|
||||
supports_usage_watch: false,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("bounded capability probe");
|
||||
if peer.ends_with("proof") {
|
||||
assert!(
|
||||
response
|
||||
.expect_err("unverified capability must fail closed")
|
||||
.to_string()
|
||||
.contains("authentication failed")
|
||||
);
|
||||
} else {
|
||||
let response = response.expect("authenticated capability");
|
||||
assert_eq!(response.server_epoch, server_epoch);
|
||||
assert_eq!(response.supports_tier_registry_generation, Some(true));
|
||||
assert_eq!(response.supports_checkpoint_coverage, Some(true));
|
||||
assert_eq!(response.supports_durable_progress, Some(true));
|
||||
assert_eq!(response.supports_complete_observation, (peer == "current").then_some(true));
|
||||
}
|
||||
assert_eq!(server.await.expect("capability server"), if peer == "strict-old" { 2 } else { 1 });
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn walk_dir_restores_explicit_remote_missing_error() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("walk-dir-error-test-secret".to_string());
|
||||
@@ -2112,6 +2552,9 @@ mod tests {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
supports_tier_registry_generation: false,
|
||||
supports_checkpoint_coverage: false,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
},
|
||||
challenge,
|
||||
);
|
||||
@@ -2132,6 +2575,9 @@ mod tests {
|
||||
endpoint: "http://node1:9000".to_string(),
|
||||
supports_tier_registry_generation: true,
|
||||
supports_checkpoint_coverage: false,
|
||||
supports_durable_progress: false,
|
||||
supports_complete_observation: false,
|
||||
supports_usage_watch: false,
|
||||
},
|
||||
challenge,
|
||||
);
|
||||
@@ -2171,13 +2617,16 @@ mod tests {
|
||||
proof: Vec::new(),
|
||||
supports_tier_registry_generation: None,
|
||||
supports_checkpoint_coverage: None,
|
||||
supports_durable_progress: None,
|
||||
supports_complete_observation: None,
|
||||
usage_watch_proof: None,
|
||||
};
|
||||
|
||||
normalize_ns_scanner_capability_response(&mut response, true, true);
|
||||
normalize_ns_scanner_capability_response(&mut response, true, true, false, false);
|
||||
assert_eq!(response.supports_tier_registry_generation, Some(true));
|
||||
|
||||
response.supports_tier_registry_generation = Some(false);
|
||||
normalize_ns_scanner_capability_response(&mut response, false, false);
|
||||
normalize_ns_scanner_capability_response(&mut response, false, false, false, false);
|
||||
assert_eq!(response.supports_tier_registry_generation, None);
|
||||
}
|
||||
|
||||
@@ -2192,6 +2641,9 @@ mod tests {
|
||||
.expect("placeholder proof should be generated"),
|
||||
supports_tier_registry_generation: None,
|
||||
supports_checkpoint_coverage: None,
|
||||
supports_durable_progress: None,
|
||||
supports_complete_observation: None,
|
||||
usage_watch_proof: None,
|
||||
};
|
||||
// A proof bound to a different challenge cannot authorize the legacy
|
||||
// fallback, even though the response has the expected shape.
|
||||
|
||||
@@ -40,12 +40,15 @@ pub use http_auth::{
|
||||
gen_tonic_signature_headers, normalize_tonic_rpc_audience, rpc_auth_is_resource_exhausted, rpc_background_admission,
|
||||
set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, set_tonic_rolling_canonical_body_digest,
|
||||
set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
|
||||
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
|
||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
sign_ns_scanner_capability_with_durable_progress, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_ns_scanner_capability_with_usage_capabilities, sign_ns_scanner_usage_watch_capability, sign_put_file_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_ns_scanner_capability_with_usage_capabilities, verify_ns_scanner_usage_watch_capability, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -97,6 +97,7 @@ const PEER_REST_RECOVERY_MAX_ATTEMPTS: u32 = 60;
|
||||
const PEER_REST_RECOVERY_MAX_BACKOFF: Duration = Duration::from_secs(30);
|
||||
const SCANNER_ACTIVITY_MAX_MESSAGE_SIZE: usize = 1024;
|
||||
const SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const SCANNER_USAGE_WATCH_PEER_CONCURRENCY: usize = 4;
|
||||
/// Reserve time for the acquire response's network/clock uncertainty. The
|
||||
/// server owns the real expiry; this local deadline is intentionally earlier
|
||||
/// so a coordinator never starts a bounded persistence operation at the edge
|
||||
@@ -298,6 +299,22 @@ fn scanner_scoped_dirty_usage_ack_payload(
|
||||
Ok(payload)
|
||||
}
|
||||
|
||||
/// Authentication binds a response to the complete current request before any
|
||||
/// returned watch can be consumed. Semantic validation still rejects a signed
|
||||
/// response for the wrong producer session, bucket, or token.
|
||||
fn validate_scanner_usage_watch_response(
|
||||
request: &rustfs_protos::proto_gen::node_service::ScannerUsageWatchRequest,
|
||||
response: &rustfs_protos::proto_gen::node_service::ScannerUsageWatchResponse,
|
||||
) -> Result<()> {
|
||||
use rustfs_protos::usage_watch::{
|
||||
canonical_usage_watch_request, canonical_usage_watch_response, validate_usage_watch_response,
|
||||
};
|
||||
let canonical = canonical_usage_watch_request(request).map_err(|error| Error::other(error.to_string()))?;
|
||||
let body = canonical_usage_watch_response(&canonical, response).map_err(|error| Error::other(error.to_string()))?;
|
||||
verify_tonic_rpc_response_proof(&body, response.response_proof.as_ref())?;
|
||||
validate_usage_watch_response(request, response).map_err(|error| Error::other(error.to_string()))
|
||||
}
|
||||
|
||||
fn scanner_scoped_dirty_usage_ack_response_matches(
|
||||
request: &ScannerScopedDirtyUsageAckRequest,
|
||||
response: &ScannerScopedDirtyUsageAckResponse,
|
||||
@@ -741,6 +758,9 @@ pub struct PeerRestClient {
|
||||
topology_member: String,
|
||||
offline: Arc<AtomicBool>,
|
||||
recovery_running: Arc<AtomicBool>,
|
||||
/// All clones used by admission, keepalive and root verification share
|
||||
/// this peer's bounded sender capacity.
|
||||
scanner_usage_watch_admission: Arc<tokio::sync::Semaphore>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -900,9 +920,20 @@ impl PeerRestClient {
|
||||
topology_member,
|
||||
offline: Arc::new(AtomicBool::new(false)),
|
||||
recovery_running: Arc::new(AtomicBool::new(false)),
|
||||
scanner_usage_watch_admission: Arc::new(tokio::sync::Semaphore::new(SCANNER_USAGE_WATCH_PEER_CONCURRENCY)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn acquire_usage_watch_send_permit(&self) -> Result<tokio::sync::OwnedSemaphorePermit> {
|
||||
timeout(
|
||||
SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT,
|
||||
self.scanner_usage_watch_admission.clone().acquire_owned(),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| Error::other("usage watch sender admission timed out"))?
|
||||
.map_err(|_| Error::other("usage watch sender admission is closed"))
|
||||
}
|
||||
|
||||
fn parse_topology_host(peer_host_port: &str, grid_host: &str) -> Result<XHost> {
|
||||
let url = url::Url::parse(grid_host).map_err(|_| Error::other("peer grid host is not a valid URL"))?;
|
||||
if !matches!(url.scheme(), "http" | "https")
|
||||
@@ -1639,6 +1670,51 @@ impl PeerRestClient {
|
||||
Err(Error::NotImplemented)
|
||||
}
|
||||
|
||||
/// Routes one authorized MRF operation without retrying ambiguous mutations.
|
||||
/// `None` means the selected node predates this protocol; it is never an empty result.
|
||||
pub async fn mrf_lifecycle(
|
||||
&self,
|
||||
topology: String,
|
||||
node_id: String,
|
||||
command: Vec<u8>,
|
||||
timeout: std::time::Duration,
|
||||
) -> Result<Option<Vec<u8>>> {
|
||||
use rustfs_protos::proto_gen::node_service::MrfLifecycleRequest;
|
||||
use rustfs_protos::{MRF_LIFECYCLE_MAX_COMMAND_BYTES, MRF_LIFECYCLE_MAX_MESSAGE_BYTES, MRF_LIFECYCLE_PROTOCOL_VERSION};
|
||||
if command.len() > MRF_LIFECYCLE_MAX_COMMAND_BYTES {
|
||||
return Err(Error::other("MRF command exceeds size limit"));
|
||||
}
|
||||
let payload = MrfLifecycleRequest {
|
||||
protocol_version: MRF_LIFECYCLE_PROTOCOL_VERSION,
|
||||
topology_fingerprint: topology,
|
||||
node_id,
|
||||
command: command.into(),
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
};
|
||||
let mut request = Request::new(payload.clone());
|
||||
request.set_timeout(timeout);
|
||||
set_tonic_mutation_body_digest(&mut request)?;
|
||||
let mut client = self
|
||||
.get_client()
|
||||
.await?
|
||||
.max_decoding_message_size(MRF_LIFECYCLE_MAX_MESSAGE_BYTES)
|
||||
.max_encoding_message_size(MRF_LIFECYCLE_MAX_MESSAGE_BYTES);
|
||||
let response = match client.mrf_lifecycle(request).await {
|
||||
Ok(response) => response.into_inner(),
|
||||
// RUSTFS_COMPAT_TODO(backlog-3177): old peers lack MRF management. Remove after every supported peer implements this RPC.
|
||||
Err(status) if status.code() == tonic::Code::Unimplemented => return Ok(None),
|
||||
Err(status) => return Err(status.into()),
|
||||
};
|
||||
let canonical = rustfs_protos::canonical_mrf_lifecycle_response_body(&payload, &response)
|
||||
.map_err(|_| Error::other("MRF response is too large to authenticate"))?;
|
||||
verify_tonic_rpc_response_proof(&canonical, &response.response_proof)
|
||||
.map_err(|_| Error::other("MRF response authentication failed"))?;
|
||||
if response.protocol_version != MRF_LIFECYCLE_PROTOCOL_VERSION {
|
||||
return Err(Error::other("unsupported MRF response protocol"));
|
||||
}
|
||||
Ok(Some(response.result.to_vec()))
|
||||
}
|
||||
|
||||
pub async fn background_heal_status(&self) -> Result<Option<Vec<u8>>> {
|
||||
self.finalize_result(
|
||||
async {
|
||||
@@ -2429,6 +2505,129 @@ impl PeerRestClient {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Independently negotiated bucket watches; an old peer returns an error
|
||||
/// and callers retain the global-generation fallback for that bucket.
|
||||
pub async fn scanner_usage_watch(
|
||||
&self,
|
||||
owner_id: Uuid,
|
||||
mut entries: Vec<crate::runtime::usage_watch::ScannerUsageWatchRequest>,
|
||||
) -> Result<Vec<crate::runtime::usage_watch::ScannerUsageWatchObservation>> {
|
||||
use crate::runtime::usage_watch::{
|
||||
SCANNER_USAGE_WATCH_COVERAGE_VERSION, SCANNER_USAGE_WATCH_LEASE_MS, ScannerUsageWatchKnown,
|
||||
ScannerUsageWatchObservation,
|
||||
};
|
||||
use rustfs_protos::proto_gen::node_service::{ScannerUsageWatchEntry, ScannerUsageWatchRequest};
|
||||
use rustfs_protos::usage_watch::*;
|
||||
entries.sort_by(|left, right| left.bucket.cmp(&right.bucket));
|
||||
if owner_id.is_nil() || entries.windows(2).any(|pair| pair[0].bucket == pair[1].bucket) {
|
||||
return Err(Error::other("usage watch request contains duplicate buckets or an invalid owner"));
|
||||
}
|
||||
// Local queue pressure is not a network failure. Keep admission errors
|
||||
// outside finalize_result so they cannot mark a healthy peer offline.
|
||||
let _send_permit = self.acquire_usage_watch_send_permit().await?;
|
||||
self.finalize_result(
|
||||
async {
|
||||
let mut client = super::client::scanner_control_time_out_client(
|
||||
&self.grid_host,
|
||||
TonicInterceptor::Signature(gen_tonic_signature_interceptor()),
|
||||
)
|
||||
.await?;
|
||||
let mut observations = Vec::with_capacity(entries.len());
|
||||
for batch in entries.chunks(USAGE_WATCH_MAX_ENTRIES) {
|
||||
let payload = ScannerUsageWatchRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: USAGE_WATCH_PROTOCOL_VERSION,
|
||||
owner_id: owner_id.to_string(),
|
||||
entries: batch
|
||||
.iter()
|
||||
.map(|entry| ScannerUsageWatchEntry {
|
||||
bucket: entry.bucket.clone(),
|
||||
bucket_incarnation: entry.bucket_incarnation.as_bytes().to_vec().into(),
|
||||
watch_id: entry.watch_id.map_or_else(Bytes::new, |id| id.as_bytes().to_vec().into()),
|
||||
release: entry.release,
|
||||
session_id: entry.session_id.map_or_else(Bytes::new, |id| id.as_bytes().to_vec().into()),
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let canonical = canonical_usage_watch_request(&payload).map_err(|error| Error::other(error.to_string()))?;
|
||||
let mut request = Request::new(payload.clone());
|
||||
set_tonic_canonical_body_digest(&mut request, &canonical)?;
|
||||
let started = std::time::Instant::now();
|
||||
let response = timeout(SCANNER_SCOPED_DIRTY_USAGE_STAGE_TIMEOUT, client.scanner_usage_watch(request))
|
||||
.await
|
||||
.map_err(|_| Error::other("usage watch request timed out"))??
|
||||
.into_inner();
|
||||
let budget_used = u64::try_from(started.elapsed().as_millis())
|
||||
.unwrap_or(u64::MAX)
|
||||
.saturating_add(250);
|
||||
validate_scanner_usage_watch_response(&payload, &response)?;
|
||||
let session = Uuid::from_slice(response.session_id.as_ref())
|
||||
.map_err(|_| Error::other("invalid usage watch session"))?;
|
||||
if response.protocol_version != USAGE_WATCH_PROTOCOL_VERSION
|
||||
|| response.owner_id != payload.owner_id
|
||||
|| session.is_nil()
|
||||
|| response.entries.len() != payload.entries.len()
|
||||
{
|
||||
return Err(Error::other("usage watch response does not match request"));
|
||||
}
|
||||
for (requested, entry) in batch.iter().zip(response.entries) {
|
||||
if entry.bucket != requested.bucket {
|
||||
return Err(Error::other("usage watch response bucket order differs"));
|
||||
}
|
||||
let known = if entry.known {
|
||||
let token = Uuid::from_slice(entry.watch_id.as_ref())
|
||||
.map_err(|_| Error::other("invalid usage watch token"))?;
|
||||
if token.is_nil()
|
||||
|| entry.bucket_incarnation.as_ref() != requested.bucket_incarnation.as_bytes()
|
||||
|| requested.watch_id.is_some_and(|expected| expected != token)
|
||||
|| entry.revision == 0
|
||||
|| entry.revision == u64::MAX
|
||||
|| entry.inflight == u64::MAX
|
||||
|| entry.coverage_version != SCANNER_USAGE_WATCH_COVERAGE_VERSION
|
||||
|| entry.coverage_epoch == 0
|
||||
|| entry.lease_remaining_ms == 0
|
||||
|| entry.lease_remaining_ms > SCANNER_USAGE_WATCH_LEASE_MS
|
||||
{
|
||||
return Err(Error::other("usage watch response contains an invalid proof"));
|
||||
}
|
||||
let lease_remaining_ms = entry.lease_remaining_ms.saturating_sub(budget_used);
|
||||
(lease_remaining_ms != 0).then_some(ScannerUsageWatchKnown {
|
||||
owner_id,
|
||||
session_id: session,
|
||||
watch_id: token,
|
||||
bucket_incarnation: requested.bucket_incarnation,
|
||||
revision: entry.revision,
|
||||
inflight: entry.inflight,
|
||||
coverage_version: entry.coverage_version,
|
||||
coverage_epoch: entry.coverage_epoch,
|
||||
lease_remaining_ms,
|
||||
})
|
||||
} else {
|
||||
if !entry.watch_id.is_empty()
|
||||
|| !entry.bucket_incarnation.is_empty()
|
||||
|| entry.revision != 0
|
||||
|| entry.inflight != 0
|
||||
|| entry.coverage_version != 0
|
||||
|| entry.coverage_epoch != 0
|
||||
|| entry.lease_remaining_ms != 0
|
||||
{
|
||||
return Err(Error::other("Unknown usage watch entry contains proof fields"));
|
||||
}
|
||||
None
|
||||
};
|
||||
observations.push(ScannerUsageWatchObservation {
|
||||
bucket: entry.bucket,
|
||||
known,
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(observations)
|
||||
}
|
||||
.await,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_scoped_dirty_usage(
|
||||
&self,
|
||||
owner_id: String,
|
||||
@@ -3263,6 +3462,74 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_watch_client_clones_share_receiver_sized_send_admission() {
|
||||
let receiver = Arc::new(tokio::sync::Semaphore::new(4));
|
||||
let occupied = (0..4)
|
||||
.map(|_| receiver.clone().try_acquire_owned().expect("four receiver slots"))
|
||||
.collect::<Vec<_>>();
|
||||
assert!(receiver.try_acquire().is_err(), "an uncoordinated fifth request is rejected");
|
||||
drop(occupied);
|
||||
|
||||
let client = test_peer_client();
|
||||
let active = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let peak = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let tasks = (0..12).map(|_| {
|
||||
// Simulate the shared client's independent admission, keeper and
|
||||
// publication callers. Each uses the actual production gate.
|
||||
let client = client.clone();
|
||||
let receiver = receiver.clone();
|
||||
let active = active.clone();
|
||||
let peak = peak.clone();
|
||||
tokio::spawn(async move {
|
||||
let _send = client
|
||||
.acquire_usage_watch_send_permit()
|
||||
.await
|
||||
.expect("bounded sender admission");
|
||||
let _receive = receiver
|
||||
.try_acquire_owned()
|
||||
.expect("coexisting client clones must never send more than the receiver's four slots");
|
||||
let count = active.fetch_add(1, Ordering::AcqRel) + 1;
|
||||
peak.fetch_max(count, Ordering::AcqRel);
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
active.fetch_sub(1, Ordering::AcqRel);
|
||||
})
|
||||
});
|
||||
for result in futures::future::join_all(tasks).await {
|
||||
result.expect("all queued callers finish");
|
||||
}
|
||||
assert_eq!(peak.load(Ordering::Acquire), 4);
|
||||
assert_eq!(active.load(Ordering::Acquire), 0);
|
||||
assert_eq!(client.scanner_usage_watch_admission.available_permits(), 4);
|
||||
assert_eq!(receiver.available_permits(), 4);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn usage_watch_sender_queue_timeout_does_not_mark_the_peer_offline() {
|
||||
let client = test_peer_client();
|
||||
let _occupied = client
|
||||
.scanner_usage_watch_admission
|
||||
.clone()
|
||||
.acquire_many_owned(4)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = client
|
||||
.scanner_usage_watch(
|
||||
Uuid::from_u128(1),
|
||||
vec![crate::runtime::usage_watch::ScannerUsageWatchRequest {
|
||||
bucket: "bucket".into(),
|
||||
bucket_incarnation: Uuid::from_u128(2),
|
||||
watch_id: None,
|
||||
session_id: None,
|
||||
release: false,
|
||||
}],
|
||||
)
|
||||
.await;
|
||||
assert!(result.unwrap_err().to_string().contains("sender admission timed out"));
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
assert!(!client.recovery_running.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
fn decode_test_scanner_activity(response: ScannerActivityResponse) -> Result<ScannerPeerActivity> {
|
||||
decode_scanner_activity_with_verifier(response, &[9; 16], |_canonical, proof| {
|
||||
(proof == b"proof")
|
||||
@@ -4338,6 +4605,112 @@ mod tests {
|
||||
assert!(!client.offline.load(Ordering::Acquire));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_watch_consumer_authenticates_the_current_request_and_rejects_replay() {
|
||||
use rustfs_protos::proto_gen::node_service::{
|
||||
ScannerUsageWatchEntry, ScannerUsageWatchRequest, ScannerUsageWatchResponse, ScannerUsageWatchResult,
|
||||
};
|
||||
use rustfs_protos::usage_watch::{canonical_usage_watch_request, canonical_usage_watch_response};
|
||||
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
let request = ScannerUsageWatchRequest {
|
||||
challenge: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
protocol_version: 1,
|
||||
owner_id: Uuid::new_v4().to_string(),
|
||||
entries: vec![ScannerUsageWatchEntry {
|
||||
bucket: "usage-watch".into(),
|
||||
bucket_incarnation: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
watch_id: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
session_id: Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
release: false,
|
||||
}],
|
||||
};
|
||||
let sign = |request: &ScannerUsageWatchRequest, response: &mut ScannerUsageWatchResponse| {
|
||||
let canonical = canonical_usage_watch_request(request).expect("valid request to sign");
|
||||
let body = canonical_usage_watch_response(&canonical, response).expect("bounded canonical watch response");
|
||||
response.response_proof = crate::cluster::rpc::sign_tonic_rpc_response_proof(&body)
|
||||
.expect("real response HMAC")
|
||||
.into();
|
||||
};
|
||||
let mut response = ScannerUsageWatchResponse {
|
||||
protocol_version: 1,
|
||||
owner_id: request.owner_id.clone(),
|
||||
session_id: request.entries[0].session_id.clone(),
|
||||
entries: vec![ScannerUsageWatchResult {
|
||||
bucket: request.entries[0].bucket.clone(),
|
||||
known: true,
|
||||
bucket_incarnation: request.entries[0].bucket_incarnation.clone(),
|
||||
watch_id: request.entries[0].watch_id.clone(),
|
||||
revision: 7,
|
||||
inflight: 0,
|
||||
coverage_version: 1,
|
||||
coverage_epoch: 3,
|
||||
lease_remaining_ms: 60_000,
|
||||
}],
|
||||
response_proof: Bytes::new(),
|
||||
};
|
||||
sign(&request, &mut response);
|
||||
validate_scanner_usage_watch_response(&request, &response).expect("current request accepts its signed response");
|
||||
|
||||
let mut rejected_replays = 0;
|
||||
for field in 0..8 {
|
||||
let mut current = request.clone();
|
||||
match field {
|
||||
0 => current.challenge = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
1 => current.owner_id = Uuid::new_v4().to_string(),
|
||||
2 => current.entries[0].bucket.push('x'),
|
||||
3 => current.entries[0].bucket_incarnation = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
4 => current.entries[0].watch_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
5 => current.entries[0].session_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
6 => current.entries[0].release = true,
|
||||
_ => current.protocol_version += 1,
|
||||
}
|
||||
assert!(
|
||||
validate_scanner_usage_watch_response(¤t, &response).is_err(),
|
||||
"a previously signed response must not authorize request mutation {field}"
|
||||
);
|
||||
rejected_replays += 1;
|
||||
}
|
||||
let mut tampered = response.clone();
|
||||
tampered.entries[0].revision += 1;
|
||||
assert!(validate_scanner_usage_watch_response(&request, &tampered).is_err());
|
||||
let mut unsigned = response.clone();
|
||||
unsigned.response_proof = Bytes::new();
|
||||
assert!(validate_scanner_usage_watch_response(&request, &unsigned).is_err());
|
||||
|
||||
let mut rejected_signed_mismatches = 0;
|
||||
for field in 0..5 {
|
||||
let mut mismatched = response.clone();
|
||||
match field {
|
||||
0 => mismatched.owner_id = Uuid::new_v4().to_string(),
|
||||
1 => mismatched.session_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
2 => mismatched.entries[0].bucket_incarnation = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
3 => mismatched.entries[0].watch_id = Uuid::new_v4().as_bytes().to_vec().into(),
|
||||
_ => mismatched.entries[0].known = false,
|
||||
}
|
||||
sign(&request, &mut mismatched);
|
||||
assert!(
|
||||
validate_scanner_usage_watch_response(&request, &mismatched).is_err(),
|
||||
"an authentic HMAC cannot replace requested watch identity or authorize malformed Unknown: {field}"
|
||||
);
|
||||
rejected_signed_mismatches += 1;
|
||||
}
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-12", "assertion_id": "USAGE-12-A07", "status": "PASS",
|
||||
"evidence": {"current_response_authenticated": true, "replayed_requests_rejected": rejected_replays,
|
||||
"signed_identity_mismatches_rejected": rejected_signed_mismatches,
|
||||
"tampered_response_rejected": true, "missing_hmac_rejected": true}
|
||||
})
|
||||
);
|
||||
drop((request, response, tampered, unsigned));
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-12", "status": "PASS", "scope": "process-local"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_control_capability_proof_must_authenticate_exact_ack() {
|
||||
runtime_sources::ensure_test_rpc_secret();
|
||||
|
||||
@@ -1221,6 +1221,15 @@ impl RemoteDisk {
|
||||
}
|
||||
|
||||
pub(crate) async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
|
||||
Ok(self
|
||||
.ns_scanner_progress_capability()
|
||||
.await?
|
||||
.map(|capability| capability.server_epoch))
|
||||
}
|
||||
|
||||
pub(crate) async fn ns_scanner_progress_capability(
|
||||
&self,
|
||||
) -> Result<Option<crate::storage_api_contracts::internode::NsScannerCapabilities>> {
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
@@ -1228,6 +1237,9 @@ impl RemoteDisk {
|
||||
endpoint: self.endpoint.grid_host(),
|
||||
supports_tier_registry_generation: true,
|
||||
supports_checkpoint_coverage: true,
|
||||
supports_durable_progress: true,
|
||||
supports_complete_observation: true,
|
||||
supports_usage_watch: true,
|
||||
});
|
||||
let result = timeout(NS_SCANNER_CAPABILITY_PROBE_TIMEOUT, probe)
|
||||
.await
|
||||
@@ -1238,7 +1250,12 @@ impl RemoteDisk {
|
||||
if response.supports_tier_registry_generation == Some(true)
|
||||
&& response.supports_checkpoint_coverage == Some(true) =>
|
||||
{
|
||||
Ok(Some(response.server_epoch))
|
||||
Ok(Some(crate::storage_api_contracts::internode::NsScannerCapabilities {
|
||||
server_epoch: response.server_epoch,
|
||||
supports_durable_progress: response.supports_durable_progress == Some(true),
|
||||
supports_complete_observation: response.supports_complete_observation == Some(true),
|
||||
supports_usage_watch: response.usage_watch_proof.is_some(),
|
||||
}))
|
||||
}
|
||||
Ok(_) => Ok(None),
|
||||
// RUSTFS_COMPAT_TODO(ns-scanner-rpc-v3): old peers and legacy transports lack the authenticated startup-epoch handshake. Remove after every supported peer implements namespace scanner protocol v3.
|
||||
@@ -7065,6 +7082,9 @@ mod tests {
|
||||
proof: Vec::new(),
|
||||
supports_tier_registry_generation,
|
||||
supports_checkpoint_coverage: *self.ns_scanner_checkpoint_support.lock().expect("checkpoint capability lock"),
|
||||
supports_durable_progress: None,
|
||||
supports_complete_observation: None,
|
||||
usage_watch_proof: None,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -314,6 +314,16 @@ mod tests {
|
||||
assert_eq!(scanner_kvs.get(SCANNER_DELAY), "");
|
||||
assert_eq!(scanner_kvs.get(SCANNER_MAX_WAIT), "");
|
||||
assert_eq!(scanner_kvs.get(SCANNER_CYCLE_MAX_OBJECTS), "0");
|
||||
for key in [
|
||||
rustfs_config::SCANNER_BUCKET_QUANTUM_DURATION,
|
||||
rustfs_config::SCANNER_BUCKET_QUANTUM_OBJECTS,
|
||||
rustfs_config::SCANNER_BUCKET_QUANTUM_DIRECTORIES,
|
||||
rustfs_config::SCANNER_BUCKET_QUANTUM_ENTRIES,
|
||||
rustfs_config::SCANNER_BUCKET_QUANTUM_NAME_BYTES,
|
||||
] {
|
||||
assert!(rustfs_config::SCANNER_KEYS.contains(&key), "admin key {key} is registered");
|
||||
assert_eq!(scanner_kvs.get(key), "0", "quantum dimension {key} stays disabled until rollout");
|
||||
}
|
||||
|
||||
let heal_kvs = cfg
|
||||
.get_value(HEAL_SUB_SYS, DEFAULT_DELIMITER)
|
||||
|
||||
@@ -25,6 +25,12 @@ use rustfs_config::{
|
||||
SCANNER_MAX_CONCURRENT_SET_SCANS, SCANNER_MAX_WAIT, SCANNER_RECOVERY_BACKOFF_CAP, SCANNER_SPEED, SCANNER_START_DELAY,
|
||||
SCANNER_WALKERS_PER_DISK, SCANNER_YIELD_EVERY_N_OBJECTS,
|
||||
};
|
||||
use rustfs_config::{
|
||||
DEFAULT_SCANNER_BUCKET_QUANTUM_DIRECTORIES, DEFAULT_SCANNER_BUCKET_QUANTUM_DURATION_SECS,
|
||||
DEFAULT_SCANNER_BUCKET_QUANTUM_ENTRIES, DEFAULT_SCANNER_BUCKET_QUANTUM_NAME_BYTES, DEFAULT_SCANNER_BUCKET_QUANTUM_OBJECTS,
|
||||
SCANNER_BUCKET_QUANTUM_DIRECTORIES, SCANNER_BUCKET_QUANTUM_DURATION, SCANNER_BUCKET_QUANTUM_ENTRIES,
|
||||
SCANNER_BUCKET_QUANTUM_NAME_BYTES, SCANNER_BUCKET_QUANTUM_OBJECTS,
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
@@ -69,6 +75,31 @@ pub static DEFAULT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
value: DEFAULT_SCANNER_CYCLE_MAX_DIRECTORIES.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BUCKET_QUANTUM_DURATION.to_owned(),
|
||||
value: DEFAULT_SCANNER_BUCKET_QUANTUM_DURATION_SECS.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BUCKET_QUANTUM_OBJECTS.to_owned(),
|
||||
value: DEFAULT_SCANNER_BUCKET_QUANTUM_OBJECTS.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BUCKET_QUANTUM_DIRECTORIES.to_owned(),
|
||||
value: DEFAULT_SCANNER_BUCKET_QUANTUM_DIRECTORIES.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BUCKET_QUANTUM_ENTRIES.to_owned(),
|
||||
value: DEFAULT_SCANNER_BUCKET_QUANTUM_ENTRIES.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BUCKET_QUANTUM_NAME_BYTES.to_owned(),
|
||||
value: DEFAULT_SCANNER_BUCKET_QUANTUM_NAME_BYTES.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: SCANNER_BITROT_CYCLE.to_owned(),
|
||||
value: DEFAULT_SCANNER_BITROT_CYCLE_SECS.to_string(),
|
||||
|
||||
@@ -20944,12 +20944,12 @@ impl ECStore {
|
||||
let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
|
||||
let mut proof = None::<DecommissionDurableIlmReceipt>;
|
||||
let mut nonterminal_receipt_found = false;
|
||||
for pool_idx in 0..self.pools.len() {
|
||||
for (pool_idx, pool) in self.pools.iter().enumerate() {
|
||||
if pool_idx == source_pool_idx {
|
||||
continue;
|
||||
}
|
||||
let data = match read_config_limited_preserve_empty(
|
||||
self.pools[pool_idx].clone(),
|
||||
pool.clone(),
|
||||
&receipt_path,
|
||||
DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
|
||||
)
|
||||
@@ -28412,7 +28412,7 @@ mod tests {
|
||||
for wrapped in [
|
||||
err.clone(),
|
||||
data_movement::data_movement_stage_error_for_test("decommission_object", "put_object", "b", "o", err.clone()),
|
||||
with_decommission_entry_context("migrate_object", "b", "o", err.clone()),
|
||||
with_decommission_entry_context("migrate_object", "b", "o", err),
|
||||
] {
|
||||
assert_eq!(
|
||||
classify_decommission_capacity_failure(&wrapped),
|
||||
@@ -40280,7 +40280,7 @@ mod pools_tests {
|
||||
)
|
||||
.expect("finalize records the completion");
|
||||
|
||||
let mut adopted = meta.clone();
|
||||
let mut adopted = meta;
|
||||
let plan = decommission_capacity_intent_adoption_plan(&adopted, 0, generation)
|
||||
.expect("plan adoption")
|
||||
.expect("the interrupted completion should be planned");
|
||||
|
||||
@@ -4242,7 +4242,7 @@ mod tests {
|
||||
"compatible aliases for {suffix} should match"
|
||||
);
|
||||
|
||||
let mut source_without_marker = source.clone();
|
||||
let mut source_without_marker = source;
|
||||
rustfs_utils::http::remove_str(Arc::make_mut(&mut source_without_marker.user_defined), suffix);
|
||||
let mut target_with_generated_marker = source_without_marker.clone();
|
||||
rustfs_utils::http::insert_str(
|
||||
|
||||
@@ -56,6 +56,8 @@ pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
|
||||
const DATA_COMPRESSION_TOTAL_NAME: &str = ".compression.json";
|
||||
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
|
||||
const DATA_USAGE_CACHE_TTL_SECS: u64 = 30;
|
||||
const ADMIN_USAGE_INCARNATION_CONCURRENCY: usize = 4;
|
||||
const ADMIN_USAGE_INCARNATION_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
// A scanner save invalidates only the saving process's snapshot cache; every
|
||||
// other node sees it only after reloading. A fail-closed usage lookup for a
|
||||
// bucket the cached snapshot does not mention at all may reload a copy this
|
||||
@@ -505,6 +507,7 @@ fn stale_data_usage_persist_reason_for_source(
|
||||
/// Store data usage info to backend storage
|
||||
#[instrument(skip(store))]
|
||||
pub async fn store_data_usage_in_backend(data_usage_info: DataUsageInfo, store: Arc<ECStore>) -> Result<(), Error> {
|
||||
reject_admin_usage_projection(&data_usage_info)?;
|
||||
if data_usage_info.usage_snapshot_converged == Some(false) {
|
||||
return Err(Error::other(
|
||||
"nonconverged data usage observations cannot replace the quota-authoritative snapshot",
|
||||
@@ -534,6 +537,7 @@ async fn save_data_usage_in_backend(
|
||||
store: Arc<ECStore>,
|
||||
expected_publication_epoch: u64,
|
||||
) -> Result<(), Error> {
|
||||
reject_admin_usage_projection(&data_usage_info)?;
|
||||
let data =
|
||||
serde_json::to_vec(&data_usage_info).map_err(|e| Error::other(format!("Failed to serialize data usage info: {e}")))?;
|
||||
|
||||
@@ -663,8 +667,9 @@ fn remove_bucket_usage_from_info(data_usage_info: &mut DataUsageInfo, bucket: &s
|
||||
|
||||
let removed_usage = data_usage_info.buckets_usage.remove(bucket).is_some();
|
||||
let removed_size = data_usage_info.bucket_sizes.remove(bucket).is_some();
|
||||
let removed_observation = data_usage_info.usage_snapshot_bucket_observations.remove(bucket).is_some();
|
||||
|
||||
if !removed_usage && !removed_size {
|
||||
if !removed_usage && !removed_size && !removed_observation {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -816,7 +821,9 @@ fn map_data_usage_metadata_read_error(err: Error, object: &str) -> Error {
|
||||
}
|
||||
|
||||
fn data_usage_contains_bucket(data_usage_info: &DataUsageInfo, bucket: &str) -> bool {
|
||||
data_usage_info.buckets_usage.contains_key(bucket) || data_usage_info.bucket_sizes.contains_key(bucket)
|
||||
data_usage_info.buckets_usage.contains_key(bucket)
|
||||
|| data_usage_info.bucket_sizes.contains_key(bucket)
|
||||
|| data_usage_info.usage_snapshot_bucket_observations.contains_key(bucket)
|
||||
}
|
||||
|
||||
fn advance_data_usage_last_update(data_usage_info: &mut DataUsageInfo) {
|
||||
@@ -1137,14 +1144,26 @@ fn record_usage_snapshot_decode_failure(operation: &'static str, object: &str, e
|
||||
}
|
||||
|
||||
fn parse_usage_snapshot(buf: &[u8]) -> Result<DataUsageInfo, Error> {
|
||||
serde_json::from_slice(buf).map_err(|e| {
|
||||
let mut info: DataUsageInfo = serde_json::from_slice(buf).map_err(|e| {
|
||||
Error::other(format!(
|
||||
"Failed to deserialize data usage info: {:?} at line {} column {}",
|
||||
e.classify(),
|
||||
e.line(),
|
||||
e.column()
|
||||
))
|
||||
})
|
||||
})?;
|
||||
reject_admin_usage_projection(&info)?;
|
||||
for usage in info.buckets_usage.values_mut() {
|
||||
usage.normalize_usage_provenance();
|
||||
}
|
||||
Ok(info)
|
||||
}
|
||||
|
||||
fn reject_admin_usage_projection(info: &DataUsageInfo) -> Result<(), Error> {
|
||||
if info.usage_snapshot_admin_projection {
|
||||
return Err(Error::other("admin usage projections cannot be persisted or used as a storage baseline"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decide which usage snapshot to serve: a primary usage object or its
|
||||
@@ -1327,6 +1346,9 @@ async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Resu
|
||||
/// Fill quota-baseline gaps from an observed (nonconverged) snapshot without
|
||||
/// overriding any bucket the authoritative baseline already covers.
|
||||
fn backfill_degraded_baseline_from_observed(degraded_baseline: &mut HashMap<String, u64>, observed: &DataUsageInfo) {
|
||||
if observed.usage_snapshot_admin_projection {
|
||||
return;
|
||||
}
|
||||
for (bucket, usage) in &observed.buckets_usage {
|
||||
degraded_baseline.entry(bucket.clone()).or_insert(usage.size);
|
||||
}
|
||||
@@ -1419,6 +1441,7 @@ fn merge_partial_observation_for_admin(mut authoritative: DataUsageInfo, observe
|
||||
authoritative.usage_snapshot_converged = Some(false);
|
||||
authoritative.usage_snapshot_authoritative_baseline = observed.usage_snapshot_authoritative_baseline;
|
||||
authoritative.usage_snapshot_set_states = observed.usage_snapshot_set_states;
|
||||
authoritative.usage_snapshot_bucket_observations = observed.usage_snapshot_bucket_observations;
|
||||
authoritative.usage_snapshot_bootstrap_pending = false;
|
||||
authoritative.buckets_count = authoritative.buckets_usage.len() as u64;
|
||||
authoritative.bucket_sizes = authoritative
|
||||
@@ -1437,10 +1460,48 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
|
||||
let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?;
|
||||
// For the one-shot admin view a failed observed read degrades to "no
|
||||
// observation", same as before the read was fallible.
|
||||
let observed = load_observed_data_usage_snapshot(store).await.ok().flatten();
|
||||
let observed = load_observed_data_usage_snapshot(store.clone()).await.ok().flatten();
|
||||
let (selected, selected_is_current_format) =
|
||||
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
|
||||
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
|
||||
let mut response = normalize_loaded_data_usage(selected, selected_is_current_format).await.0;
|
||||
use futures::StreamExt as _;
|
||||
let deadline = tokio::time::Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT;
|
||||
let mut incarnations = HashMap::new();
|
||||
{
|
||||
// Display-only identity reads must not serialize every bucket behind a
|
||||
// slow metadata lock while holding the shared admin refresh permit.
|
||||
// The common deadline bounds their total added latency; dropping this
|
||||
// stream cancels all unfinished reads without spawning detached work.
|
||||
// Own the names and store across awaits so this future remains Send
|
||||
// when composed into spawned metrics collectors.
|
||||
let buckets = response
|
||||
.usage_snapshot_bucket_observations
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let mut identities = futures::stream::iter(buckets)
|
||||
.map(move |bucket| {
|
||||
let store = store.clone();
|
||||
async move {
|
||||
let identity = store.bucket_incarnation_id_from_disk(&bucket).await;
|
||||
(bucket, identity)
|
||||
}
|
||||
})
|
||||
.buffer_unordered(ADMIN_USAGE_INCARNATION_CONCURRENCY);
|
||||
while tokio::time::Instant::now() < deadline {
|
||||
let Ok(Some((bucket, identity))) = tokio::time::timeout_at(deadline, identities.next()).await else {
|
||||
break;
|
||||
};
|
||||
// Unavailable, timed-out or recreated buckets retain canonical values.
|
||||
if let Ok(incarnation) = identity
|
||||
&& !incarnation.is_nil()
|
||||
{
|
||||
incarnations.insert(bucket, incarnation.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
response.project_bucket_observations_for_admin(&incarnations);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
|
||||
@@ -1512,7 +1573,7 @@ async fn normalize_loaded_data_usage(
|
||||
&& !cfg.role.is_empty()
|
||||
{
|
||||
data_usage_info.replication_info.insert(
|
||||
cfg.role.clone(),
|
||||
cfg.role,
|
||||
BucketTargetUsageInfo {
|
||||
replication_failed_size: bui.replication_failed_size_v1,
|
||||
replication_failed_count: bui.replication_failed_count_v1,
|
||||
@@ -1725,7 +1786,7 @@ fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapsh
|
||||
match aggregated.buckets_usage.entry(bucket.clone()) {
|
||||
Entry::Occupied(mut entry) => entry.get_mut().merge(&usage),
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(usage.clone());
|
||||
entry.insert(usage);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2082,7 +2143,9 @@ where
|
||||
fn apply_live_bucket_usage_to_response(data_usage_info: &mut DataUsageInfo, bucket: &str, usage: &BucketUsageInfo) {
|
||||
let inserted_bucket = !data_usage_info.buckets_usage.contains_key(bucket);
|
||||
data_usage_info.bucket_sizes.insert(bucket.to_string(), usage.size);
|
||||
data_usage_info.buckets_usage.insert(bucket.to_string(), usage.clone());
|
||||
let mut observed = usage.clone();
|
||||
observed.mark_observed();
|
||||
data_usage_info.buckets_usage.insert(bucket.to_string(), observed);
|
||||
if inserted_bucket {
|
||||
data_usage_info.usage_snapshot_complete = false;
|
||||
}
|
||||
@@ -2774,7 +2837,9 @@ async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mu
|
||||
}
|
||||
|
||||
added_bucket |= !data_usage_info.buckets_usage.contains_key(bucket);
|
||||
data_usage_info.buckets_usage.insert(bucket.clone(), cached.usage.clone());
|
||||
let mut observed = cached.usage.clone();
|
||||
observed.mark_observed();
|
||||
data_usage_info.buckets_usage.insert(bucket.clone(), observed);
|
||||
data_usage_info.bucket_sizes.insert(bucket.clone(), cached.usage.size);
|
||||
changed = true;
|
||||
}
|
||||
@@ -3024,6 +3089,7 @@ pub async fn init_compression_total_memory_from_backend(store: Arc<ECStore>) {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
mod companion_projection;
|
||||
mod snapshot_projection;
|
||||
|
||||
use super::*;
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use rustfs_data_usage::{
|
||||
BucketUsageCertification, BucketUsageValueState, DATA_USAGE_BUCKET_OBSERVATION_VERSION, DataUsageBucketObservation,
|
||||
DataUsageBucketObservationSource, DataUsageSnapshotSetState,
|
||||
};
|
||||
|
||||
mod native_compatibility;
|
||||
|
||||
fn companion_snapshot(bucket: &str, incarnation: uuid::Uuid) -> DataUsageInfo {
|
||||
let completed_at = SystemTime::UNIX_EPOCH + Duration::from_secs(40);
|
||||
let mut snapshot = data_usage_info_for_test(bucket, 10, 100, completed_at + Duration::from_secs(10));
|
||||
snapshot.scanner_epoch = Some(2);
|
||||
snapshot.scanner_cycle = Some(9);
|
||||
snapshot.usage_snapshot_converged = Some(false);
|
||||
snapshot.usage_snapshot_pending_buckets = vec![bucket.to_string()];
|
||||
snapshot.usage_snapshot_pending_bucket_count = 1;
|
||||
let canonical = snapshot.buckets_usage.get_mut(bucket).expect("canonical bucket");
|
||||
canonical.bucket_incarnation = Some(incarnation.to_string());
|
||||
canonical.certify(BucketUsageCertification {
|
||||
epoch: 2,
|
||||
cycle: 3,
|
||||
certified_at: SystemTime::UNIX_EPOCH + Duration::from_secs(30),
|
||||
});
|
||||
canonical.mark_carried();
|
||||
snapshot.usage_snapshot_set_states = vec![DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
scanner_epoch: Some(2),
|
||||
scanner_cycle: Some(9),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: true,
|
||||
..Default::default()
|
||||
}];
|
||||
let mut observed = BucketUsageInfo {
|
||||
size: 250,
|
||||
objects_count: 25,
|
||||
versions_count: 25,
|
||||
bucket_incarnation: Some(incarnation.to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
observed.mark_observed();
|
||||
snapshot.usage_snapshot_bucket_observations.insert(
|
||||
bucket.to_string(),
|
||||
DataUsageBucketObservation {
|
||||
version: DATA_USAGE_BUCKET_OBSERVATION_VERSION,
|
||||
usage: observed,
|
||||
snapshot_identity: snapshot.snapshot_identity(),
|
||||
completed_at,
|
||||
sources: vec![DataUsageBucketObservationSource {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
bucket_incarnation: incarnation.to_string(),
|
||||
scanner_epoch: 2,
|
||||
scanner_cycle: 4,
|
||||
scan_plan_digest: [1; 32],
|
||||
started_plan: [2; 32],
|
||||
requested_plan: [3; 32],
|
||||
completed_at,
|
||||
}],
|
||||
},
|
||||
);
|
||||
snapshot
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn companion_cannot_backfill_quota_and_projection_is_rejected_at_storage_boundaries() {
|
||||
let incarnation = uuid::Uuid::from_u128(31);
|
||||
let mut snapshot = companion_snapshot("bucket", incarnation);
|
||||
let mut canonical_quota = HashMap::new();
|
||||
backfill_degraded_baseline_from_observed(&mut canonical_quota, &snapshot);
|
||||
assert_eq!(canonical_quota.get("bucket"), Some(&100), "quota reads only the unchanged main map");
|
||||
assert_eq!(
|
||||
snapshot.project_bucket_observations_for_admin(&HashMap::from([("bucket".to_string(), incarnation.to_string())])),
|
||||
1
|
||||
);
|
||||
let mut accidental_quota = HashMap::new();
|
||||
backfill_degraded_baseline_from_observed(&mut accidental_quota, &snapshot);
|
||||
assert!(accidental_quota.is_empty());
|
||||
assert!(reject_admin_usage_projection(&snapshot).is_err());
|
||||
assert!(certified_bucket_usage(&snapshot, "bucket", incarnation).is_none());
|
||||
let wire = serde_json::to_vec(&snapshot).expect("admin response remains serializable");
|
||||
assert!(parse_usage_snapshot(&wire).is_err(), "a serialized response is not a persisted snapshot");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn companion_only_observed_snapshot_does_not_create_a_legacy_quota_baseline() {
|
||||
let incarnation = uuid::Uuid::from_u128(31);
|
||||
let mut snapshot = companion_snapshot("bucket", incarnation);
|
||||
snapshot.buckets_usage.clear();
|
||||
snapshot.bucket_sizes.clear();
|
||||
snapshot.buckets_count = 0;
|
||||
snapshot.calculate_totals();
|
||||
snapshot.usage_snapshot_complete = false;
|
||||
snapshot.usage_snapshot_partial = true;
|
||||
snapshot.usage_snapshot_observational = true;
|
||||
snapshot.usage_snapshot_set_states[0].complete = false;
|
||||
let decoded = parse_usage_snapshot(&serde_json::to_vec(&snapshot).expect("encode companion-only snapshot"))
|
||||
.expect("read valid partial snapshot");
|
||||
assert!(decoded.is_valid_partial_snapshot());
|
||||
let mut baseline = HashMap::new();
|
||||
backfill_degraded_baseline_from_observed(&mut baseline, &decoded);
|
||||
assert!(baseline.is_empty(), "unknown usage must not become either zero or mixed quota usage");
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-08", "assertion_id": "USAGE-08-A02", "status": "PASS",
|
||||
"evidence": {"primary_buckets": decoded.buckets_usage.len(), "legacy_bucket_sizes": decoded.bucket_sizes.len(), "quota_baseline_buckets": baseline.len(), "companion_buckets": decoded.usage_snapshot_bucket_observations.len(), "partial": decoded.is_valid_partial_snapshot()}
|
||||
})
|
||||
);
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-08", "status": "PASS", "scope": "process-local"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bucket_removal_discards_companions_even_without_a_canonical_entry() {
|
||||
let mut snapshot = companion_snapshot("bucket", uuid::Uuid::from_u128(31));
|
||||
snapshot.buckets_usage.clear();
|
||||
snapshot.bucket_sizes.clear();
|
||||
assert!(data_usage_contains_bucket(&snapshot, "bucket"));
|
||||
assert!(remove_bucket_usage_from_info(&mut snapshot, "bucket"));
|
||||
assert!(!data_usage_contains_bucket(&snapshot, "bucket"));
|
||||
assert!(snapshot.usage_snapshot_bucket_observations.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_selection_replaces_value_provenance_together() {
|
||||
let incarnation = uuid::Uuid::from_u128(31);
|
||||
let authoritative = companion_snapshot("bucket", incarnation);
|
||||
let mut observed = authoritative.clone();
|
||||
observed.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
|
||||
observed.scanner_cycle = Some(10);
|
||||
observed.last_update = Some(SystemTime::UNIX_EPOCH + Duration::from_secs(60));
|
||||
observed.usage_snapshot_complete = false;
|
||||
observed.usage_snapshot_partial = true;
|
||||
observed.usage_snapshot_observational = true;
|
||||
observed.usage_snapshot_bucket_observations.clear();
|
||||
let usage = observed.buckets_usage.get_mut("bucket").expect("observed main bucket");
|
||||
usage.size = 175;
|
||||
usage.mark_observed();
|
||||
let (selected, _) = select_admin_data_usage_snapshot(authoritative, true, Some(observed));
|
||||
assert_eq!(selected.buckets_usage["bucket"].size, 175);
|
||||
assert_eq!(selected.buckets_usage["bucket"].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert_eq!(selected.buckets_usage["bucket"].usage_certification, None);
|
||||
assert!(
|
||||
selected.usage_snapshot_bucket_observations.is_empty(),
|
||||
"old root companions cannot cross a baseline change"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn companion_admin_cache_isolated_from_quota_memory_and_persistence() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::config::com::save_config;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "usage-companion";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket for incarnation validation");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("trusted incarnation");
|
||||
let canonical = companion_snapshot(bucket, incarnation);
|
||||
let bytes = serde_json::to_vec(&canonical).expect("encode canonical root with companion");
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), bytes.clone())
|
||||
.await
|
||||
.expect("save canonical root");
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let mut samples = Vec::new();
|
||||
let mut rejected_writes = 0;
|
||||
for _ in 0..2 {
|
||||
let response = load_admin_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("read admin cache");
|
||||
assert_eq!(response.buckets_usage[bucket].size, 250);
|
||||
assert!(response.usage_snapshot_admin_projection);
|
||||
replace_bucket_usage_memory_from_info(&response).await;
|
||||
assert!(memory_cache().read().await.is_empty(), "response cannot seed quota memory");
|
||||
let admin_bytes = response.buckets_usage[bucket].size;
|
||||
let rejected = store_data_usage_in_backend(response, store.clone()).await.is_err();
|
||||
assert!(rejected);
|
||||
rejected_writes += usize::from(rejected);
|
||||
let quota = load_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("read independent quota cache");
|
||||
assert_eq!(quota.buckets_usage[bucket].size, 100);
|
||||
assert!(!quota.usage_snapshot_admin_projection);
|
||||
samples.push((admin_bytes, quota.buckets_usage[bucket].size));
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
let persisted = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("read unchanged canonical root");
|
||||
assert_eq!(persisted, bytes);
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-18", "assertion_id": "USAGE-18-A03", "status": "PASS",
|
||||
"evidence": {"admin_and_quota_byte_samples": samples, "rejected_projected_writes": rejected_writes, "quota_memory_entries": memory_cache().read().await.len(), "persisted_canonical_unchanged": persisted == bytes}
|
||||
})
|
||||
);
|
||||
let paths = _dirs.iter().map(|dir| dir.path().to_path_buf()).collect::<Vec<_>>();
|
||||
drop(_dirs);
|
||||
assert!(paths.iter().all(|path| !path.exists()));
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-18", "status": "PASS", "fixtures_removed": paths.iter().all(|path| !path.exists())})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buckets() {
|
||||
use crate::bucket::metadata_sys::{
|
||||
acquire_metadata_transaction_guard, init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
|
||||
};
|
||||
use crate::config::com::save_config;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
|
||||
// Admin reads also feed spawned metrics collectors across crate boundaries.
|
||||
fn assert_send<T: Send>(future: T) -> T {
|
||||
future
|
||||
}
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let blocked = "usage-blocked-identity";
|
||||
let healthy = "usage-healthy-identity";
|
||||
let mut snapshots = Vec::new();
|
||||
for bucket in [blocked, healthy] {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create companion identity fixture");
|
||||
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("real incarnation");
|
||||
snapshots.push(companion_snapshot(bucket, incarnation));
|
||||
}
|
||||
let mut canonical = snapshots.remove(0);
|
||||
let other = snapshots.remove(0);
|
||||
canonical.buckets_usage.extend(other.buckets_usage);
|
||||
canonical.bucket_sizes.extend(other.bucket_sizes);
|
||||
canonical
|
||||
.usage_snapshot_bucket_observations
|
||||
.extend(other.usage_snapshot_bucket_observations);
|
||||
canonical.usage_snapshot_pending_buckets.push(healthy.to_string());
|
||||
canonical.usage_snapshot_pending_bucket_count = 2;
|
||||
canonical.buckets_count = 2;
|
||||
canonical.calculate_totals();
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_OBJ_NAME_PATH.as_str(),
|
||||
serde_json::to_vec(&canonical).expect("encode both canonical values and companions"),
|
||||
)
|
||||
.await
|
||||
.expect("persist usage fixture");
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let blocked_guard = acquire_metadata_transaction_guard(
|
||||
store.clone(),
|
||||
blocked,
|
||||
"admin_identity_timeout_test",
|
||||
rustfs_lock::LockType::Exclusive,
|
||||
)
|
||||
.await
|
||||
.expect("hold the real bucket metadata writer lock");
|
||||
let response = tokio::time::timeout(
|
||||
ADMIN_USAGE_INCARNATION_TIMEOUT + Duration::from_secs(3),
|
||||
assert_send(load_admin_data_usage_from_backend_cached(store.clone())),
|
||||
)
|
||||
.await
|
||||
.expect("optional projection must return before the blocked metadata lock's long timeout")
|
||||
.expect("canonical admin response stays available");
|
||||
assert_eq!(response.buckets_usage[blocked].size, 100);
|
||||
assert_eq!(response.buckets_usage[blocked].usage_value_state, Some(BucketUsageValueState::Carried));
|
||||
assert_eq!(response.buckets_usage[healthy].size, 250);
|
||||
assert_eq!(response.buckets_usage[healthy].usage_value_state, Some(BucketUsageValueState::Observed));
|
||||
assert!(response.usage_snapshot_admin_projection);
|
||||
assert!(
|
||||
ADMIN_DATA_USAGE_SNAPSHOT_REFRESH
|
||||
.get()
|
||||
.expect("admin refresh mutex")
|
||||
.try_lock()
|
||||
.is_ok()
|
||||
);
|
||||
drop(blocked_guard);
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let recovered = load_admin_data_usage_from_backend_cached(store.clone())
|
||||
.await
|
||||
.expect("refresh after lock recovery");
|
||||
assert_eq!(
|
||||
recovered.buckets_usage[blocked].size, 250,
|
||||
"unavailable identity can recover on the next refresh"
|
||||
);
|
||||
assert_eq!(recovered.buckets_usage[healthy].size, 250);
|
||||
let quota = load_data_usage_from_backend_cached(store)
|
||||
.await
|
||||
.expect("separate canonical quota read");
|
||||
assert_eq!(quota.buckets_usage[blocked].size, 100);
|
||||
assert_eq!(quota.buckets_usage[healthy].size, 100);
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::*;
|
||||
|
||||
// Freeze the pre-companion reader's primary fields. Serde's unknown-field
|
||||
// behavior models the existing JSON reader without importing the new schema.
|
||||
#[derive(serde::Deserialize)]
|
||||
struct LegacyPrimarySnapshot {
|
||||
buckets_usage: HashMap<String, LegacyPrimaryBucket>,
|
||||
bucket_sizes: HashMap<String, u64>,
|
||||
objects_total_count: u64,
|
||||
objects_total_size: u64,
|
||||
#[serde(default)]
|
||||
usage_snapshot_complete: bool,
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct LegacyPrimaryBucket {
|
||||
size: u64,
|
||||
objects_count: u64,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_legacy_reader_never_promotes_companion_values_into_primary_lkg() {
|
||||
let incarnation = uuid::Uuid::from_u128(31);
|
||||
let mut snapshot = companion_snapshot("bucket", incarnation);
|
||||
let original_certificate = snapshot.buckets_usage["bucket"].usage_certification;
|
||||
let mut decoded_sizes = Vec::new();
|
||||
for cycle in 10..=12 {
|
||||
snapshot.scanner_cycle = Some(cycle);
|
||||
let identity = snapshot.snapshot_identity();
|
||||
snapshot
|
||||
.usage_snapshot_bucket_observations
|
||||
.get_mut("bucket")
|
||||
.unwrap()
|
||||
.snapshot_identity = identity;
|
||||
let bytes = serde_json::to_vec(&snapshot).expect("production persisted snapshot encoding");
|
||||
let old: LegacyPrimarySnapshot = serde_json::from_slice(&bytes).expect("old reader tolerates additive fields");
|
||||
assert!(old.usage_snapshot_complete);
|
||||
assert_eq!(old.buckets_usage["bucket"].size, 100);
|
||||
assert_eq!(old.buckets_usage["bucket"].objects_count, 10);
|
||||
assert_eq!(old.bucket_sizes["bucket"], 100);
|
||||
assert_eq!(old.objects_total_size, 100);
|
||||
assert_eq!(old.objects_total_count, 10);
|
||||
let current = parse_usage_snapshot(&bytes).expect("current reader accepts persisted primary plus companion");
|
||||
assert_eq!(current.buckets_usage["bucket"].usage_certification, original_certificate);
|
||||
assert_eq!(current.usage_snapshot_bucket_observations["bucket"].usage.size, 250);
|
||||
decoded_sizes.push(old.buckets_usage["bucket"].size);
|
||||
}
|
||||
|
||||
snapshot.buckets_usage.clear();
|
||||
snapshot.bucket_sizes.clear();
|
||||
snapshot.buckets_count = 0;
|
||||
snapshot.calculate_totals();
|
||||
snapshot.usage_snapshot_complete = false;
|
||||
snapshot.usage_snapshot_partial = true;
|
||||
snapshot.usage_snapshot_observational = true;
|
||||
snapshot.usage_snapshot_set_states[0].complete = false;
|
||||
let bytes = serde_json::to_vec(&snapshot).expect("persisted companion-only encoding");
|
||||
let old: LegacyPrimarySnapshot = serde_json::from_slice(&bytes).expect("old reader handles the absent baseline");
|
||||
assert!(old.buckets_usage.is_empty() && old.bucket_sizes.is_empty());
|
||||
assert!(!old.usage_snapshot_complete);
|
||||
assert_eq!((old.objects_total_count, old.objects_total_size), (0, 0));
|
||||
let current = parse_usage_snapshot(&bytes).expect("valid current observational snapshot");
|
||||
assert!(current.is_valid_partial_snapshot());
|
||||
assert_eq!(current.usage_snapshot_bucket_observations["bucket"].usage.size, 250);
|
||||
|
||||
assert_eq!(
|
||||
snapshot.project_bucket_observations_for_admin(&HashMap::from([("bucket".into(), incarnation.to_string())])),
|
||||
1
|
||||
);
|
||||
let response_bytes = serde_json::to_vec(&snapshot).expect("admin response encoding");
|
||||
assert!(
|
||||
parse_usage_snapshot(&response_bytes).is_err(),
|
||||
"a projected response cannot be written as a baseline"
|
||||
);
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-16", "assertion_id": "USAGE-16-A01", "status": "PASS",
|
||||
"evidence": {
|
||||
"wire_format": "production JSON, frozen legacy named primary fields",
|
||||
"legacy_primary_sizes_across_rounds": decoded_sizes,
|
||||
"new_companion_size": 250,
|
||||
"companion_only_legacy_primary_count": old.buckets_usage.len(),
|
||||
"companion_only_legacy_complete": old.usage_snapshot_complete,
|
||||
"projected_response_rejected_by_persistence_parser": true
|
||||
}
|
||||
})
|
||||
);
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-16", "status": "PASS", "scope": "process-local"})
|
||||
);
|
||||
}
|
||||
@@ -521,7 +521,7 @@ impl EndpointTopology {
|
||||
}
|
||||
};
|
||||
|
||||
topology.aliases.entry(host_port.clone()).or_insert(member_index);
|
||||
topology.aliases.entry(host_port).or_insert(member_index);
|
||||
topology.aliases.entry(ep.to_string()).or_insert(member_index);
|
||||
if let Some(display_endpoint) = ep.url.host_str() {
|
||||
match display_aliases.entry(display_endpoint.to_owned()) {
|
||||
|
||||
@@ -1626,7 +1626,7 @@ mod tests {
|
||||
|
||||
for disk_error in disk_errors {
|
||||
let original_message = disk_error.to_string();
|
||||
let io_error: std::io::Error = disk_error.clone().into();
|
||||
let io_error: std::io::Error = disk_error.into();
|
||||
|
||||
// The io::Error should contain the original error message
|
||||
assert!(io_error.to_string().contains(&original_message));
|
||||
|
||||
@@ -175,6 +175,7 @@ fn is_windows_dir_access_denied(_path: &Path, _e: &io::Error) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn remove(path: impl AsRef<Path>) -> io::Result<()> {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
|
||||
@@ -76,11 +76,11 @@ use crate::disk::{
|
||||
error_conv::{to_access_error, to_file_error, to_unformatted_disk_error, to_volume_error},
|
||||
format::FormatV3,
|
||||
fs::{
|
||||
O_APPEND, O_CREATE, O_RDONLY, O_WRONLY, access, cached_access, invalidate_bucket_cache, lstat, lstat_std, remove,
|
||||
remove_all_std, remove_std, rename,
|
||||
O_APPEND, O_CREATE, O_RDONLY, O_WRONLY, access, cached_access, invalidate_bucket_cache, lstat, lstat_std, remove_all_std,
|
||||
remove_std, rename,
|
||||
},
|
||||
is_quota_mutation_fence_path, os,
|
||||
os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all, rename_all_ignore_missing_source},
|
||||
os::{check_path_length, is_dir_not_empty_error, is_empty_dir, is_root_disk, rename_all_ignore_missing_source},
|
||||
quota_mutation_fence_path,
|
||||
};
|
||||
use crate::erasure::coding::{self, bitrot_verify};
|
||||
@@ -3339,11 +3339,7 @@ pub(crate) fn has_delete_version_test_failpoints() -> bool {
|
||||
if os::prepared_publication_test_hooks::has_any_hooks() {
|
||||
return true;
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").is_some() {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
cfg!(feature = "e2e-test-hooks") && std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").is_some()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -7587,6 +7583,7 @@ pub struct LocalDisk {
|
||||
pub root: PathBuf,
|
||||
publication_root: os::PublicationRoot,
|
||||
multipart_proof_authority: OnceLock<crate::disk::multipart_source::ProofVerificationContext>,
|
||||
pub(crate) usage_watch_binding: Arc<super::usage_watch::DiskUsageWatchBinding>,
|
||||
multipart_index: Arc<multipart_index::IndexOwner>,
|
||||
/// I/O root pinned to the mount instance that was opened while the disk
|
||||
/// was initialized. On Linux this is `/proc/self/fd/<dirfd>/.`; resolving
|
||||
@@ -8137,6 +8134,7 @@ impl LocalDisk {
|
||||
root: root.clone(),
|
||||
publication_root,
|
||||
multipart_proof_authority: OnceLock::new(),
|
||||
usage_watch_binding: Arc::new(super::usage_watch::DiskUsageWatchBinding::default()),
|
||||
io_root: io_root.clone(),
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease: mount_lease.clone(),
|
||||
@@ -8980,6 +8978,7 @@ impl LocalDisk {
|
||||
fi: FileInfo,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
|
||||
if volume == RUSTFS_META_BUCKET {
|
||||
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
|
||||
}
|
||||
@@ -9189,6 +9188,7 @@ impl LocalDisk {
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Vec<Option<Error>> {
|
||||
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
|
||||
let concurrency = delete_version_batch_concurrency(&versions, &opts);
|
||||
let mut errors = vec![None; versions.len()];
|
||||
let mut pending = stream::iter(
|
||||
@@ -9221,6 +9221,7 @@ impl LocalDisk {
|
||||
opts: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Vec<Option<DiskError>> {
|
||||
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
|
||||
let payload = super::GuardedDeleteVersionsPayload {
|
||||
versions,
|
||||
opts,
|
||||
@@ -9256,6 +9257,8 @@ impl LocalDisk {
|
||||
mut fi: FileInfo,
|
||||
mutation: DeleteVersionMutation,
|
||||
) -> Result<()> {
|
||||
let mut mutation = mutation;
|
||||
mutation.namespace_owner = self.usage_watch_binding.owner(volume, mutation.namespace_owner.take());
|
||||
if volume == RUSTFS_META_BUCKET {
|
||||
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
|
||||
}
|
||||
@@ -9878,7 +9881,12 @@ impl LocalDisk {
|
||||
.await
|
||||
.err()
|
||||
} else {
|
||||
match rename(&delete_path, &trash_path).await {
|
||||
let renamed = if namespace_owner.is_some() {
|
||||
os::rename_with_namespace_owner(delete_path, &trash_path, namespace_owner.clone()).await
|
||||
} else {
|
||||
rename(delete_path, &trash_path).await
|
||||
};
|
||||
match renamed {
|
||||
Ok(()) => None,
|
||||
Err(err)
|
||||
if err.kind() == ErrorKind::NotFound && os::rename_source_is_missing(delete_path, &self.publication_root) =>
|
||||
@@ -9930,6 +9938,7 @@ impl LocalDisk {
|
||||
opt: DeleteOptions,
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
|
||||
if volume == RUSTFS_META_BUCKET {
|
||||
crate::bucket::metadata_sys::bump_local_meta_mutation_gen();
|
||||
}
|
||||
@@ -10365,6 +10374,7 @@ impl LocalDisk {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn delete_file(&self, base_path: &Path, delete_path: &PathBuf, recursive: bool, immediate_purge: bool) -> Result<()> {
|
||||
self.delete_file_with_namespace_owner(base_path, delete_path, recursive, immediate_purge, None, None)
|
||||
.await
|
||||
@@ -11642,6 +11652,15 @@ impl LocalDisk {
|
||||
// Executes the given SyncMode verbatim: durability policy (tier gating,
|
||||
// system-critical pinning) is resolved by callers via effective_durability.
|
||||
async fn write_all_internal(&self, file_path: &Path, data: Bytes, sync: SyncMode, skip_parent: &Path) -> Result<()> {
|
||||
let relative = file_path
|
||||
.strip_prefix(self.io_root())
|
||||
.or_else(|_| file_path.strip_prefix(&self.root));
|
||||
let bucket = relative
|
||||
.ok()
|
||||
.and_then(|path| path.components().next())
|
||||
.and_then(|component| component.as_os_str().to_str())
|
||||
.unwrap_or("");
|
||||
let usage_owner = self.usage_watch_binding.owner(bucket, None);
|
||||
let skip_parent = if skip_parent.as_os_str().is_empty() {
|
||||
self.io_root()
|
||||
} else {
|
||||
@@ -11660,6 +11679,7 @@ impl LocalDisk {
|
||||
|
||||
let root = self.publication_root.clone();
|
||||
let operation = move |is_cancelled: &dyn Fn() -> bool| {
|
||||
let _usage_owner = usage_owner;
|
||||
if is_cancelled() {
|
||||
return Err(std::io::Error::other("blocking namespace operation cancelled"));
|
||||
}
|
||||
@@ -14562,7 +14582,15 @@ impl DiskAPI for LocalDisk {
|
||||
multipart_staging::reject_native_control_write(&self.root, self.io_root(), src_volume, src_path)?;
|
||||
multipart_staging::reject_native_control_write(&self.root, self.io_root(), dst_volume, dst_path)?;
|
||||
multipart_staging::reject_noncanonical_scope(&self.root, self.io_root(), src_volume, src_path)?;
|
||||
let source_owner = self.multipart_source_owner(dst_volume, dst_path, true, false, None).await?;
|
||||
let usage_owner = self.usage_watch_binding.owner(src_volume, None);
|
||||
let usage_owner = if src_volume == dst_volume {
|
||||
usage_owner
|
||||
} else {
|
||||
self.usage_watch_binding.owner(dst_volume, usage_owner)
|
||||
};
|
||||
let source_owner = self
|
||||
.multipart_source_owner(dst_volume, dst_path, true, false, usage_owner)
|
||||
.await?;
|
||||
let admission = self.multipart_source_admission(dst_volume, dst_path, true, false)?;
|
||||
let (src_volume_dir, src_file_path) = self.io_get_bucket_and_object_path(src_volume, src_path)?;
|
||||
let (dst_volume_dir, dst_file_path) = self.io_get_bucket_and_object_path(dst_volume, dst_path)?;
|
||||
@@ -14751,7 +14779,8 @@ impl DiskAPI for LocalDisk {
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn settle_part_transaction(&self, volume: &str, path: &str, action: PartTransactionAction) -> Result<()> {
|
||||
let source_owner = self.multipart_source_owner(volume, path, true, false, None).await?;
|
||||
let usage_owner = self.usage_watch_binding.owner(volume, None);
|
||||
let source_owner = self.multipart_source_owner(volume, path, true, false, usage_owner).await?;
|
||||
let admission = self.multipart_source_admission(volume, path, true, false)?;
|
||||
let admission_source = admission.as_ref().map(|a| a.source().to_path_buf());
|
||||
let txn_rel_path = crate::disk::part_transaction_path(path);
|
||||
@@ -14902,7 +14931,15 @@ impl DiskAPI for LocalDisk {
|
||||
multipart_staging::reject_native_control_write(&self.root, self.io_root(), src_volume, src_path)?;
|
||||
multipart_staging::reject_native_control_write(&self.root, self.io_root(), dst_volume, dst_path)?;
|
||||
multipart_staging::reject_noncanonical_scope(&self.root, self.io_root(), src_volume, src_path)?;
|
||||
let source_owner = self.multipart_source_owner(dst_volume, dst_path, true, false, None).await?;
|
||||
let usage_owner = self.usage_watch_binding.owner(src_volume, None);
|
||||
let usage_owner = if src_volume == dst_volume {
|
||||
usage_owner
|
||||
} else {
|
||||
self.usage_watch_binding.owner(dst_volume, usage_owner)
|
||||
};
|
||||
let source_owner = self
|
||||
.multipart_source_owner(dst_volume, dst_path, true, false, usage_owner)
|
||||
.await?;
|
||||
multipart_staging::reject_reserved_anchor(dst_path)?;
|
||||
let admission = self.multipart_source_admission(dst_volume, dst_path, true, false)?;
|
||||
let admission_source = admission.as_ref().map(|a| a.source().to_path_buf());
|
||||
@@ -15214,8 +15251,15 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
if !published_transaction_meta && let Some(parent) = src_file_for_cleanup.parent() {
|
||||
self.delete_file(&src_volume_for_cleanup, &parent.to_path_buf(), false, false)
|
||||
.await?;
|
||||
self.delete_file_with_namespace_owner(
|
||||
&src_volume_for_cleanup,
|
||||
&parent.to_path_buf(),
|
||||
false,
|
||||
false,
|
||||
source_owner,
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -15223,6 +15267,12 @@ impl DiskAPI for LocalDisk {
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
|
||||
let usage_owner = self.usage_watch_binding.owner(src_volume, None);
|
||||
let usage_owner = if src_volume == dst_volume {
|
||||
usage_owner
|
||||
} else {
|
||||
self.usage_watch_binding.owner(dst_volume, usage_owner)
|
||||
};
|
||||
multipart_staging::reject_native_control_write(&self.root, self.io_root(), src_volume, src_path)?;
|
||||
multipart_staging::reject_native_control_write(&self.root, self.io_root(), dst_volume, dst_path)?;
|
||||
if src_volume == RUSTFS_META_MULTIPART_BUCKET || dst_volume == RUSTFS_META_MULTIPART_BUCKET {
|
||||
@@ -15280,14 +15330,21 @@ impl DiskAPI for LocalDisk {
|
||||
// destination is the normal case when renaming a directory to a new
|
||||
// location, so tolerate NotFound instead of aborting the whole rename
|
||||
// (MinIO's RenameFile ignores osIsNotExist here).
|
||||
if let Err(e) = remove(&dst_file_path).await
|
||||
if let Err(e) = os::remove_with_owner(&dst_file_path, usage_owner.clone()).await
|
||||
&& e.kind() != ErrorKind::NotFound
|
||||
{
|
||||
return Err(to_file_error(e).into());
|
||||
}
|
||||
}
|
||||
|
||||
rename_all(&src_file_path, &dst_file_path, &dst_volume_dir, &self.publication_root).await?;
|
||||
os::rename_all_with_owner(
|
||||
&src_file_path,
|
||||
&dst_file_path,
|
||||
&dst_volume_dir,
|
||||
&self.publication_root,
|
||||
usage_owner.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Both ends changed identity: the source path no longer exists and the
|
||||
// destination now resolves to a different inode (backlog#1145).
|
||||
@@ -15300,7 +15357,9 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
if let Some(parent) = src_file_path.parent() {
|
||||
let _ = self.delete_file(&src_volume_dir, &parent.to_path_buf(), false, false).await;
|
||||
let _ = self
|
||||
.delete_file_with_namespace_owner(&src_volume_dir, &parent.to_path_buf(), false, false, usage_owner, None)
|
||||
.await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -15308,6 +15367,10 @@ impl DiskAPI for LocalDisk {
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn create_file(&self, origvolume: &str, volume: &str, path: &str, _file_size: i64) -> Result<FileWriter> {
|
||||
// Normal PUT streams into the internal staging volume and is covered
|
||||
// by rename_data. A raw stream into a live user volume has no retained
|
||||
// physical completion owner, so it revokes the watch capability.
|
||||
self.usage_watch_binding.mark_uncovered_user_stream(volume);
|
||||
multipart_staging::reject_native_control_write(&self.root, self.io_root(), volume, path)?;
|
||||
multipart_staging::reject_reserved_anchor(path)?;
|
||||
multipart_staging::reject_noncanonical_scope(&self.root, self.io_root(), volume, path)?;
|
||||
@@ -15350,6 +15413,12 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
|
||||
async fn rename_file_durable(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> {
|
||||
let usage_owner = self.usage_watch_binding.owner(src_volume, None);
|
||||
let usage_owner = if src_volume == dst_volume {
|
||||
usage_owner
|
||||
} else {
|
||||
self.usage_watch_binding.owner(dst_volume, usage_owner)
|
||||
};
|
||||
let source = self.io_get_object_path(src_volume, src_path)?;
|
||||
let destination = self.io_get_object_path(dst_volume, dst_path)?;
|
||||
check_path_length(source.to_string_lossy().as_ref())?;
|
||||
@@ -15360,9 +15429,11 @@ impl DiskAPI for LocalDisk {
|
||||
return Err(DiskError::FileAccessDenied);
|
||||
}
|
||||
if durability.syncs_data_shards() {
|
||||
let sync_owner = usage_owner.clone();
|
||||
#[cfg(target_os = "linux")]
|
||||
let mount_lease = self.mount_lease.clone();
|
||||
spawn_blocking_with_disk_io_permit(None, move || {
|
||||
let _usage_owner = sync_owner;
|
||||
#[cfg(target_os = "linux")]
|
||||
let _mount_lease = mount_lease;
|
||||
os::sync_file(&source)
|
||||
@@ -15373,7 +15444,7 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
self.rename_file(src_volume, src_path, dst_volume, dst_path).await?;
|
||||
if durability.syncs_commit_metadata() {
|
||||
os::fsync_dir(destination.parent().ok_or(DiskError::InvalidPath)?)
|
||||
os::fsync_dir_with_owner(destination.parent().ok_or(DiskError::InvalidPath)?, usage_owner.clone())
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
}
|
||||
@@ -15383,6 +15454,7 @@ impl DiskAPI for LocalDisk {
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
// async fn append_file(&self, volume: &str, path: &str, mut r: DuplexStream) -> Result<File> {
|
||||
async fn append_file(&self, volume: &str, path: &str) -> Result<FileWriter> {
|
||||
self.usage_watch_binding.mark_uncovered_user_stream(volume);
|
||||
multipart_staging::reject_native_control_write(&self.root, self.io_root(), volume, path)?;
|
||||
multipart_staging::reject_reserved_anchor(path)?;
|
||||
multipart_staging::reject_noncanonical_scope(&self.root, self.io_root(), volume, path)?;
|
||||
@@ -15732,6 +15804,7 @@ impl DiskAPI for LocalDisk {
|
||||
} else {
|
||||
self.multipart_source_owner(volume, path, true, true, None).await?
|
||||
};
|
||||
let owner = self.usage_watch_binding.owner(volume, owner);
|
||||
self.move_to_trash_with_namespace_owner(&file_path, false, false, owner)
|
||||
.await?;
|
||||
|
||||
@@ -15887,6 +15960,7 @@ impl DiskAPI for LocalDisk {
|
||||
|
||||
#[tracing::instrument(level = "trace", skip_all)]
|
||||
async fn update_metadata(&self, volume: &str, path: &str, fi: FileInfo, opts: &UpdateMetadataOpts) -> Result<()> {
|
||||
let status_only = super::usage_watch::status_only_metadata_update(&fi, opts);
|
||||
multipart_staging::reject_native_control_write(&self.root, self.io_root(), volume, path)?;
|
||||
pool_meta_native::reject_raw_path(&self.root, volume, path)?;
|
||||
let object_path = self.get_object_path(volume, path)?;
|
||||
@@ -15899,13 +15973,18 @@ impl DiskAPI for LocalDisk {
|
||||
condition,
|
||||
)
|
||||
.await?;
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(authority.clone());
|
||||
let owner = self.usage_watch_binding.owner(volume, Some(authority.clone()));
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(os::acquire_metadata_mutation_lease(&object_path, owner).await);
|
||||
return self
|
||||
.reconcile_transition_state_metadata(volume, path, fi.version_id, condition, owner, authority)
|
||||
.await;
|
||||
}
|
||||
let source_owner = self.multipart_source_owner(volume, path, false, true, None).await?;
|
||||
let source_owner = if status_only {
|
||||
source_owner
|
||||
} else {
|
||||
self.usage_watch_binding.owner(volume, source_owner)
|
||||
};
|
||||
let namespace_owner: Option<Arc<dyn Send + Sync>> =
|
||||
Some(os::acquire_metadata_mutation_lease(&object_path, source_owner).await);
|
||||
if !fi.metadata.is_empty() {
|
||||
@@ -16293,18 +16372,35 @@ impl DiskAPI for LocalDisk {
|
||||
// non-recursive rmdir calls. A file that exists during the scan, or
|
||||
// appears before its parent is removed, fails closed with
|
||||
// VolumeNotEmpty. Only an explicit force delete removes recursively.
|
||||
let res = if force_delete {
|
||||
fs::remove_dir_all(&p).await
|
||||
} else {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
remove_empty_directory_tree_under_mount_lease(&self.mount_lease, volume, p.clone()).await
|
||||
let usage_owner = self.usage_watch_binding.owner(volume, None);
|
||||
let physical_path = p.clone();
|
||||
let physical_volume = volume.to_owned();
|
||||
let backend = Arc::clone(&self.io_backend);
|
||||
#[cfg(target_os = "linux")]
|
||||
let mount_lease = Arc::clone(&self.mount_lease);
|
||||
let res = tokio::spawn(async move {
|
||||
let _usage_owner = usage_owner;
|
||||
let _volume_mutation_guard = _volume_mutation_guard;
|
||||
let result = if force_delete {
|
||||
fs::remove_dir_all(&physical_path).await
|
||||
} else {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
remove_empty_directory_tree_under_mount_lease(&mount_lease, &physical_volume, physical_path.clone()).await
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
remove_empty_directory_tree(&physical_path).await
|
||||
}
|
||||
};
|
||||
if result.is_ok() {
|
||||
backend.invalidate_cached_fds_for_volume(&physical_volume);
|
||||
invalidate_bucket_cache(&physical_path);
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
remove_empty_directory_tree(&p).await
|
||||
}
|
||||
};
|
||||
result
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?;
|
||||
|
||||
if let Err(err) = res {
|
||||
let e: DiskError = classify_delete_volume_error(err);
|
||||
@@ -16484,6 +16580,9 @@ async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInf
|
||||
Ok((disk_info, root_drive))
|
||||
}
|
||||
|
||||
#[cfg(all(test, not(windows)))]
|
||||
mod usage_watch_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use super::commit::create_local_inline_rollback_backup;
|
||||
|
||||
@@ -500,6 +500,10 @@ impl LocalDisk {
|
||||
dst_path: &str,
|
||||
state: &mut RenameDataState,
|
||||
) -> Result<RenameDataResp> {
|
||||
state.namespace_owner = self.usage_watch_binding.owner(dst_volume, state.namespace_owner.take());
|
||||
if src_volume != dst_volume {
|
||||
state.namespace_owner = self.usage_watch_binding.owner(src_volume, state.namespace_owner.take());
|
||||
}
|
||||
super::multipart_staging::reject_noncanonical_scope(&self.root, self.io_root(), src_volume, src_path).inspect_err(
|
||||
|_| {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
|
||||
@@ -1827,6 +1827,7 @@ impl LocalDisk {
|
||||
action: DeleteRecoveryAction,
|
||||
owner: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<DeleteRecoveryInspection> {
|
||||
let owner = self.usage_watch_binding.owner(volume, owner);
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if let Ok(root) = self.io_get_object_path(volume, object) {
|
||||
let _ = take_prepared_delete_receipt(&root.join(transaction.to_string()));
|
||||
@@ -2779,6 +2780,7 @@ impl LocalDisk {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<Option<(DeleteRecoveryJournal, bool, bool)>> {
|
||||
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
|
||||
if transaction.is_nil() {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
@@ -2974,6 +2976,7 @@ impl LocalDisk {
|
||||
namespace_owner: Option<Arc<dyn Send + Sync>>,
|
||||
work_budget: Option<Arc<crate::object_api::DeleteRecoveryWorkBudget>>,
|
||||
) -> Result<bool> {
|
||||
let namespace_owner = self.usage_watch_binding.owner(volume, namespace_owner);
|
||||
fn ignore_not_found(res: std::io::Result<()>) -> std::io::Result<()> {
|
||||
match res {
|
||||
Ok(()) => Ok(()),
|
||||
|
||||
@@ -0,0 +1,234 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::runtime::usage_watch::{ScannerUsageWatchRequest, UsageWatchRegistry};
|
||||
|
||||
fn watch_request(bucket: &str) -> ScannerUsageWatchRequest {
|
||||
ScannerUsageWatchRequest {
|
||||
bucket: bucket.into(),
|
||||
bucket_incarnation: Uuid::from_u128(2),
|
||||
watch_id: None,
|
||||
release: false,
|
||||
session_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn fixture() -> (tempfile::TempDir, Arc<LocalDisk>, Arc<UsageWatchRegistry>) {
|
||||
let root = tempfile::tempdir().expect("watch physical fixture directory");
|
||||
let endpoint = Endpoint::try_from(root.path().to_string_lossy().as_ref()).expect("watch physical endpoint");
|
||||
let disk = Arc::new(LocalDisk::new(&endpoint, false).await.expect("watch physical disk"));
|
||||
disk.make_volume("bucket").await.expect("watch test bucket");
|
||||
let registry = Arc::new(UsageWatchRegistry::default());
|
||||
disk.usage_watch_binding.bind(®istry);
|
||||
(root, disk, registry)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn usage_watch_stays_inflight_after_metadata_is_visible_and_waiter_is_canceled() {
|
||||
let (_root, disk, registry) = fixture().await;
|
||||
let target = disk
|
||||
.io_get_object_path("bucket", "object")
|
||||
.expect("target metadata directory");
|
||||
let (entered, wait_for_enter) = tokio::sync::oneshot::channel();
|
||||
let (release, hold) = std::sync::mpsc::channel::<()>();
|
||||
let _hook = os::prepared_publication_test_hooks::install_at(
|
||||
os::prepared_publication_test_hooks::Stage::DirFsync,
|
||||
&target,
|
||||
move || {
|
||||
let _ = entered.send(());
|
||||
let _ = hold.recv();
|
||||
},
|
||||
);
|
||||
let writer_disk = Arc::clone(&disk);
|
||||
let mut writer = tokio::spawn(async move {
|
||||
let info =
|
||||
super::test::test_file_info("object", Uuid::new_v4(), Some(Uuid::new_v4()), Some(Bytes::from_static(b"payload")));
|
||||
writer_disk.write_metadata("", "bucket", "object", info).await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(10), wait_for_enter)
|
||||
.await
|
||||
.expect("reach physical fsync barrier")
|
||||
.expect("barrier entered");
|
||||
assert!(
|
||||
target.join(STORAGE_FORMAT_FILE).exists(),
|
||||
"metadata is already visible at the blocked fsync"
|
||||
);
|
||||
let during = registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("install during active physical mutation");
|
||||
assert!(during.inflight > 0, "installation must include pre-existing writes");
|
||||
writer.abort();
|
||||
assert!((&mut writer).await.expect_err("cancel logical waiter").is_cancelled());
|
||||
let canceled = registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("watch survives logical cancellation");
|
||||
assert!(canceled.inflight > 0, "physical tail retains its guard");
|
||||
drop(release);
|
||||
let settled = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let observation = registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("settlement proof");
|
||||
if observation.inflight == 0 {
|
||||
break observation;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical tail settles");
|
||||
assert!(settled.revision > during.revision);
|
||||
assert!(!settled.unchanged_since(&during));
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-12", "assertion_id": "USAGE-12-A03", "status": "PASS",
|
||||
"evidence": {"inflight_at_install": during.inflight, "inflight_after_waiter_cancel": canceled.inflight, "inflight_after_physical_end": settled.inflight, "revision_at_install": during.revision, "revision_after_physical_end": settled.revision}
|
||||
})
|
||||
);
|
||||
drop(disk);
|
||||
drop(registry);
|
||||
let root_path = _root.path().to_path_buf();
|
||||
drop(_root);
|
||||
assert!(!root_path.exists());
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-12", "status": "PASS", "fixture_removed": !root_path.exists()})
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_watch_status_only_metadata_keeps_maintenance_semantics() {
|
||||
let (_root, disk, registry) = fixture().await;
|
||||
let info = super::test::test_file_info("object", Uuid::new_v4(), Some(Uuid::new_v4()), Some(Bytes::from_static(b"payload")));
|
||||
disk.write_metadata("", "bucket", "object", info.clone())
|
||||
.await
|
||||
.expect("initial counted version");
|
||||
let before = registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("idle watch");
|
||||
let mut status = info.clone();
|
||||
status.metadata.clear();
|
||||
rustfs_utils::http::metadata_compat::insert_str(
|
||||
&mut status.metadata,
|
||||
rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
|
||||
"arn=COMPLETED;".into(),
|
||||
);
|
||||
disk.update_metadata("bucket", "object", status, &UpdateMetadataOpts::default())
|
||||
.await
|
||||
.expect("status-class metadata update");
|
||||
let after = registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("status result watch");
|
||||
assert!(after.unchanged_since(&before), "replication counters retain maintenance cadence");
|
||||
let mut next = info;
|
||||
next.version_id = Some(Uuid::new_v4());
|
||||
disk.write_metadata("", "bucket", "object", next).await.expect("new version");
|
||||
let changed = registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("new version watch");
|
||||
assert!(changed.revision > after.revision);
|
||||
assert_eq!(changed.inflight, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn usage_watch_raw_live_stream_revokes_coverage_and_internal_staging_does_not() {
|
||||
let (_root, disk, registry) = fixture().await;
|
||||
let before = registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("idle watch");
|
||||
disk.usage_watch_binding.mark_uncovered_user_stream(RUSTFS_META_TMP_BUCKET);
|
||||
assert!(
|
||||
registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("internal staging stays covered")
|
||||
.unchanged_since(&before)
|
||||
);
|
||||
let _stream = disk.create_file("bucket", "bucket", "raw-file", 0).await;
|
||||
assert_eq!(registry.observe(Uuid::from_u128(1), &watch_request("bucket")), None);
|
||||
assert!(!disk.usage_watch_binding.matches(®istry));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn usage_watch_delete_paths_retains_ownership_through_canceled_trash_rename() {
|
||||
let (_root, disk, registry) = fixture().await;
|
||||
let target = disk.io_get_object_path("bucket", "delete-me").expect("delete source");
|
||||
std::fs::write(&target, b"payload").expect("initial source");
|
||||
let trash = disk.io_get_bucket_path(RUSTFS_META_TMP_DELETED_BUCKET).expect("trash volume");
|
||||
std::fs::create_dir_all(&trash).expect("trash directory");
|
||||
let before = registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("initial proof");
|
||||
let (entered, wait_for_enter) = tokio::sync::oneshot::channel();
|
||||
let (release, hold) = std::sync::mpsc::channel::<()>();
|
||||
let _hook =
|
||||
os::prepared_publication_test_hooks::install_at(os::prepared_publication_test_hooks::Stage::Rename, &target, move || {
|
||||
let _ = entered.send(());
|
||||
let _ = hold.recv();
|
||||
});
|
||||
let deleting_disk = Arc::clone(&disk);
|
||||
let mut deleting = tokio::spawn(async move { deleting_disk.delete_paths("bucket", &["delete-me".into()]).await });
|
||||
tokio::time::timeout(Duration::from_secs(10), wait_for_enter)
|
||||
.await
|
||||
.expect("reach raw trash rename")
|
||||
.expect("rename entered");
|
||||
deleting.abort();
|
||||
assert!((&mut deleting).await.expect_err("logical waiter canceled").is_cancelled());
|
||||
let during = registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("active proof");
|
||||
assert!(during.inflight > 0, "trash syscall owns the watch after waiter cancellation");
|
||||
assert!(during.revision > before.revision);
|
||||
drop(release);
|
||||
let after = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let observation = registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("draining proof");
|
||||
if observation.inflight == 0 {
|
||||
break observation;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("physical rename settles");
|
||||
assert!(!target.exists());
|
||||
assert!(after.revision > during.revision);
|
||||
assert!(!after.unchanged_since(&before));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_stale_destination_removal_preserves_file_directory_and_symlink_semantics() {
|
||||
let (_root, disk, registry) = fixture().await;
|
||||
let target = disk.io_get_object_path("bucket", "stale").expect("stale destination");
|
||||
std::fs::write(&target, b"stale-file").expect("stale file");
|
||||
os::remove_with_owner(&target, disk.usage_watch_binding.owner("bucket", None))
|
||||
.await
|
||||
.expect("remove stale file");
|
||||
assert!(!target.exists());
|
||||
std::fs::create_dir(&target).expect("stale directory");
|
||||
os::remove_with_owner(&target, disk.usage_watch_binding.owner("bucket", None))
|
||||
.await
|
||||
.expect("remove empty directory");
|
||||
assert!(!target.exists());
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let referent = disk.io_get_object_path("bucket", "retained").expect("symlink referent");
|
||||
std::fs::write(&referent, b"retained").expect("retained file");
|
||||
std::os::unix::fs::symlink(&referent, &target).expect("stale symlink");
|
||||
os::remove_with_owner(&target, disk.usage_watch_binding.owner("bucket", None))
|
||||
.await
|
||||
.expect("remove symlink itself");
|
||||
assert!(!target.exists());
|
||||
assert_eq!(std::fs::read(referent).expect("referent remains"), b"retained");
|
||||
}
|
||||
assert_eq!(
|
||||
registry
|
||||
.observe(Uuid::from_u128(1), &watch_request("bucket"))
|
||||
.expect("all work drained")
|
||||
.inflight,
|
||||
0
|
||||
);
|
||||
}
|
||||
@@ -30,6 +30,7 @@ pub mod local;
|
||||
pub mod multipart_index;
|
||||
pub mod multipart_source;
|
||||
pub mod os;
|
||||
mod usage_watch;
|
||||
|
||||
pub(crate) use io_admission::memory as background_memory;
|
||||
pub(crate) use io_admission::remote as remote_io_admission;
|
||||
@@ -1386,6 +1387,16 @@ impl Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
/// The progress flag is derived from a verified capability scope.
|
||||
pub async fn ns_scanner_progress_capability(
|
||||
&self,
|
||||
) -> Result<Option<crate::storage_api_contracts::internode::NsScannerCapabilities>> {
|
||||
match self {
|
||||
Disk::Local(_) => Ok(None),
|
||||
Disk::Remote(remote_disk) => remote_disk.ns_scanner_progress_capability().await,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn ns_scanner_server_epoch(&self) -> Result<Option<Uuid>> {
|
||||
match self {
|
||||
Disk::Local(_) => Ok(None),
|
||||
|
||||
@@ -2137,7 +2137,7 @@ mod multipart_proof_authority_tests {
|
||||
assert_eq!(expected, snapshot);
|
||||
assert_eq!(actual_round, round);
|
||||
let publication = PublicationIdentity {
|
||||
lineage: round.lineage.clone(),
|
||||
lineage: round.lineage,
|
||||
source_generation: snapshot.generation,
|
||||
attempt: Uuid::new_v4(),
|
||||
version_id: None,
|
||||
@@ -2145,7 +2145,7 @@ mod multipart_proof_authority_tests {
|
||||
modified_unix_nanos: nanos,
|
||||
completion_identity: [6; 32],
|
||||
};
|
||||
snapshot.publication = Some(publication.clone());
|
||||
snapshot.publication = Some(publication);
|
||||
let response = MultipartSourceResponse::TerminalConfirmed(
|
||||
MultipartSourceProof::issue(
|
||||
&authority,
|
||||
|
||||
@@ -384,14 +384,11 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
}
|
||||
|
||||
pub(crate) fn has_any_hooks() -> bool {
|
||||
if !BEFORE_PUBLICATION.lock().is_empty() {
|
||||
return true;
|
||||
}
|
||||
#[cfg(test)]
|
||||
if !RENAME_DESTINATIONS.lock().is_empty() || !HARDLINK_DESTINATIONS.lock().is_empty() {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
!BEFORE_PUBLICATION.lock().is_empty()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -3511,20 +3508,6 @@ pub async fn read_dir(path: impl AsRef<Path>, count: i32) -> std::io::Result<Vec
|
||||
Ok(volumes)
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip_all)]
|
||||
pub async fn rename_all(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
base_dir: impl AsRef<Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> Result<()> {
|
||||
reliable_rename(src_file_path, dst_file_path.as_ref(), base_dir, publication_root)
|
||||
.await
|
||||
.map_err(to_file_error)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Retain mutation ownership through both metadata publication and its directory
|
||||
/// barriers, even when the async caller stops waiting after the rename starts.
|
||||
pub(crate) async fn rename_metadata_and_sync_with_lease(
|
||||
@@ -3569,6 +3552,19 @@ pub(crate) async fn fsync_dir_with_owner(path: &Path, namespace_owner: Option<Ar
|
||||
}
|
||||
}
|
||||
|
||||
/// Preserve file, symlink, and empty-directory removal semantics while the
|
||||
/// actual filesystem executor retains namespace ownership after cancellation.
|
||||
pub(crate) async fn remove_with_owner(path: impl AsRef<Path>, namespace_owner: Option<Arc<dyn Send + Sync>>) -> io::Result<()> {
|
||||
let path = path.as_ref().to_path_buf();
|
||||
let lease = acquire_namespace_mutation_lease_with_owner(&path, namespace_owner).await;
|
||||
run_blocking_namespace_operation(lease, move || {
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
prepared_publication_test_hooks::run(prepared_publication_test_hooks::Stage::Remove, &path);
|
||||
super::fs::remove_std(path)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Retain namespace ownership in the actual filesystem executor after timeout.
|
||||
pub(crate) async fn remove_file_with_owner(
|
||||
path: impl AsRef<Path>,
|
||||
@@ -4635,15 +4631,6 @@ pub(crate) fn rename_source_is_missing(src_file_path: &Path, _publication_root:
|
||||
matches!(std::fs::symlink_metadata(src_file_path), Err(err) if err.kind() == io::ErrorKind::NotFound)
|
||||
}
|
||||
|
||||
async fn reliable_rename(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
base_dir: impl AsRef<Path>,
|
||||
publication_root: &PublicationRoot,
|
||||
) -> io::Result<()> {
|
||||
reliable_rename_inner(src_file_path, dst_file_path, base_dir, publication_root, true).await
|
||||
}
|
||||
|
||||
async fn reliable_rename_inner(
|
||||
src_file_path: impl AsRef<Path>,
|
||||
dst_file_path: impl AsRef<Path>,
|
||||
@@ -8658,7 +8645,7 @@ mod tests {
|
||||
let dst_file_path = dst_file_path.as_ref();
|
||||
let base_dir = base_dir.as_ref();
|
||||
let publication_root = test_publication_root(&[src_file_path, dst_file_path, base_dir]);
|
||||
super::rename_all(src_file_path, dst_file_path, base_dir, &publication_root).await
|
||||
super::rename_all_with_owner(src_file_path, dst_file_path, base_dir, &publication_root, None).await
|
||||
}
|
||||
|
||||
async fn rename_all_ignore_missing_source(
|
||||
@@ -9503,7 +9490,7 @@ mod tests {
|
||||
let configured_dst = configured_base.join("object");
|
||||
std::fs::create_dir(&configured_base).expect("create bucket through configured root");
|
||||
std::fs::write(&configured_src, b"configured").expect("write staged object through configured root");
|
||||
super::rename_all(&configured_src, &configured_dst, &configured_base, &publication_root)
|
||||
super::rename_all_with_owner(&configured_src, &configured_dst, &configured_base, &publication_root, None)
|
||||
.await
|
||||
.expect("publish a configured path relative to the pinned root");
|
||||
assert_eq!(
|
||||
@@ -9516,7 +9503,7 @@ mod tests {
|
||||
let resolved_dst = resolved_base.join("object");
|
||||
std::fs::create_dir(&resolved_base).expect("create bucket through resolved root");
|
||||
std::fs::write(&resolved_src, b"resolved").expect("write staged object through resolved root");
|
||||
super::rename_all(&resolved_src, &resolved_dst, &resolved_base, &publication_root)
|
||||
super::rename_all_with_owner(&resolved_src, &resolved_dst, &resolved_base, &publication_root, None)
|
||||
.await
|
||||
.expect("publish a resolved path relative to the pinned root");
|
||||
|
||||
@@ -9936,7 +9923,7 @@ mod tests {
|
||||
std::fs::write(&dst, b"uncommitted-metadata").expect("write metadata to replace");
|
||||
let publication_root = PublicationRoot::new(temp_dir.path()).expect("open publication root above the object tree");
|
||||
|
||||
super::rename_all(&src, &dst, &object_dir, &publication_root)
|
||||
super::rename_all_with_owner(&src, &dst, &object_dir, &publication_root, None)
|
||||
.await
|
||||
.expect("a rollback source below its destination parent must not conflict with its own guards");
|
||||
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Explicit instance binding at the physical disk boundary. Lock order is this
|
||||
//! binding mutex, then the registry mutex; registry code never locks a disk.
|
||||
|
||||
use crate::runtime::usage_watch::{UsageWatchMutationGuard, UsageWatchRegistry};
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
|
||||
/// Existing-version replication status writes change neither size nor version
|
||||
/// membership. Replication aggregates keep their existing maintenance cadence.
|
||||
pub(crate) fn status_only_metadata_update(fi: &rustfs_filemeta::FileInfo, opts: &super::UpdateMetadataOpts) -> bool {
|
||||
use rustfs_utils::http::*;
|
||||
!opts.replace_user_metadata
|
||||
&& opts.transition_reconcile.is_none()
|
||||
&& !fi.metadata.is_empty()
|
||||
&& fi.metadata.keys().all(|key| {
|
||||
strip_internal_prefix_preserving_case(key).is_some_and(|suffix| {
|
||||
[
|
||||
SUFFIX_REPLICA_STATUS,
|
||||
SUFFIX_REPLICA_TIMESTAMP,
|
||||
SUFFIX_REPLICATION_STATUS,
|
||||
SUFFIX_REPLICATION_TIMESTAMP,
|
||||
SUFFIX_REPLICATION_GENERATION,
|
||||
SUFFIX_REPLICATION_RESET,
|
||||
SUFFIX_PURGESTATUS,
|
||||
]
|
||||
.iter()
|
||||
.any(|known| suffix.eq_ignore_ascii_case(known))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct BindingState {
|
||||
registry: Option<Weak<UsageWatchRegistry>>,
|
||||
unbound_inflight: u64,
|
||||
prebinding_guard: Option<UsageWatchMutationGuard>,
|
||||
conflicted: bool,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct DiskUsageWatchBinding {
|
||||
state: Mutex<BindingState>,
|
||||
}
|
||||
|
||||
struct UnboundGuard(Arc<DiskUsageWatchBinding>);
|
||||
|
||||
impl Drop for UnboundGuard {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self.0.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if state.unbound_inflight != 0 {
|
||||
state.unbound_inflight -= 1;
|
||||
}
|
||||
if state.unbound_inflight == 0 {
|
||||
state.prebinding_guard = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DiskUsageWatchBinding {
|
||||
pub(crate) fn mark_uncovered_user_stream(&self, bucket: &str) {
|
||||
if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
return;
|
||||
}
|
||||
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
state.conflicted = true;
|
||||
if let Some(registry) = state.registry.as_ref().and_then(Weak::upgrade) {
|
||||
registry.mark_coverage_unavailable();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn bind(&self, registry: &Arc<UsageWatchRegistry>) {
|
||||
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if let Some(existing) = state.registry.as_ref().and_then(Weak::upgrade) {
|
||||
if !Arc::ptr_eq(&existing, registry) {
|
||||
state.conflicted = true;
|
||||
// A query may already have checked this binding before
|
||||
// awaiting disk metadata. Never allow it to reinstall a
|
||||
// proof after this permanently ambiguous ownership change.
|
||||
existing.mark_coverage_unavailable();
|
||||
registry.mark_coverage_unavailable();
|
||||
}
|
||||
return;
|
||||
}
|
||||
// A newly registered disk can have pre-registration physical tails.
|
||||
// Keep the whole registry Unknown until those exact guards drain.
|
||||
if state.unbound_inflight != 0 {
|
||||
state.prebinding_guard = Some(registry.begin(None));
|
||||
}
|
||||
if state.conflicted {
|
||||
registry.mark_coverage_unavailable();
|
||||
}
|
||||
state.registry = Some(Arc::downgrade(registry));
|
||||
}
|
||||
|
||||
pub(crate) fn matches(&self, registry: &Arc<UsageWatchRegistry>) -> bool {
|
||||
let state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
!state.conflicted
|
||||
&& state
|
||||
.registry
|
||||
.as_ref()
|
||||
.and_then(Weak::upgrade)
|
||||
.is_some_and(|bound| Arc::ptr_eq(&bound, registry))
|
||||
}
|
||||
|
||||
pub(crate) fn owner(self: &Arc<Self>, bucket: &str, parent: Option<Arc<dyn Send + Sync>>) -> Option<Arc<dyn Send + Sync>> {
|
||||
if crate::bucket::utils::is_meta_bucketname(bucket) {
|
||||
return parent;
|
||||
}
|
||||
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
let guard: Arc<dyn Send + Sync> = if let Some(registry) = state.registry.as_ref().and_then(Weak::upgrade) {
|
||||
Arc::new(registry.begin((!state.conflicted).then_some(bucket)))
|
||||
} else {
|
||||
match state.unbound_inflight.checked_add(1) {
|
||||
Some(count) => state.unbound_inflight = count,
|
||||
None => state.conflicted = true,
|
||||
}
|
||||
Arc::new(UnboundGuard(Arc::clone(self)))
|
||||
};
|
||||
Some(match parent {
|
||||
Some(parent) => Arc::new((parent, guard)),
|
||||
None => guard,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::runtime::usage_watch::ScannerUsageWatchRequest;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[test]
|
||||
fn binding_preserves_mutations_that_started_before_registration() {
|
||||
let binding = Arc::new(DiskUsageWatchBinding::default());
|
||||
let registry = Arc::new(UsageWatchRegistry::default());
|
||||
let tail = binding.owner("bucket", None);
|
||||
binding.bind(®istry);
|
||||
assert!(binding.matches(®istry));
|
||||
let request = ScannerUsageWatchRequest {
|
||||
bucket: "bucket".into(),
|
||||
bucket_incarnation: Uuid::from_u128(2),
|
||||
watch_id: None,
|
||||
release: false,
|
||||
session_id: None,
|
||||
};
|
||||
assert_eq!(registry.observe(Uuid::from_u128(1), &request), None);
|
||||
drop(tail);
|
||||
assert_eq!(
|
||||
registry
|
||||
.observe(Uuid::from_u128(1), &request)
|
||||
.expect("drained binding")
|
||||
.inflight,
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn composite_owner_and_instance_conflicts_fail_closed() {
|
||||
let binding = Arc::new(DiskUsageWatchBinding::default());
|
||||
let registry = Arc::new(UsageWatchRegistry::default());
|
||||
binding.bind(®istry);
|
||||
let request = ScannerUsageWatchRequest {
|
||||
bucket: "bucket".into(),
|
||||
bucket_incarnation: Uuid::from_u128(2),
|
||||
watch_id: None,
|
||||
release: false,
|
||||
session_id: None,
|
||||
};
|
||||
registry.observe(Uuid::from_u128(1), &request).expect("initial watch");
|
||||
let owner = binding.owner("bucket", None).expect("physical owner");
|
||||
let nested = binding.owner("bucket", Some(owner));
|
||||
assert_eq!(
|
||||
registry
|
||||
.observe(Uuid::from_u128(1), &request)
|
||||
.expect("both operations active")
|
||||
.inflight,
|
||||
2
|
||||
);
|
||||
drop(nested);
|
||||
assert_eq!(
|
||||
registry
|
||||
.observe(Uuid::from_u128(1), &request)
|
||||
.expect("drained owners")
|
||||
.inflight,
|
||||
0
|
||||
);
|
||||
let other = Arc::new(UsageWatchRegistry::default());
|
||||
binding.bind(&other);
|
||||
assert!(!binding.matches(®istry));
|
||||
assert!(!binding.matches(&other));
|
||||
assert_eq!(registry.observe(Uuid::from_u128(1), &request), None);
|
||||
assert_eq!(other.observe(Uuid::from_u128(1), &request), None);
|
||||
}
|
||||
}
|
||||
@@ -439,7 +439,7 @@ mod tests {
|
||||
reconstruct_with(&engine, &mut shards).expect("restore missing data and parity");
|
||||
assert_eq!(shards, expected, "every restored byte must match at width {}", shards.len());
|
||||
|
||||
let mut corrupt = expected.clone();
|
||||
let mut corrupt = expected;
|
||||
corrupt[0] = None;
|
||||
corrupt[data_shards] = None;
|
||||
corrupt[data_shards + 1].as_mut().expect("surplus parity present")[0] ^= 1;
|
||||
|
||||
@@ -2002,7 +2002,7 @@ mod tests {
|
||||
|
||||
for storage_error in storage_errors {
|
||||
let original_message = storage_error.to_string();
|
||||
let io_error: std::io::Error = storage_error.clone().into();
|
||||
let io_error: std::io::Error = storage_error.into();
|
||||
|
||||
// The io::Error should contain the original error message or info
|
||||
assert!(io_error.to_string().contains(&original_message));
|
||||
|
||||
@@ -391,7 +391,7 @@ impl PoolEndpointList {
|
||||
e.get_mut().extend(host_ip_set.iter());
|
||||
}
|
||||
Entry::Vacant(e) => {
|
||||
e.insert(host_ip_set.clone());
|
||||
e.insert(host_ip_set);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ pub(crate) fn formats_to_drives_info(
|
||||
before_drives.push(HealDriveInfo {
|
||||
uuid,
|
||||
endpoint: drive,
|
||||
state: state.to_string(),
|
||||
state,
|
||||
});
|
||||
}
|
||||
before_drives
|
||||
|
||||
@@ -99,6 +99,7 @@ pub(crate) struct ScannerPublicationLeaseEntry {
|
||||
pub(crate) struct NamespaceCommitGuard {
|
||||
ctx: Arc<InstanceContext>,
|
||||
counted: bool,
|
||||
_usage_watch: super::usage_watch::UsageWatchMutationGuard,
|
||||
}
|
||||
|
||||
impl Drop for NamespaceCommitGuard {
|
||||
@@ -365,6 +366,7 @@ pub struct InstanceContext {
|
||||
scanner_publication_state: AtomicU8,
|
||||
namespace_commits: AtomicU64,
|
||||
namespace_commit_generation: AtomicU64,
|
||||
pub(crate) usage_watches: Arc<super::usage_watch::UsageWatchRegistry>,
|
||||
/// Resolves object-encryption material at the application boundary.
|
||||
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
|
||||
delete_recovery_hints: DeleteRecoveryHints,
|
||||
@@ -444,6 +446,7 @@ impl InstanceContext {
|
||||
scanner_publication_state: AtomicU8::new(SCANNER_PUBLICATION_STATE_UNKNOWN),
|
||||
namespace_commits: AtomicU64::new(0),
|
||||
namespace_commit_generation: AtomicU64::new(0),
|
||||
usage_watches: Arc::new(super::usage_watch::UsageWatchRegistry::default()),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
delete_recovery_hints: DeleteRecoveryHints::default(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
@@ -683,6 +686,15 @@ impl InstanceContext {
|
||||
}
|
||||
|
||||
pub(crate) fn begin_namespace_commit(self: &Arc<Self>) -> Arc<NamespaceCommitGuard> {
|
||||
self.begin_namespace_commit_scoped(None)
|
||||
}
|
||||
|
||||
pub(crate) fn begin_namespace_commit_for_bucket(self: &Arc<Self>, bucket: &str) -> Arc<NamespaceCommitGuard> {
|
||||
self.begin_namespace_commit_scoped(Some(bucket))
|
||||
}
|
||||
|
||||
fn begin_namespace_commit_scoped(self: &Arc<Self>, bucket: Option<&str>) -> Arc<NamespaceCommitGuard> {
|
||||
let usage_watch = self.usage_watches.begin(bucket);
|
||||
let counted = self
|
||||
.namespace_commits
|
||||
.try_update(Ordering::AcqRel, Ordering::Acquire, |count| count.checked_add(1))
|
||||
@@ -695,6 +707,7 @@ impl InstanceContext {
|
||||
Arc::new(NamespaceCommitGuard {
|
||||
ctx: Arc::clone(self),
|
||||
counted,
|
||||
_usage_watch: usage_watch,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -17,3 +17,4 @@
|
||||
pub(crate) mod global;
|
||||
pub(crate) mod instance;
|
||||
pub(crate) mod sources;
|
||||
pub(crate) mod usage_watch;
|
||||
|
||||
@@ -525,8 +525,11 @@ pub(crate) async fn record_local_disks(instance_ctx: &Arc<InstanceContext>, disk
|
||||
let map = instance_ctx.local_disk_map();
|
||||
let mut global_local_disk_map = map.write().await;
|
||||
for disk in disks {
|
||||
if let crate::disk::Disk::Local(wrapper) = disk.as_ref() {
|
||||
wrapper.get_disk().usage_watch_binding.bind(&instance_ctx.usage_watches);
|
||||
}
|
||||
let path = disk.endpoint().to_string();
|
||||
global_local_disk_map.insert(path, Some(disk.clone()));
|
||||
global_local_disk_map.insert(path, Some(disk));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -611,6 +614,7 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
Err(err) => return Err(err.into()),
|
||||
};
|
||||
if let crate::disk::Disk::Local(wrapper) = disk.as_ref() {
|
||||
wrapper.get_disk().usage_watch_binding.bind(&instance_ctx.usage_watches);
|
||||
wrapper.get_disk().bind_multipart_proof_authority(&multipart_authority)?;
|
||||
}
|
||||
let path = disk.endpoint().to_string();
|
||||
@@ -625,7 +629,7 @@ pub(crate) async fn initialize_local_disk_maps(
|
||||
})?;
|
||||
|
||||
global_local_disk_map.insert(path, Some(disk.clone()));
|
||||
global_set_drives[pool_idx][set_idx][disk_idx] = Some(disk.clone());
|
||||
global_set_drives[pool_idx][set_idx][disk_idx] = Some(disk);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Bounded, same-session bucket mutation watches. The registry lock is a short
|
||||
//! synchronous critical section shared by installation, mutation begin/end,
|
||||
//! and observation. No I/O or await occurs while it is held.
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, Mutex},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const SCANNER_USAGE_WATCH_COVERAGE_VERSION: u32 = rustfs_protos::usage_watch::USAGE_WATCH_COVERAGE_VERSION;
|
||||
pub const SCANNER_USAGE_WATCH_MAX_ENTRIES: usize = 256;
|
||||
pub const SCANNER_USAGE_WATCH_MAX_ACTIVE_BUCKETS: usize = 4096;
|
||||
pub const SCANNER_USAGE_WATCH_MAX_NAME_BYTES: usize = 16 * 1024;
|
||||
pub const SCANNER_USAGE_WATCH_MAX_ACTIVE_NAME_BYTES: usize = 256 * 1024;
|
||||
pub const SCANNER_USAGE_WATCH_LEASE_MS: u64 = rustfs_protos::usage_watch::USAGE_WATCH_MAX_LEASE_MS;
|
||||
|
||||
/// A missing watch ID installs a new watch; a supplied ID can only refresh the
|
||||
/// exact, still-live token. An expired token never silently creates a new one.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct ScannerUsageWatchRequest {
|
||||
pub bucket: String,
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub watch_id: Option<Uuid>,
|
||||
/// Release only this exact producer session and token after publication.
|
||||
pub release: bool,
|
||||
pub session_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub struct ScannerUsageWatchKnown {
|
||||
pub owner_id: Uuid,
|
||||
pub session_id: Uuid,
|
||||
pub watch_id: Uuid,
|
||||
pub bucket_incarnation: Uuid,
|
||||
pub revision: u64,
|
||||
pub inflight: u64,
|
||||
pub coverage_version: u32,
|
||||
pub coverage_epoch: u64,
|
||||
/// Duration from the receiver's observation, never a wall-clock deadline.
|
||||
pub lease_remaining_ms: u64,
|
||||
}
|
||||
|
||||
impl ScannerUsageWatchKnown {
|
||||
pub fn is_well_formed(&self) -> bool {
|
||||
!self.owner_id.is_nil()
|
||||
&& !self.session_id.is_nil()
|
||||
&& !self.watch_id.is_nil()
|
||||
&& !self.bucket_incarnation.is_nil()
|
||||
&& self.revision != 0
|
||||
&& self.revision != u64::MAX
|
||||
&& self.inflight != u64::MAX
|
||||
&& self.coverage_version == SCANNER_USAGE_WATCH_COVERAGE_VERSION
|
||||
&& self.coverage_epoch != 0
|
||||
&& self.coverage_epoch != u64::MAX
|
||||
&& self.lease_remaining_ms != 0
|
||||
&& self.lease_remaining_ms <= SCANNER_USAGE_WATCH_LEASE_MS
|
||||
}
|
||||
|
||||
/// Equality of stable proof fields; the caller must also respect its
|
||||
/// conservative local lease deadline and validate the source/owner set.
|
||||
pub fn unchanged_since(&self, earlier: &Self) -> bool {
|
||||
self.is_well_formed()
|
||||
&& earlier.is_well_formed()
|
||||
&& self.owner_id == earlier.owner_id
|
||||
&& self.session_id == earlier.session_id
|
||||
&& self.watch_id == earlier.watch_id
|
||||
&& self.bucket_incarnation == earlier.bucket_incarnation
|
||||
&& self.revision == earlier.revision
|
||||
&& self.inflight == 0
|
||||
&& earlier.inflight == 0
|
||||
&& self.coverage_version == SCANNER_USAGE_WATCH_COVERAGE_VERSION
|
||||
&& self.coverage_version == earlier.coverage_version
|
||||
&& self.coverage_epoch == earlier.coverage_epoch
|
||||
&& self.lease_remaining_ms != 0
|
||||
&& earlier.lease_remaining_ms != 0
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerUsageWatchObservation {
|
||||
pub bucket: String,
|
||||
/// Missing means Unknown. It is never a revision-zero clean value.
|
||||
pub known: Option<ScannerUsageWatchKnown>,
|
||||
}
|
||||
|
||||
struct Watch {
|
||||
owner: Uuid,
|
||||
token: Uuid,
|
||||
incarnation: Uuid,
|
||||
revision: u64,
|
||||
expires_at: Instant,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct State {
|
||||
watches: HashMap<Arc<str>, Watch>,
|
||||
active: HashMap<Arc<str>, u64>,
|
||||
watch_name_bytes: usize,
|
||||
active_name_bytes: usize,
|
||||
untracked: u64,
|
||||
exhausted: bool,
|
||||
coverage_epoch: u64,
|
||||
}
|
||||
|
||||
pub(crate) struct UsageWatchRegistry {
|
||||
session: Uuid,
|
||||
state: Mutex<State>,
|
||||
}
|
||||
|
||||
enum Mutation {
|
||||
Tracked(Arc<str>),
|
||||
Untracked,
|
||||
}
|
||||
|
||||
/// The physical mutation owner retains this value through all detached tails.
|
||||
pub(crate) struct UsageWatchMutationGuard {
|
||||
registry: Arc<UsageWatchRegistry>,
|
||||
mutation: Mutation,
|
||||
}
|
||||
|
||||
impl Default for UsageWatchRegistry {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
session: Uuid::new_v4(),
|
||||
state: Mutex::new(State {
|
||||
coverage_epoch: 1,
|
||||
..Default::default()
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl State {
|
||||
fn invalidate_all(&mut self) {
|
||||
self.watches.clear();
|
||||
self.watch_name_bytes = 0;
|
||||
match self.coverage_epoch.checked_add(1).filter(|epoch| *epoch != u64::MAX) {
|
||||
Some(epoch) => self.coverage_epoch = epoch,
|
||||
None => self.exhausted = true,
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_bucket(&mut self, bucket: &str) {
|
||||
if let Some(watch) = self.watches.get_mut(bucket) {
|
||||
if let Some(next) = watch.revision.checked_add(1).filter(|next| *next != u64::MAX) {
|
||||
watch.revision = next;
|
||||
} else {
|
||||
self.remove_watch(bucket);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_watch(&mut self, bucket: &str) {
|
||||
if let Some((name, _)) = self.watches.remove_entry(bucket) {
|
||||
self.watch_name_bytes -= name.len();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl UsageWatchRegistry {
|
||||
pub(crate) fn session_id(&self) -> Uuid {
|
||||
self.session
|
||||
}
|
||||
pub(crate) fn begin(self: &Arc<Self>, bucket: Option<&str>) -> UsageWatchMutationGuard {
|
||||
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
// Internal metadata is outside the user bucket counters. Callers with
|
||||
// an unknown user scope must pass None, which invalidates every watch.
|
||||
let bucket = bucket.filter(|bucket| !bucket.is_empty() && bucket.len() <= 63);
|
||||
let tracked = bucket.and_then(|bucket| {
|
||||
if let Some((key, count)) = state.active.get_key_value(bucket) {
|
||||
let key = Arc::clone(key);
|
||||
let next = count.checked_add(1)?;
|
||||
state.active.insert(Arc::clone(&key), next);
|
||||
return Some(key);
|
||||
}
|
||||
if state.active.len() >= SCANNER_USAGE_WATCH_MAX_ACTIVE_BUCKETS
|
||||
|| state.active_name_bytes + bucket.len() > SCANNER_USAGE_WATCH_MAX_ACTIVE_NAME_BYTES
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let key: Arc<str> = Arc::from(bucket);
|
||||
state.active_name_bytes += bucket.len();
|
||||
state.active.insert(Arc::clone(&key), 1);
|
||||
Some(key)
|
||||
});
|
||||
let mutation = match tracked {
|
||||
Some(bucket) => {
|
||||
state.advance_bucket(&bucket);
|
||||
Mutation::Tracked(bucket)
|
||||
}
|
||||
None => {
|
||||
state.invalidate_all();
|
||||
match state.untracked.checked_add(1) {
|
||||
Some(count) => state.untracked = count,
|
||||
None => state.exhausted = true,
|
||||
}
|
||||
Mutation::Untracked
|
||||
}
|
||||
};
|
||||
UsageWatchMutationGuard {
|
||||
registry: Arc::clone(self),
|
||||
mutation,
|
||||
}
|
||||
}
|
||||
|
||||
/// Replay and unsupported runtime mode transitions revoke tokens. They
|
||||
/// never copy a durable generation into a same-session watch revision.
|
||||
pub(crate) fn invalidate(&self) {
|
||||
self.state
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
.invalidate_all();
|
||||
}
|
||||
|
||||
/// A write mode without a physical completion owner cannot establish
|
||||
/// same-session coverage again. Keep the global fallback until restart.
|
||||
pub(crate) fn mark_coverage_unavailable(&self) {
|
||||
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
state.exhausted = true;
|
||||
state.invalidate_all();
|
||||
}
|
||||
|
||||
pub(crate) fn observe(&self, owner: Uuid, request: &ScannerUsageWatchRequest) -> Option<ScannerUsageWatchKnown> {
|
||||
self.observe_at(owner, request, Instant::now())
|
||||
}
|
||||
|
||||
fn observe_at(&self, owner: Uuid, request: &ScannerUsageWatchRequest, now: Instant) -> Option<ScannerUsageWatchKnown> {
|
||||
if owner.is_nil() || request.bucket_incarnation.is_nil() || request.bucket.is_empty() || request.bucket.len() > 63 {
|
||||
return None;
|
||||
}
|
||||
let mut state = self.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if request.release {
|
||||
if request.session_id == Some(self.session)
|
||||
&& state.watches.get(request.bucket.as_str()).is_some_and(|watch| {
|
||||
watch.owner == owner
|
||||
&& watch.incarnation == request.bucket_incarnation
|
||||
&& request.watch_id == Some(watch.token)
|
||||
})
|
||||
{
|
||||
state.remove_watch(&request.bucket);
|
||||
}
|
||||
// Release is resource retirement, never an assertion of cleanliness.
|
||||
return None;
|
||||
}
|
||||
if request.session_id.is_some_and(|session| session != self.session) {
|
||||
return None;
|
||||
}
|
||||
if state.exhausted || state.untracked != 0 {
|
||||
return None;
|
||||
}
|
||||
// Sweeps are bounded by the fixed watch limit. Never accumulate tombstones.
|
||||
state.watches.retain(|_, watch| watch.expires_at > now);
|
||||
state.watch_name_bytes = state.watches.keys().map(|key| key.len()).sum();
|
||||
let matches = state.watches.get(request.bucket.as_str()).is_some_and(|watch| {
|
||||
watch.owner == owner
|
||||
&& watch.incarnation == request.bucket_incarnation
|
||||
&& request.watch_id.is_none_or(|id| id == watch.token)
|
||||
});
|
||||
if !matches {
|
||||
if request.watch_id.is_some() {
|
||||
return None;
|
||||
}
|
||||
state.remove_watch(&request.bucket);
|
||||
if state.watches.len() >= SCANNER_USAGE_WATCH_MAX_ENTRIES
|
||||
|| state.watch_name_bytes + request.bucket.len() > SCANNER_USAGE_WATCH_MAX_NAME_BYTES
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let key: Arc<str> = Arc::from(request.bucket.as_str());
|
||||
state.watch_name_bytes += key.len();
|
||||
state.watches.insert(
|
||||
key,
|
||||
Watch {
|
||||
owner,
|
||||
token: Uuid::new_v4(),
|
||||
incarnation: request.bucket_incarnation,
|
||||
revision: 1,
|
||||
expires_at: now + Duration::from_millis(SCANNER_USAGE_WATCH_LEASE_MS),
|
||||
},
|
||||
);
|
||||
}
|
||||
let inflight = state.active.get(request.bucket.as_str()).copied().unwrap_or(0);
|
||||
let coverage_epoch = state.coverage_epoch;
|
||||
let watch = state.watches.get_mut(request.bucket.as_str())?;
|
||||
watch.expires_at = now + Duration::from_millis(SCANNER_USAGE_WATCH_LEASE_MS);
|
||||
Some(ScannerUsageWatchKnown {
|
||||
owner_id: owner,
|
||||
session_id: self.session,
|
||||
watch_id: watch.token,
|
||||
bucket_incarnation: watch.incarnation,
|
||||
revision: watch.revision,
|
||||
inflight,
|
||||
coverage_version: SCANNER_USAGE_WATCH_COVERAGE_VERSION,
|
||||
coverage_epoch,
|
||||
lease_remaining_ms: SCANNER_USAGE_WATCH_LEASE_MS,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for UsageWatchMutationGuard {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self.registry.state.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
match &self.mutation {
|
||||
Mutation::Tracked(bucket) => {
|
||||
// Advance before exposing inflight=0, including canceled and
|
||||
// failed operations whose physical outcome may be uncertain.
|
||||
state.advance_bucket(bucket);
|
||||
if let Some(count) = state.active.get_mut(bucket) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
state.active.remove(bucket);
|
||||
state.active_name_bytes -= bucket.len();
|
||||
}
|
||||
} else {
|
||||
state.exhausted = true;
|
||||
state.invalidate_all();
|
||||
}
|
||||
}
|
||||
Mutation::Untracked => {
|
||||
if state.untracked != 0 {
|
||||
state.untracked -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,321 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
fn request(bucket: &str) -> ScannerUsageWatchRequest {
|
||||
ScannerUsageWatchRequest {
|
||||
bucket: bucket.into(),
|
||||
bucket_incarnation: Uuid::from_u128(7),
|
||||
watch_id: None,
|
||||
release: false,
|
||||
session_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn query(registry: &UsageWatchRegistry, request: &ScannerUsageWatchRequest, at: Instant) -> ScannerUsageWatchKnown {
|
||||
registry
|
||||
.observe_at(Uuid::from_u128(1), request, at)
|
||||
.expect("known bucket proof")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn watch_installation_detects_an_already_active_mutation() {
|
||||
let registry = Arc::new(UsageWatchRegistry::default());
|
||||
let now = Instant::now();
|
||||
let mutation = registry.begin(Some("bucket"));
|
||||
let before = query(®istry, &request("bucket"), now);
|
||||
assert_eq!(before.inflight, 1);
|
||||
drop(mutation);
|
||||
let after = query(®istry, &request("bucket"), now);
|
||||
assert_eq!(after.inflight, 0);
|
||||
assert!(after.revision > before.revision);
|
||||
assert!(!after.unchanged_since(&before));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_begin_and_physical_tail_end_advances_only_the_target_bucket() {
|
||||
let registry = Arc::new(UsageWatchRegistry::default());
|
||||
let now = Instant::now();
|
||||
let before = query(®istry, &request("cold"), now);
|
||||
let hot = registry.begin(Some("hot"));
|
||||
assert!(query(®istry, &request("cold"), now).unchanged_since(&before));
|
||||
drop(hot);
|
||||
assert!(query(®istry, &request("cold"), now).unchanged_since(&before));
|
||||
let owner = Arc::new(registry.begin(Some("cold")));
|
||||
let physical_tail = Arc::clone(&owner);
|
||||
let during = query(®istry, &request("cold"), now);
|
||||
assert_eq!(during.revision, before.revision + 1);
|
||||
assert_eq!(during.inflight, 1);
|
||||
drop(owner);
|
||||
assert_eq!(query(®istry, &request("cold"), now).inflight, 1);
|
||||
drop(physical_tail);
|
||||
let after = query(®istry, &request("cold"), now);
|
||||
assert_eq!(after.revision, before.revision + 2);
|
||||
assert_eq!(after.inflight, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn expiry_reincarnation_and_restart_do_not_resurrect_old_tokens() {
|
||||
let registry = UsageWatchRegistry::default();
|
||||
let now = Instant::now();
|
||||
let before = query(®istry, &request("bucket"), now);
|
||||
let mut refresh = request("bucket");
|
||||
refresh.watch_id = Some(before.watch_id);
|
||||
let expired = now + Duration::from_millis(SCANNER_USAGE_WATCH_LEASE_MS);
|
||||
assert_eq!(registry.observe_at(Uuid::from_u128(1), &refresh, expired), None);
|
||||
let after = query(®istry, &request("bucket"), expired);
|
||||
assert_ne!(after.watch_id, before.watch_id);
|
||||
assert!(!after.unchanged_since(&before));
|
||||
refresh.bucket_incarnation = Uuid::new_v4();
|
||||
assert_eq!(registry.observe_at(Uuid::from_u128(1), &refresh, expired), None);
|
||||
refresh.watch_id = None;
|
||||
let recreated = query(®istry, &refresh, expired);
|
||||
assert_ne!(recreated.watch_id, after.watch_id);
|
||||
let restarted = UsageWatchRegistry::default();
|
||||
let restart_proof = query(&restarted, &refresh, expired);
|
||||
assert!(!restart_proof.unchanged_since(&recreated));
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-12", "assertion_id": "USAGE-12-A05", "status": "PASS",
|
||||
"evidence": {"expired_token": before.watch_id, "replacement_token": after.watch_id, "lease_ms": SCANNER_USAGE_WATCH_LEASE_MS, "expired_proof_accepted": false}
|
||||
})
|
||||
);
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-12", "assertion_id": "USAGE-12-A06", "status": "PASS",
|
||||
"evidence": {"previous_session": recreated.session_id, "new_session": restart_proof.session_id, "unchanged": restart_proof.unchanged_since(&recreated)}
|
||||
})
|
||||
);
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-12", "status": "PASS", "scope": "process-local"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn watch_capacity_is_bounded_and_rejected_entries_are_unknown() {
|
||||
let registry = UsageWatchRegistry::default();
|
||||
let now = Instant::now();
|
||||
for index in 0..SCANNER_USAGE_WATCH_MAX_ENTRIES {
|
||||
query(®istry, &request(&format!("bucket-{index}")), now);
|
||||
}
|
||||
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("overflow"), now), None);
|
||||
let state = registry.state.lock().expect("registry state");
|
||||
assert_eq!(state.watches.len(), SCANNER_USAGE_WATCH_MAX_ENTRIES);
|
||||
assert!(state.watch_name_bytes <= SCANNER_USAGE_WATCH_MAX_NAME_BYTES);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn active_mutation_overflow_revokes_all_proofs_until_untracked_work_ends() {
|
||||
let registry = Arc::new(UsageWatchRegistry::default());
|
||||
let now = Instant::now();
|
||||
let old = query(®istry, &request("cold"), now);
|
||||
let mut active = Vec::new();
|
||||
for index in 0..SCANNER_USAGE_WATCH_MAX_ACTIVE_BUCKETS {
|
||||
active.push(registry.begin(Some(&format!("active-{index}"))));
|
||||
}
|
||||
let overflow = registry.begin(Some("not-in-active-map"));
|
||||
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("cold"), now), None);
|
||||
{
|
||||
let state = registry.state.lock().expect("registry state");
|
||||
assert_eq!(state.active.len(), SCANNER_USAGE_WATCH_MAX_ACTIVE_BUCKETS);
|
||||
assert!(state.active_name_bytes <= SCANNER_USAGE_WATCH_MAX_ACTIVE_NAME_BYTES);
|
||||
}
|
||||
drop(overflow);
|
||||
let new = query(®istry, &request("cold"), now);
|
||||
assert_ne!(new.watch_id, old.watch_id);
|
||||
assert!(new.coverage_epoch > old.coverage_epoch);
|
||||
drop(active);
|
||||
println!(
|
||||
"usage_assertion: {}",
|
||||
serde_json::json!({
|
||||
"case_id": "USAGE-12", "assertion_id": "USAGE-12-A04", "status": "PASS",
|
||||
"evidence": {"active_capacity": SCANNER_USAGE_WATCH_MAX_ACTIVE_BUCKETS, "previous_token": old.watch_id, "replacement_token": new.watch_id, "previous_coverage": old.coverage_epoch, "new_coverage": new.coverage_epoch, "active_after_cleanup": registry.state.lock().expect("drained registry").active.len()}
|
||||
})
|
||||
);
|
||||
println!(
|
||||
"usage_cleanup: {}",
|
||||
serde_json::json!({"case_id": "USAGE-12", "status": "PASS", "scope": "process-local"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_scope_and_replay_invalidate_without_reusing_generation() {
|
||||
let registry = Arc::new(UsageWatchRegistry::default());
|
||||
let now = Instant::now();
|
||||
let old = query(®istry, &request("bucket"), now);
|
||||
let mutation = registry.begin(None);
|
||||
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("bucket"), now), None);
|
||||
drop(mutation);
|
||||
let new = query(®istry, &request("bucket"), now);
|
||||
assert_ne!(new.watch_id, old.watch_id);
|
||||
registry.invalidate();
|
||||
let replayed = query(®istry, &request("bucket"), now);
|
||||
assert_ne!(replayed.watch_id, new.watch_id);
|
||||
assert!(replayed.coverage_epoch > new.coverage_epoch);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revision_exhaustion_invalidates_the_token_without_aba() {
|
||||
let registry = Arc::new(UsageWatchRegistry::default());
|
||||
let now = Instant::now();
|
||||
let old = query(®istry, &request("bucket"), now);
|
||||
registry
|
||||
.state
|
||||
.lock()
|
||||
.expect("registry state")
|
||||
.watches
|
||||
.get_mut("bucket")
|
||||
.expect("installed watch")
|
||||
.revision = u64::MAX - 1;
|
||||
let mutation = registry.begin(Some("bucket"));
|
||||
let refresh = ScannerUsageWatchRequest {
|
||||
watch_id: Some(old.watch_id),
|
||||
..request("bucket")
|
||||
};
|
||||
assert_eq!(registry.observe_at(Uuid::from_u128(1), &refresh, now), None);
|
||||
let during = query(®istry, &request("bucket"), now);
|
||||
assert_ne!(during.watch_id, old.watch_id);
|
||||
assert_eq!(during.inflight, 1);
|
||||
drop(mutation);
|
||||
assert_eq!(query(®istry, &request("bucket"), now).revision, during.revision + 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn coverage_epoch_exhaustion_stays_unknown_without_wrapping() {
|
||||
let registry = UsageWatchRegistry::default();
|
||||
let now = Instant::now();
|
||||
query(®istry, &request("bucket"), now);
|
||||
registry.state.lock().expect("registry state").coverage_epoch = u64::MAX - 1;
|
||||
registry.invalidate();
|
||||
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("bucket"), now), None);
|
||||
registry.invalidate();
|
||||
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("bucket"), now), None);
|
||||
let state = registry.state.lock().expect("registry state");
|
||||
assert!(state.exhausted);
|
||||
assert!(state.watches.is_empty());
|
||||
assert_eq!(state.coverage_epoch, u64::MAX - 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proof_comparison_rejects_every_changed_identity_and_busy_endpoint() {
|
||||
let registry = UsageWatchRegistry::default();
|
||||
let original = query(®istry, &request("bucket"), Instant::now());
|
||||
for field in 0..10 {
|
||||
let mut changed = original.clone();
|
||||
match field {
|
||||
0 => changed.owner_id = Uuid::new_v4(),
|
||||
1 => changed.session_id = Uuid::new_v4(),
|
||||
2 => changed.watch_id = Uuid::new_v4(),
|
||||
3 => changed.bucket_incarnation = Uuid::new_v4(),
|
||||
4 => changed.revision += 1,
|
||||
5 => changed.inflight = 1,
|
||||
6 => changed.coverage_version += 1,
|
||||
7 => changed.coverage_epoch += 1,
|
||||
8 => changed.lease_remaining_ms = 0,
|
||||
_ => {
|
||||
changed.revision = 0;
|
||||
changed.watch_id = Uuid::nil();
|
||||
}
|
||||
}
|
||||
assert!(!changed.unchanged_since(&original));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn watch_contention_cost_preserves_all_mutation_boundaries() {
|
||||
let registry = Arc::new(UsageWatchRegistry::default());
|
||||
let before = query(®istry, &request("hot"), Instant::now());
|
||||
let threads = 8_u64;
|
||||
let writes_per_thread = 10_000_u64;
|
||||
let started = Instant::now();
|
||||
std::thread::scope(|scope| {
|
||||
for _ in 0..threads {
|
||||
let registry = Arc::clone(®istry);
|
||||
scope.spawn(move || {
|
||||
for _ in 0..writes_per_thread {
|
||||
drop(registry.begin(Some("hot")));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
let elapsed = started.elapsed();
|
||||
let after = query(®istry, &request("hot"), Instant::now());
|
||||
let writes = threads * writes_per_thread;
|
||||
assert_eq!(after.revision, before.revision + 2 * writes);
|
||||
assert_eq!(after.inflight, 0);
|
||||
assert!(registry.state.lock().expect("drained registry").active.is_empty());
|
||||
eprintln!(
|
||||
"usage_watch_cost threads={threads} writes={writes} elapsed_us={} wall_ns_per_begin_end={}",
|
||||
elapsed.as_micros(),
|
||||
elapsed.as_nanos() / u128::from(writes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn exact_release_preserves_other_sessions_tokens_and_active_mutations() {
|
||||
let registry = Arc::new(UsageWatchRegistry::default());
|
||||
let owner = Uuid::from_u128(1);
|
||||
let now = Instant::now();
|
||||
let before = query(®istry, &request("bucket"), now);
|
||||
let release = ScannerUsageWatchRequest {
|
||||
watch_id: Some(before.watch_id),
|
||||
session_id: Some(before.session_id),
|
||||
release: true,
|
||||
..request("bucket")
|
||||
};
|
||||
for case in 0..5 {
|
||||
let mut wrong = release.clone();
|
||||
let mut wrong_owner = owner;
|
||||
match case {
|
||||
0 => wrong.watch_id = Some(Uuid::new_v4()),
|
||||
1 => wrong.session_id = Some(Uuid::new_v4()),
|
||||
2 => wrong.bucket_incarnation = Uuid::new_v4(),
|
||||
3 => wrong.watch_id = None,
|
||||
_ => wrong_owner = Uuid::new_v4(),
|
||||
}
|
||||
assert_eq!(registry.observe_at(wrong_owner, &wrong, now), None);
|
||||
assert!(query(®istry, &request("bucket"), now).unchanged_since(&before));
|
||||
}
|
||||
let mutation = registry.begin(Some("bucket"));
|
||||
assert_eq!(registry.observe_at(owner, &release, now), None);
|
||||
assert!(registry.state.lock().expect("registry state").watches.is_empty());
|
||||
let replacement = query(®istry, &request("bucket"), now);
|
||||
assert_ne!(replacement.watch_id, before.watch_id);
|
||||
assert_eq!(replacement.inflight, 1, "release cannot discard physical mutation ownership");
|
||||
assert_eq!(registry.observe_at(owner, &release, now), None);
|
||||
assert_eq!(query(®istry, &request("bucket"), now).watch_id, replacement.watch_id);
|
||||
drop(mutation);
|
||||
let after = query(®istry, &request("bucket"), now);
|
||||
assert_eq!(after.inflight, 0);
|
||||
assert!(after.revision > replacement.revision);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_watch_release_admits_later_buckets_before_lease_expiry() {
|
||||
let registry = UsageWatchRegistry::default();
|
||||
let now = Instant::now();
|
||||
let mut completed = Vec::new();
|
||||
for index in 0..SCANNER_USAGE_WATCH_MAX_ENTRIES {
|
||||
let req = request(&format!("bucket-{index:03}"));
|
||||
let known = query(®istry, &req, now);
|
||||
completed.push(ScannerUsageWatchRequest {
|
||||
watch_id: Some(known.watch_id),
|
||||
session_id: Some(known.session_id),
|
||||
release: true,
|
||||
..req
|
||||
});
|
||||
}
|
||||
assert_eq!(registry.observe_at(Uuid::from_u128(1), &request("later-bucket"), now), None);
|
||||
for release in completed {
|
||||
assert_eq!(registry.observe_at(Uuid::from_u128(1), &release, now), None);
|
||||
}
|
||||
let state = registry.state.lock().expect("registry state");
|
||||
assert_eq!(state.watches.len(), 0);
|
||||
assert_eq!(state.watch_name_bytes, 0);
|
||||
drop(state);
|
||||
assert!(query(®istry, &request("later-bucket"), now).is_well_formed());
|
||||
}
|
||||
@@ -107,7 +107,7 @@ pub(super) async fn run(
|
||||
result.state = ItemState::Prepared;
|
||||
result.detail = None;
|
||||
etag = checkpoint(api, job, &etag, guard).await?;
|
||||
let mut file = match stage(api, job, index, &info, &opts, guard).await {
|
||||
let staged = match verify_source(api, job, index, &info, &opts, guard).await {
|
||||
Ok(file) => file,
|
||||
Err(IntegrityError::Conflict) => {
|
||||
job.results[index].state = ItemState::Stale;
|
||||
@@ -125,6 +125,7 @@ pub(super) async fn run(
|
||||
if job.request.mode == JobMode::Audit {
|
||||
job.results[index].state = ItemState::Verified;
|
||||
} else {
|
||||
let mut file = staged.ok_or(IntegrityError::Invalid("missing migration payload"))?;
|
||||
let snapshot = validate_destination(api, job).await?;
|
||||
file.rewind().await?;
|
||||
let raw = sha256_bytes(&expected)?;
|
||||
@@ -231,30 +232,29 @@ async fn ensure_running(api: &Arc<ECStore>, job: &Job, guard: &rustfs_lock::Name
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn stage(
|
||||
async fn verify_source(
|
||||
api: &Arc<ECStore>,
|
||||
job: &Job,
|
||||
index: usize,
|
||||
info: &ObjectInfo,
|
||||
opts: &ObjectOptions,
|
||||
guard: &rustfs_lock::NamespaceLockGuard,
|
||||
) -> Result<tokio::fs::File> {
|
||||
) -> Result<Option<tokio::fs::File>> {
|
||||
let mut reader = api
|
||||
.get_object_reader(&job.bucket, &job.request.items[index].key, None, Default::default(), opts)
|
||||
.await?;
|
||||
if fingerprint(&reader.object_info) != fingerprint(info) {
|
||||
return Err(IntegrityError::Conflict);
|
||||
}
|
||||
let std_file = tokio::task::spawn_blocking(tempfile::tempfile)
|
||||
.await
|
||||
.map_err(|_| IntegrityError::Invalid("staging task failed"))??;
|
||||
let mut file = tokio::fs::File::from_std(std_file);
|
||||
let mut file = staging_file(job.request.mode, tempfile::tempfile).await?;
|
||||
let expected = job.results[index]
|
||||
.expected_sha256
|
||||
.as_deref()
|
||||
.ok_or(IntegrityError::Invalid("missing expected digest"))?;
|
||||
copy_verified(api, job, &mut reader.stream, Some(&mut file), info.size, expected, guard).await?;
|
||||
file.flush().await?;
|
||||
copy_verified(api, job, &mut reader.stream, file.as_mut(), info.size, expected, guard).await?;
|
||||
if let Some(file) = file.as_mut() {
|
||||
file.flush().await?;
|
||||
}
|
||||
let after = api.get_object_info(&job.bucket, &job.request.items[index].key, opts).await?;
|
||||
if fingerprint(&after) != fingerprint(info) {
|
||||
return Err(IntegrityError::Conflict);
|
||||
@@ -262,6 +262,20 @@ async fn stage(
|
||||
Ok(file)
|
||||
}
|
||||
|
||||
async fn staging_file(
|
||||
mode: JobMode,
|
||||
create: impl FnOnce() -> std::io::Result<std::fs::File> + Send + 'static,
|
||||
) -> Result<Option<tokio::fs::File>> {
|
||||
// Migration must publish the exact bytes verified here. Audit has no second consumer.
|
||||
if mode == JobMode::Audit {
|
||||
return Ok(None);
|
||||
}
|
||||
let file = tokio::task::spawn_blocking(create)
|
||||
.await
|
||||
.map_err(|_| IntegrityError::Invalid("staging task failed"))??;
|
||||
Ok(Some(tokio::fs::File::from_std(file)))
|
||||
}
|
||||
|
||||
async fn copy_verified(
|
||||
api: &Arc<ECStore>,
|
||||
job: &Job,
|
||||
@@ -466,3 +480,31 @@ impl tokio::io::AsyncRead for ThrottledFile {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod staging_tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn audit_does_not_need_a_writable_payload_staging_volume() {
|
||||
let result = staging_file(JobMode::Audit, || Err(std::io::Error::other("staging volume full")))
|
||||
.await
|
||||
.expect("audit must not open staging storage");
|
||||
assert!(result.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn migration_requires_writable_payload_staging() {
|
||||
assert!(
|
||||
staging_file(JobMode::Migrate, || { Err(std::io::Error::other("staging volume full")) })
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
let mut file = staging_file(JobMode::Migrate, tempfile::tempfile).await.unwrap().unwrap();
|
||||
file.write_all(b"verified bytes").await.unwrap();
|
||||
file.rewind().await.unwrap();
|
||||
let mut bytes = Vec::new();
|
||||
file.read_to_end(&mut bytes).await.unwrap();
|
||||
assert_eq!(bytes, b"verified bytes");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3038,7 +3038,7 @@ impl NotificationSys {
|
||||
return Err(Error::other("scanner scoped dirty usage capability requires scoped acknowledgements"));
|
||||
}
|
||||
};
|
||||
if by_host.insert(host.clone(), acknowledgement).is_some() {
|
||||
if by_host.insert(host, acknowledgement).is_some() {
|
||||
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
|
||||
}
|
||||
}
|
||||
@@ -3081,6 +3081,42 @@ impl NotificationSys {
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Query exactly the requested peers and buckets. Missing peers and old
|
||||
/// protocol endpoints fail the query instead of fabricating clean entries.
|
||||
pub async fn scanner_usage_watches(
|
||||
&self,
|
||||
requests: Vec<(String, uuid::Uuid, Vec<crate::runtime::usage_watch::ScannerUsageWatchRequest>)>,
|
||||
) -> Result<Vec<(String, Vec<crate::runtime::usage_watch::ScannerUsageWatchObservation>)>> {
|
||||
let clients = self
|
||||
.peer_clients
|
||||
.iter()
|
||||
.flatten()
|
||||
.map(|client| (client.grid_host.clone(), client.clone()))
|
||||
.collect::<HashMap<_, _>>();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
let mut futures = Vec::with_capacity(requests.len());
|
||||
for (host, owner, entries) in requests {
|
||||
if !seen.insert(host.clone()) {
|
||||
return Err(Error::other("duplicate usage watch peer"));
|
||||
}
|
||||
let client = clients
|
||||
.get(&host)
|
||||
.cloned()
|
||||
.ok_or_else(|| Error::other("usage watch peer is not reachable"))?;
|
||||
futures.push(async move {
|
||||
let observations = timeout(SCANNER_ACTIVITY_PROBE_TIMEOUT, client.scanner_usage_watch(owner, entries))
|
||||
.await
|
||||
.map_err(|_| Error::other("usage watch peer timed out"))??;
|
||||
Ok::<_, Error>((host, observations))
|
||||
});
|
||||
}
|
||||
let mut results = Vec::with_capacity(futures.len());
|
||||
for result in join_all(futures).await {
|
||||
results.push(result?);
|
||||
}
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
pub async fn acknowledge_scanner_dirty_usage(&self, acknowledgements: Vec<ScannerDirtyUsageAcknowledgement>) -> Result<bool> {
|
||||
let mut by_host = HashMap::with_capacity(acknowledgements.len());
|
||||
for acknowledgement in acknowledgements {
|
||||
@@ -3088,7 +3124,7 @@ impl NotificationSys {
|
||||
ScannerDirtyUsageAcknowledgement::Generation { host, .. }
|
||||
| ScannerDirtyUsageAcknowledgement::Scoped { host, .. } => host.clone(),
|
||||
};
|
||||
if by_host.insert(host.clone(), acknowledgement).is_some() {
|
||||
if by_host.insert(host, acknowledgement).is_some() {
|
||||
return Err(Error::other("duplicate scanner dirty usage acknowledgement target"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -802,7 +802,6 @@ impl ECStore {
|
||||
let set = set.clone();
|
||||
let rx = rx.clone();
|
||||
let bucket = bucket.clone();
|
||||
let entry_tasks = entry_tasks.clone();
|
||||
|
||||
let job = tokio::spawn(async move {
|
||||
let list_rx = rx.clone();
|
||||
|
||||
@@ -8144,9 +8144,10 @@ mod tests {
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_DISK_SHARD_OPEN), 4);
|
||||
assert_eq!(setup.available_shards(), 4);
|
||||
assert_eq!(setup.deferred_shards(), 4);
|
||||
for index in 4..8 {
|
||||
assert!(setup.attempted.len() >= 8);
|
||||
for (index, attempted) in setup.attempted.iter().enumerate().take(8).skip(4) {
|
||||
assert_eq!(calls.for_disk(disk_call_counters::KIND_DISK_SHARD_OPEN, index), 0);
|
||||
assert!(!setup.attempted[index], "parity must stay unopened until needed");
|
||||
assert!(!attempted, "parity must stay unopened until needed");
|
||||
}
|
||||
let (output, error) = local_batch_quorum_decode(setup, files[0].erasure.block_size).await;
|
||||
assert!(error.is_none(), "healthy decode failed: {error:?}");
|
||||
|
||||
@@ -5611,6 +5611,20 @@ impl SetDisks {
|
||||
metadata_sys::find_bucket_incarnation_id_in(&self.ctx, bucket).await
|
||||
}
|
||||
|
||||
/// Confirm deletion of this exact generation using immutable retirement evidence.
|
||||
/// Missing directories or unavailable bucket metadata do not prove retirement.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when the instance owner or retirement record is unavailable,
|
||||
/// malformed, or bound to a different deployment or generation.
|
||||
pub async fn is_bucket_generation_retired(&self, bucket: &str, incarnation: Uuid) -> Result<bool> {
|
||||
let store = metadata_sys::object_store_in(&self.ctx).await?;
|
||||
if !Arc::ptr_eq(&store.ctx, &self.ctx) {
|
||||
return Err(Error::other("bucket retirement metadata belongs to another instance"));
|
||||
}
|
||||
crate::bucket::retirement::is_retired(store, bucket, incarnation).await
|
||||
}
|
||||
|
||||
/// Admit one short scanner cache publication under this set's instance
|
||||
/// movement fence. The caller must hold the returned guard through its
|
||||
/// final conditional cache write; no scan-round work belongs under it.
|
||||
@@ -6045,14 +6059,10 @@ fn should_use_inline_fast_path_with_plan(
|
||||
// a stale/corrupt marker cannot route an oversized payload into the
|
||||
// in-memory decoder. The independent direct-memory reader remains capped
|
||||
// at 128 KiB below.
|
||||
if !plan.is_plain()
|
||||
|| shape.object_size > usize::try_from(INLINE_FAST_PATH_MAX_OBJECT_SIZE).expect("inline fast path limit fits usize")
|
||||
|| fi.erasure.data_blocks == 0
|
||||
|| fi.erasure.block_size == 0
|
||||
{
|
||||
return false;
|
||||
}
|
||||
true
|
||||
plan.is_plain()
|
||||
&& shape.object_size <= usize::try_from(INLINE_FAST_PATH_MAX_OBJECT_SIZE).expect("inline fast path limit fits usize")
|
||||
&& fi.erasure.data_blocks != 0
|
||||
&& fi.erasure.block_size != 0
|
||||
}
|
||||
|
||||
enum SmallWritePath {
|
||||
|
||||
@@ -515,7 +515,7 @@ impl SetDisks {
|
||||
let owner = context
|
||||
.namespace_owner
|
||||
.take()
|
||||
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
|
||||
.unwrap_or_else(|| self.ctx.begin_namespace_commit_for_bucket(bucket));
|
||||
context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
|
||||
let can_commit = context.can_commit;
|
||||
let can_try_local_fast_path = disks.len() == self.set_drive_count
|
||||
@@ -791,7 +791,7 @@ impl SetDisks {
|
||||
let namespace_owner = context
|
||||
.namespace_owner
|
||||
.clone()
|
||||
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
|
||||
.unwrap_or_else(|| self.ctx.begin_namespace_commit_for_bucket(bucket));
|
||||
context.namespace_owner = Some(Arc::new((namespace_owner, guard.clone())));
|
||||
}
|
||||
let can_commit = context.can_commit;
|
||||
@@ -821,7 +821,7 @@ impl SetDisks {
|
||||
let owner = context
|
||||
.namespace_owner
|
||||
.take()
|
||||
.unwrap_or_else(|| self.ctx.begin_namespace_commit());
|
||||
.unwrap_or_else(|| self.ctx.begin_namespace_commit_for_bucket(bucket));
|
||||
context.namespace_owner = Some(Arc::new((owner, Arc::clone(&transaction_guard))));
|
||||
}
|
||||
let DeleteRecoveryContext {
|
||||
@@ -1025,7 +1025,7 @@ impl SetDisks {
|
||||
let namespace_owner = if dry_run {
|
||||
None
|
||||
} else {
|
||||
Some(namespace_owner.unwrap_or_else(|| self.ctx.begin_namespace_commit()))
|
||||
Some(namespace_owner.unwrap_or_else(|| self.ctx.begin_namespace_commit_for_bucket(bucket)))
|
||||
};
|
||||
if !dry_run && metadata_may_change {
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
|
||||
@@ -1285,7 +1285,7 @@ impl SetDisks {
|
||||
result.after.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint: self.set_endpoints[index].to_string(),
|
||||
state: drive_state.to_string(),
|
||||
state: drive_state,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2060,7 +2060,7 @@ impl SetDisks {
|
||||
_write_lock_guard.clone(),
|
||||
_read_repair_write_lock_guard.clone(),
|
||||
bucket_heal_scope.clone(),
|
||||
self.ctx.begin_namespace_commit(),
|
||||
self.ctx.begin_namespace_commit_for_bucket(bucket),
|
||||
))
|
||||
});
|
||||
match disk
|
||||
@@ -3102,7 +3102,7 @@ impl SetDisks {
|
||||
result.after.drives.push(HealDriveInfo {
|
||||
uuid: "".to_string(),
|
||||
endpoint,
|
||||
state: drive_state.to_string(),
|
||||
state: drive_state,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3147,7 +3147,7 @@ impl SetDisks {
|
||||
_ => DriveState::Corrupt.to_string(),
|
||||
},
|
||||
};
|
||||
result.after.drives[index].state = drive_state.to_string();
|
||||
result.after.drives[index].state = drive_state;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3761,7 +3761,7 @@ impl SetDisks {
|
||||
namespace_owner: (!opts.dry_run).then(|| {
|
||||
Arc::new((
|
||||
scope.clone(),
|
||||
self.ctx.begin_namespace_commit(),
|
||||
self.ctx.begin_namespace_commit_for_bucket(bucket),
|
||||
_write_lock_guard.clone(),
|
||||
transaction_guard.clone(),
|
||||
)) as Arc<dyn Send + Sync>
|
||||
@@ -10994,7 +10994,7 @@ mod heal_result_report_tests {
|
||||
set.heal_object(bucket, object, "", &heal_opts),
|
||||
);
|
||||
let put_info = put_res.expect("overwrite must succeed under lock serialization");
|
||||
last_etag = put_info.etag.clone().unwrap_or_default();
|
||||
last_etag = put_info.etag.unwrap_or_default();
|
||||
// Heal outcome is unconstrained (may hit the tail race and report a
|
||||
// retryable error); the invariant is checked on the end state.
|
||||
drop(heal_res);
|
||||
|
||||
@@ -610,6 +610,9 @@ impl SetDisks {
|
||||
new_disk.enable_health_check();
|
||||
|
||||
if new_disk.is_local() {
|
||||
if let crate::disk::Disk::Local(wrapper) = new_disk.as_ref() {
|
||||
wrapper.get_disk().usage_watch_binding.bind(&self.ctx.usage_watches);
|
||||
}
|
||||
let local_disk_map = runtime_sources::local_disk_map_handle();
|
||||
let mut global_local_disk_map = local_disk_map.write().await;
|
||||
let path = new_disk.endpoint().to_string();
|
||||
@@ -690,9 +693,10 @@ impl SetDisks {
|
||||
return Err(Error::other("DriveID: offline"));
|
||||
}
|
||||
|
||||
for i in 0..self.format.erasure.sets.len() {
|
||||
for j in 0..self.format.erasure.sets[0].len() {
|
||||
if fm.erasure.this == self.format.erasure.sets[i][j] {
|
||||
let set_drive_count = self.format.erasure.sets[0].len();
|
||||
for (i, set) in self.format.erasure.sets.iter().enumerate() {
|
||||
for (j, drive) in set.iter().enumerate().take(set_drive_count) {
|
||||
if fm.erasure.this == *drive {
|
||||
return Ok((i, j));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4688,7 +4688,7 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.with_multipart_commit_owner(multipart_commit_owner)
|
||||
.with_namespace_commit_guard(
|
||||
(!crate::bucket::utils::is_meta_bucketname(&commit_bucket))
|
||||
.then(|| commit_set.ctx.begin_namespace_commit()),
|
||||
.then(|| commit_set.ctx.begin_namespace_commit_for_bucket(&commit_bucket)),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
@@ -11546,7 +11546,7 @@ mod tests {
|
||||
break;
|
||||
}
|
||||
key_marker = page.next_key_marker.clone();
|
||||
upload_id_marker = page.next_upload_id_marker.clone();
|
||||
upload_id_marker = page.next_upload_id_marker;
|
||||
assert!(key_marker.is_some(), "a truncated page must carry a next key marker");
|
||||
assert!(upload_id_marker.is_some(), "a truncated upload page must carry a next upload marker");
|
||||
}
|
||||
|
||||
@@ -3918,7 +3918,7 @@ impl SetDisks {
|
||||
explicit_delete: true,
|
||||
transaction_guard: transaction_guard.cloned(),
|
||||
namespace_owner: Some(Arc::new((
|
||||
self.ctx.begin_namespace_commit(),
|
||||
self.ctx.begin_namespace_commit_for_bucket(bucket),
|
||||
lock_guard.cloned(),
|
||||
opts.object_lock_config_snapshot.clone(),
|
||||
opts.scanner_publication_commit_scope.clone(),
|
||||
@@ -5661,7 +5661,8 @@ impl SetDisks {
|
||||
.with_publication_scope(commit_scanner_publication_scope.clone())
|
||||
.with_rollback_receipt(commit_rollback_receipt.clone())
|
||||
.with_namespace_commit_guard(
|
||||
(!is_meta_bucketname(&commit_bucket)).then(|| commit_set.ctx.begin_namespace_commit()),
|
||||
(!is_meta_bucketname(&commit_bucket))
|
||||
.then(|| commit_set.ctx.begin_namespace_commit_for_bucket(&commit_bucket)),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
@@ -9063,7 +9064,7 @@ impl SetDisks {
|
||||
let disks = self.disk_inventory().await;
|
||||
let namespace_owner = (!is_meta_bucketname(bucket)).then(|| {
|
||||
Arc::new((
|
||||
self.ctx.begin_namespace_commit(),
|
||||
self.ctx.begin_namespace_commit_for_bucket(bucket),
|
||||
commit_context.guard.clone(),
|
||||
commit_context.transaction_guard.clone(),
|
||||
commit_context
|
||||
@@ -10469,7 +10470,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let rollback_dir = Uuid::new_v4();
|
||||
let namespace_owner = (!is_meta_bucketname(bucket)).then(|| {
|
||||
Arc::new((
|
||||
self.ctx.begin_namespace_commit(),
|
||||
self.ctx.begin_namespace_commit_for_bucket(bucket),
|
||||
Arc::clone(&transaction_guards),
|
||||
opts.delete_mutation_owner.clone(),
|
||||
opts.object_lock_config_snapshot.clone(),
|
||||
@@ -10973,7 +10974,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
explicit_delete,
|
||||
namespace_owner: Some(Arc::new((
|
||||
self.ctx.begin_namespace_commit(),
|
||||
self.ctx.begin_namespace_commit_for_bucket(bucket),
|
||||
opts.delete_mutation_owner.clone(),
|
||||
opts.object_lock_config_snapshot.clone(),
|
||||
))),
|
||||
@@ -11329,7 +11330,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
work_budget: opts.delete_recovery_work_budget.clone(),
|
||||
transaction_guard: transaction_guard.clone(),
|
||||
namespace_owner: Some(Arc::new((
|
||||
self.ctx.begin_namespace_commit(),
|
||||
self.ctx.begin_namespace_commit_for_bucket(bucket),
|
||||
_lock_guard.clone(),
|
||||
opts.object_lock_config_snapshot.clone(),
|
||||
opts.scanner_publication_commit_scope.clone(),
|
||||
|
||||
@@ -25,14 +25,15 @@ pub(crate) mod heal {
|
||||
pub(crate) mod internode {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
NS_SCANNER_BODY_SHA256_QUERY, NS_SCANNER_CAPABILITY_CHALLENGE_QUERY, NS_SCANNER_CHECKPOINT_COVERAGE_QUERY,
|
||||
NS_SCANNER_CYCLE_QUERY, NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
|
||||
NS_SCANNER_COMPLETE_OBSERVATION_QUERY, NS_SCANNER_CYCLE_QUERY, NS_SCANNER_DURABLE_PROGRESS_QUERY,
|
||||
NS_SCANNER_LEADER_EPOCH_QUERY, NS_SCANNER_PROTOCOL_VERSION, NS_SCANNER_PROTOCOL_VERSION_QUERY,
|
||||
NS_SCANNER_REQUEST_ID_QUERY, NS_SCANNER_SERVER_EPOCH_QUERY, NS_SCANNER_SESSION_ID_QUERY,
|
||||
NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse,
|
||||
PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_TRAILER_MAC_LEN,
|
||||
PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY, PUT_FILE_CAPABILITY_QUERY,
|
||||
PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY, PutFileCapabilityResponse,
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilities,
|
||||
NsScannerCapabilityResponse, PUT_FILE_AUTH_QUERY, PUT_FILE_AUTH_TRAILER_DIGEST_LEN, PUT_FILE_AUTH_TRAILER_LEN,
|
||||
PUT_FILE_AUTH_TRAILER_MAC_LEN, PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
|
||||
PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
|
||||
PutFileCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
|
||||
@@ -385,7 +385,7 @@ impl ECStore {
|
||||
dry_run: opts.dry_run,
|
||||
explicit_delete,
|
||||
namespace_owner: (!opts.dry_run)
|
||||
.then(|| Arc::new((scope.clone(), store.ctx.begin_namespace_commit())) as Arc<dyn Send + Sync>),
|
||||
.then(|| Arc::new((scope.clone(), store.ctx.begin_namespace_commit_for_bucket(&bucket))) as Arc<dyn Send + Sync>),
|
||||
..crate::set_disk::DeleteRecoveryContext::new(Some(scope.incarnation), &can_commit)
|
||||
};
|
||||
let mut summary = crate::set_disk::DeleteRecoverySummary::default();
|
||||
|
||||
@@ -17932,10 +17932,10 @@ mod tests {
|
||||
.expect("shared-tuple recursive prefix delete should commit");
|
||||
|
||||
assert_eq!(tier_delete_journal_count(store.clone()).await, 1);
|
||||
for pool_idx in 0..2 {
|
||||
assert!(store.pools.len() >= 2);
|
||||
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
|
||||
assert!(
|
||||
store.pools[pool_idx]
|
||||
.get_disks_by_key(object)
|
||||
pool.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("shared-tuple post-delete metadata should decode")
|
||||
@@ -19768,8 +19768,9 @@ mod tests {
|
||||
}
|
||||
wait_for_expiry_workers_idle(&store).await;
|
||||
let mut free_version_ids = Vec::new();
|
||||
for pool_idx in 0..2 {
|
||||
let retained = store.pools[pool_idx]
|
||||
assert!(store.pools.len() >= 2);
|
||||
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
|
||||
let retained = pool
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
@@ -19805,10 +19806,10 @@ mod tests {
|
||||
);
|
||||
wait_for_expiry_workers_idle(&store).await;
|
||||
assert_eq!(backend.remove_count().await, 1, "shared remote tuple should be deleted exactly once");
|
||||
for pool_idx in 0..2 {
|
||||
assert!(store.pools.len() >= 2);
|
||||
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
|
||||
assert!(
|
||||
store.pools[pool_idx]
|
||||
.get_disks_by_key(object)
|
||||
pool.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(&bucket, object)
|
||||
.await
|
||||
.expect("post-delete metadata should decode")
|
||||
@@ -19858,9 +19859,10 @@ mod tests {
|
||||
.expect("multi-pool source bucket should be created");
|
||||
|
||||
let mut remote_tuples = Vec::new();
|
||||
for pool_idx in 0..2 {
|
||||
assert!(store.pools.len() >= 2);
|
||||
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'a' + pool_idx as u8; 1024 * 1024]);
|
||||
let source = store.pools[pool_idx]
|
||||
let source = pool
|
||||
.put_object(
|
||||
bucket,
|
||||
object,
|
||||
@@ -19872,31 +19874,27 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("pool-local source should be written");
|
||||
store.pools[pool_idx]
|
||||
.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: source.etag.clone().expect("pool-local source should have an etag"),
|
||||
..Default::default()
|
||||
},
|
||||
mod_time: source.mod_time,
|
||||
pool.transition_object(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
transition: TransitionOptions {
|
||||
status: TRANSITION_PENDING.to_string(),
|
||||
tier: tier_name.to_string(),
|
||||
etag: source.etag.clone().expect("pool-local source should have an etag"),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("pool-local source should transition");
|
||||
let transitioned = store.pools[pool_idx]
|
||||
mod_time: source.mod_time,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("pool-local source should transition");
|
||||
let transitioned = pool
|
||||
.get_object_info(bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("pool-local transitioned source should be readable");
|
||||
remote_tuples.push((
|
||||
transitioned.transitioned_object.name.clone(),
|
||||
transitioned.transitioned_object.version_id.clone(),
|
||||
));
|
||||
remote_tuples.push((transitioned.transitioned_object.name.clone(), transitioned.transitioned_object.version_id));
|
||||
}
|
||||
assert_ne!(remote_tuples[0], remote_tuples[1], "fixture must use distinct remote cleanup tuples");
|
||||
assert_eq!(backend.object_count().await, 2);
|
||||
@@ -19906,8 +19904,9 @@ mod tests {
|
||||
.delete_object_with_tier_delete_journal(bucket, object, ObjectOptions::default())
|
||||
.await
|
||||
.expect("store-level delete should remove both pool copies");
|
||||
for pool_idx in 0..2 {
|
||||
let retained = store.pools[pool_idx]
|
||||
assert!(store.pools.len() >= 2);
|
||||
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
|
||||
let retained = pool
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
@@ -19946,10 +19945,10 @@ mod tests {
|
||||
removed.sort();
|
||||
remote_tuples.sort();
|
||||
assert_eq!(removed, remote_tuples, "recovery must use each pool's exact remote tuple");
|
||||
for pool_idx in 0..2 {
|
||||
assert!(store.pools.len() >= 2);
|
||||
for (pool_idx, pool) in store.pools.iter().enumerate().take(2) {
|
||||
assert!(
|
||||
store.pools[pool_idx]
|
||||
.get_disks_by_key(object)
|
||||
pool.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("post-recovery pool metadata should decode")
|
||||
|
||||
@@ -8504,7 +8504,7 @@ mod test {
|
||||
|
||||
let mut results = Vec::new();
|
||||
while let Some(entry) = out_rx.recv().await {
|
||||
results.push(entry.name.clone());
|
||||
results.push(entry.name);
|
||||
}
|
||||
|
||||
handle.await.unwrap().unwrap();
|
||||
@@ -8533,7 +8533,7 @@ mod test {
|
||||
|
||||
let mut results = Vec::new();
|
||||
while let Some(entry) = out_rx.recv().await {
|
||||
results.push(entry.name.clone());
|
||||
results.push(entry.name);
|
||||
}
|
||||
|
||||
handle.await.unwrap().unwrap();
|
||||
@@ -9177,7 +9177,7 @@ mod test {
|
||||
|
||||
let mut results = Vec::new();
|
||||
while let Some(entry) = out_rx.recv().await {
|
||||
results.push(entry.name.clone());
|
||||
results.push(entry.name);
|
||||
}
|
||||
|
||||
handle.await.unwrap().unwrap();
|
||||
@@ -9265,7 +9265,7 @@ mod test {
|
||||
|
||||
let mut results = Vec::new();
|
||||
while let Some(entry) = out_rx.recv().await {
|
||||
results.push(entry.name.clone());
|
||||
results.push(entry.name);
|
||||
}
|
||||
handle.await.unwrap().unwrap();
|
||||
|
||||
@@ -9295,7 +9295,7 @@ mod test {
|
||||
|
||||
let mut results = Vec::new();
|
||||
while let Some(entry) = out_rx.recv().await {
|
||||
results.push(entry.name.clone());
|
||||
results.push(entry.name);
|
||||
}
|
||||
handle.await.unwrap().unwrap();
|
||||
|
||||
@@ -9367,7 +9367,7 @@ mod test {
|
||||
|
||||
let mut results = Vec::new();
|
||||
while let Some(entry) = out_rx.recv().await {
|
||||
results.push(entry.name.clone());
|
||||
results.push(entry.name);
|
||||
}
|
||||
handle.await.unwrap().unwrap();
|
||||
|
||||
@@ -9401,7 +9401,7 @@ mod test {
|
||||
|
||||
let mut results = Vec::new();
|
||||
while let Some(entry) = out_rx.recv().await {
|
||||
results.push(entry.name.clone());
|
||||
results.push(entry.name);
|
||||
}
|
||||
handle.await.unwrap().unwrap();
|
||||
|
||||
@@ -9491,7 +9491,7 @@ mod test {
|
||||
let mut results = Vec::new();
|
||||
while let Some(entry) = out_rx.recv().await {
|
||||
assert!(entry.is_object(), "the directory-marker object must shadow the same-named prefix dir");
|
||||
results.push(entry.name.clone());
|
||||
results.push(entry.name);
|
||||
}
|
||||
handle.await.unwrap().unwrap();
|
||||
|
||||
@@ -9514,7 +9514,7 @@ mod test {
|
||||
|
||||
let mut results = Vec::new();
|
||||
while let Some(entry) = out_rx.recv().await {
|
||||
results.push(entry.name.clone());
|
||||
results.push(entry.name);
|
||||
}
|
||||
handle.await.unwrap().unwrap();
|
||||
|
||||
@@ -9538,7 +9538,7 @@ mod test {
|
||||
|
||||
let mut results = Vec::new();
|
||||
while let Some(entry) = out_rx.recv().await {
|
||||
results.push(entry.name.clone());
|
||||
results.push(entry.name);
|
||||
}
|
||||
handle.await.unwrap().unwrap();
|
||||
|
||||
|
||||
@@ -451,6 +451,8 @@ pub use object::{
|
||||
mod peer;
|
||||
mod rebalance;
|
||||
#[cfg(test)]
|
||||
mod usage_watch_tests;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rebalance::PoolSpaceForTest;
|
||||
pub(crate) mod utils;
|
||||
|
||||
@@ -829,6 +831,13 @@ impl ECStore {
|
||||
runtime_sources::bucket_metadata_sys()
|
||||
}
|
||||
|
||||
/// Recover this instance's weak metadata owner without accepting a
|
||||
/// process-global store belonging to another scanner or test fixture.
|
||||
pub async fn scanner_usage_watch_store_handle(&self) -> Option<Arc<Self>> {
|
||||
let owner = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await?;
|
||||
std::ptr::eq(self, owner.as_ref()).then_some(owner)
|
||||
}
|
||||
|
||||
/// Get the global endpoints
|
||||
pub fn endpoints(&self) -> EndpointServerPools {
|
||||
runtime_sources::endpoint_pools().unwrap_or_else(|| Vec::new().into())
|
||||
@@ -906,6 +915,72 @@ impl ECStore {
|
||||
list_objects::scanner_namespace_mutation_generation().saturating_add(self.ctx.namespace_commit_generation())
|
||||
}
|
||||
|
||||
/// Query named buckets against their current storage-owned incarnation.
|
||||
/// Unknown identities, expired tokens and resource pressure remain explicit
|
||||
/// Unknown entries; absence from a dirty list never creates a clean proof.
|
||||
pub async fn scanner_usage_watch(
|
||||
&self,
|
||||
requests: &[crate::runtime::usage_watch::ScannerUsageWatchRequest],
|
||||
) -> Vec<crate::runtime::usage_watch::ScannerUsageWatchObservation> {
|
||||
use crate::runtime::usage_watch::ScannerUsageWatchObservation;
|
||||
let mut observations = Vec::with_capacity(requests.len().min(32));
|
||||
if requests.len() > 32 {
|
||||
return observations;
|
||||
}
|
||||
let local_map = self.ctx.local_disk_map();
|
||||
let local_disks = local_map.read().await;
|
||||
let bindings_complete = !local_disks.is_empty()
|
||||
&& local_disks.values().all(|disk| {
|
||||
disk.as_ref().is_some_and(|disk| match disk.as_ref() {
|
||||
crate::disk::Disk::Local(wrapper) => wrapper.get_disk().usage_watch_binding.matches(&self.ctx.usage_watches),
|
||||
crate::disk::Disk::Remote(_) => false,
|
||||
})
|
||||
});
|
||||
drop(local_disks);
|
||||
for request in requests {
|
||||
if request.release {
|
||||
// Retirement needs the exact saved token, not current bucket
|
||||
// metadata. Deleted buckets must not retain scarce watch slots.
|
||||
self.ctx.usage_watches.observe(self.id, request);
|
||||
observations.push(ScannerUsageWatchObservation {
|
||||
bucket: request.bucket.clone(),
|
||||
known: None,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
let current = self.bucket_incarnation_id_from_disk(&request.bucket).await;
|
||||
let known = match current {
|
||||
Ok(incarnation) if bindings_complete && incarnation == request.bucket_incarnation && !incarnation.is_nil() => {
|
||||
self.ctx.usage_watches.observe(self.id, request)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
observations.push(ScannerUsageWatchObservation {
|
||||
bucket: request.bucket.clone(),
|
||||
known,
|
||||
});
|
||||
}
|
||||
observations
|
||||
}
|
||||
|
||||
/// Invalidate session proofs after replay or an unsupported producer mode
|
||||
/// transition; this does not change durable dirty ACK responsibility.
|
||||
pub fn invalidate_scanner_usage_watches(&self) {
|
||||
self.ctx.usage_watches.invalidate();
|
||||
}
|
||||
|
||||
pub fn scanner_usage_watch_session_id(&self) -> Uuid {
|
||||
self.ctx.usage_watches.session_id()
|
||||
}
|
||||
|
||||
/// Revoke an exact local token without awaiting storage or peer I/O. A
|
||||
/// failed keepalive must not let a later successful query revive its proof.
|
||||
pub fn scanner_release_usage_watch(&self, request: &crate::runtime::usage_watch::ScannerUsageWatchRequest) {
|
||||
if request.release {
|
||||
self.ctx.usage_watches.observe(self.id, request);
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn scanner_data_movement_active(&self) -> bool {
|
||||
let (decommission, rebalance) = tokio::join!(self.is_decommission_running(), self.is_rebalance_started());
|
||||
decommission || rebalance
|
||||
|
||||
@@ -88,7 +88,7 @@ impl ECStore {
|
||||
/// Resolve a quota lease against this request's captured storage instance.
|
||||
pub async fn quota_mutation_fence_disk(&self, disk_ref: &str) -> DiskResult<DiskStore> {
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false).await?;
|
||||
let _owner = admit_local_disk(&self.ctx, &disk, id, false, None).await?;
|
||||
Ok(disk)
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ impl ECStore {
|
||||
let path = target.1.to_owned();
|
||||
self.run_bucket_heal_at_incarnation(target.0, expected, &Default::default(), move |store, volume, _| async move {
|
||||
let (disk, id) = local_disk_candidate(&store.ctx, &disk_ref).await?;
|
||||
let _owner = admit_local_disk(&store.ctx, &disk, id, true).await?;
|
||||
let _owner = admit_local_disk(&store.ctx, &disk, id, true, Some(&volume)).await?;
|
||||
disk.write_metadata("", &volume, &path, fi).await.map_err(Into::into)
|
||||
})
|
||||
.await
|
||||
@@ -127,7 +127,7 @@ impl ECStore {
|
||||
let path = target.1.to_owned();
|
||||
self.run_bucket_heal_at_incarnation(target.0, expected, &Default::default(), move |store, volume, _| async move {
|
||||
let (disk, id) = local_disk_candidate(&store.ctx, &disk_ref).await?;
|
||||
let owner = admit_local_disk(&store.ctx, &disk, id, true).await?;
|
||||
let owner = admit_local_disk(&store.ctx, &disk, id, true, Some(&volume)).await?;
|
||||
let scope = super::bucket_heal_scope(&volume).ok_or_else(|| StorageError::other("missing bucket heal scope"))?;
|
||||
scope.check()?;
|
||||
disk.delete_with_namespace_owner(&volume, &path, opts, Some(Arc::new((scope, owner))))
|
||||
@@ -159,7 +159,7 @@ impl ECStore {
|
||||
.map_err(|error| DiskError::other(error.to_string()))?,
|
||||
);
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let owner = admit_local_disk(&self.ctx, &disk, id, true).await?;
|
||||
let owner = admit_local_disk(&self.ctx, &disk, id, true, Some(target.0)).await?;
|
||||
if fence.is_lock_lost() {
|
||||
return Err(DiskError::other("bucket heal incarnation fence was lost"));
|
||||
}
|
||||
@@ -283,7 +283,7 @@ impl ECStore {
|
||||
) -> DiskResult<crate::disk::multipart_source::MultipartSourceResponse> {
|
||||
request.validate()?;
|
||||
let (disk, id) = local_disk_candidate(&self.ctx, disk_ref).await?;
|
||||
let owner = admit_local_disk(&self.ctx, &disk, id, request.operation.is_mutation()).await?;
|
||||
let owner = admit_local_disk(&self.ctx, &disk, id, request.operation.is_mutation(), None).await?;
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
};
|
||||
@@ -356,6 +356,7 @@ async fn admit_local_disk(
|
||||
disk: &DiskStore,
|
||||
disk_id: Option<Uuid>,
|
||||
mutates_namespace: bool,
|
||||
bucket: Option<&str>,
|
||||
) -> DiskResult<Option<Arc<NamespaceCommitGuard>>> {
|
||||
if !matches!(disk.as_ref(), Disk::Local(_)) {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
@@ -378,7 +379,10 @@ async fn admit_local_disk(
|
||||
}
|
||||
// Admission linearizes under the registry read: replacement/quarantine
|
||||
// before this point rejects; later changes do not revoke physical I/O.
|
||||
Ok(mutates_namespace.then(|| ctx.begin_namespace_commit()))
|
||||
Ok(mutates_namespace.then(|| match bucket {
|
||||
Some(bucket) => ctx.begin_namespace_commit_for_bucket(bucket),
|
||||
None => ctx.begin_namespace_commit(),
|
||||
}))
|
||||
}
|
||||
|
||||
async fn rename_local_data_with_ctx(
|
||||
@@ -398,7 +402,18 @@ async fn rename_local_data_with_ctx(
|
||||
}
|
||||
let mutates_namespace = !is_meta_bucketname(source.0) || !is_meta_bucketname(destination.0);
|
||||
let admit_started = rustfs_io_metrics::put_stage_timer();
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, mutates_namespace).await?;
|
||||
let owner = admit_local_disk(
|
||||
ctx,
|
||||
&disk,
|
||||
disk_id,
|
||||
mutates_namespace,
|
||||
Some(if !is_meta_bucketname(destination.0) {
|
||||
destination.0
|
||||
} else {
|
||||
source.0
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
if source.0 == crate::disk::RUSTFS_META_MULTIPART_BUCKET {
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
@@ -444,7 +459,7 @@ async fn undo_local_write_with_ctx(
|
||||
return Err(DiskError::other("target undo requires undo_write"));
|
||||
}
|
||||
let (disk, disk_id) = local_disk_candidate(ctx, disk_ref).await?;
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, !is_meta_bucketname(volume)).await?;
|
||||
let owner = admit_local_disk(ctx, &disk, disk_id, !is_meta_bucketname(volume), Some(volume)).await?;
|
||||
let physical_owner = owner.as_ref().map(|owner| owner.clone() as Arc<dyn Send + Sync>);
|
||||
let result = disk
|
||||
.undo_write_with_namespace_owner(volume, path, fi, opts, physical_owner)
|
||||
@@ -874,7 +889,7 @@ mod tests {
|
||||
}
|
||||
drop(entries);
|
||||
assert!(
|
||||
matches!(admit_local_disk(&ctx, &disk, None, true).await, Err(DiskError::DiskNotFound)),
|
||||
matches!(admit_local_disk(&ctx, &disk, None, true, None).await, Err(DiskError::DiskNotFound)),
|
||||
"{state}"
|
||||
);
|
||||
assert!(!ctx.namespace_commits_pending());
|
||||
|
||||
@@ -1722,20 +1722,20 @@ mod tests {
|
||||
.await
|
||||
.expect("bucket should be created in both pools");
|
||||
|
||||
for pool_index in 0..2 {
|
||||
assert!(store.pools.len() >= 2);
|
||||
for (pool_index, pool) in store.pools.iter().enumerate().take(2) {
|
||||
let mut reader = PutObjReader::from_vec(format!("pool-{pool_index}-history").into_bytes());
|
||||
store.pools[pool_index]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("historical version should be stored");
|
||||
pool.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
versioned: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("historical version should be stored");
|
||||
}
|
||||
let marker = store.pools[0]
|
||||
.delete_object(
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::runtime::usage_watch::{ScannerUsageWatchKnown, ScannerUsageWatchRequest};
|
||||
use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations as _};
|
||||
|
||||
async fn known(store: &ECStore, request: &ScannerUsageWatchRequest) -> ScannerUsageWatchKnown {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let response = store.scanner_usage_watch(std::slice::from_ref(request)).await;
|
||||
assert_eq!(response.len(), 1);
|
||||
assert_eq!(response[0].bucket, request.bucket);
|
||||
let proof = response[0].known.clone().expect("storage-backed watch proof");
|
||||
if proof.inflight == 0 {
|
||||
return proof;
|
||||
}
|
||||
// API quorum completion can precede a remaining physical tail.
|
||||
tokio::time::sleep(Duration::from_millis(1)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("all counted physical tails settle")
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn usage_watch_api_tracks_real_put_delete_and_bucket_recreation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "watch-real-store";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create watched bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation");
|
||||
let mut request = ScannerUsageWatchRequest {
|
||||
bucket: bucket.into(),
|
||||
bucket_incarnation: incarnation,
|
||||
watch_id: None,
|
||||
release: false,
|
||||
session_id: None,
|
||||
};
|
||||
let first = known(&store, &request).await;
|
||||
assert!(first.is_well_formed());
|
||||
assert_eq!(first.inflight, 0);
|
||||
request.watch_id = Some(first.watch_id);
|
||||
assert!(known(&store, &request).await.unchanged_since(&first));
|
||||
|
||||
let reader =
|
||||
rustfs_rio::HashReader::from_stream(std::io::Cursor::new(b"payload"), 7, 7, None, None, false).expect("PUT body reader");
|
||||
store
|
||||
.put_object(bucket, "object", &mut PutObjReader::new(reader), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("real PUT");
|
||||
let written = known(&store, &request).await;
|
||||
assert_eq!(written.watch_id, first.watch_id);
|
||||
assert!(written.revision > first.revision);
|
||||
assert_eq!(written.inflight, 0);
|
||||
|
||||
store
|
||||
.delete_object(bucket, "object", ObjectOptions::default())
|
||||
.await
|
||||
.expect("real object delete");
|
||||
let deleted = known(&store, &request).await;
|
||||
assert_eq!(deleted.watch_id, first.watch_id);
|
||||
assert!(deleted.revision > written.revision);
|
||||
assert_eq!(deleted.inflight, 0);
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete empty bucket");
|
||||
assert_eq!(store.scanner_usage_watch(std::slice::from_ref(&request)).await[0].known, None);
|
||||
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate watched bucket");
|
||||
let recreated_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("new incarnation");
|
||||
assert_ne!(recreated_incarnation, incarnation);
|
||||
assert_eq!(store.scanner_usage_watch(std::slice::from_ref(&request)).await[0].known, None);
|
||||
request.bucket_incarnation = recreated_incarnation;
|
||||
assert_eq!(store.scanner_usage_watch(std::slice::from_ref(&request)).await[0].known, None);
|
||||
request.watch_id = None;
|
||||
let recreated = known(&store, &request).await;
|
||||
assert_ne!(recreated.watch_id, first.watch_id);
|
||||
assert!(!recreated.unchanged_since(&first));
|
||||
assert_eq!(recreated.inflight, 0);
|
||||
}
|
||||
@@ -905,7 +905,7 @@ mod tests {
|
||||
assert_indexed_matches_scalar(&original, selected);
|
||||
}
|
||||
if inline_mode > 0 {
|
||||
let mut legacy_null = original.clone();
|
||||
let mut legacy_null = original;
|
||||
let payload = legacy_null
|
||||
.data
|
||||
.find(NULL_VERSION_ID)
|
||||
|
||||
@@ -113,11 +113,7 @@ impl FileMetaVersionHeader {
|
||||
}
|
||||
|
||||
// Check erasure coding parameters
|
||||
if self.has_ec() && (self.ec_n == 0 || self.ec_m == 0 || self.ec_m < self.ec_n) {
|
||||
return false;
|
||||
}
|
||||
|
||||
true
|
||||
!(self.has_ec() && (self.ec_n == 0 || self.ec_m == 0 || self.ec_m < self.ec_n))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4223,9 +4223,10 @@ mod tests {
|
||||
o_hdr.ec_m = o_m;
|
||||
o_hdr.ec_n = o_n;
|
||||
|
||||
let actual = FileMeta::headers_match_strict(&s_hdr, &o_hdr);
|
||||
let expected = {
|
||||
let mut s_cmp = s_hdr.clone();
|
||||
let mut o_cmp = o_hdr.clone();
|
||||
let mut s_cmp = s_hdr;
|
||||
let mut o_cmp = o_hdr;
|
||||
if s_cmp.has_ec() != o_cmp.has_ec() {
|
||||
s_cmp.ec_m = 0;
|
||||
s_cmp.ec_n = 0;
|
||||
@@ -4234,11 +4235,7 @@ mod tests {
|
||||
}
|
||||
s_cmp == o_cmp
|
||||
};
|
||||
assert_eq!(
|
||||
FileMeta::headers_match_strict(&s_hdr, &o_hdr),
|
||||
expected,
|
||||
"mismatch for s=({s_m},{s_n}) vs o=({o_m},{o_n})"
|
||||
);
|
||||
assert_eq!(actual, expected, "mismatch for s=({s_m},{s_n}) vs o=({o_m},{o_n})");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -570,7 +570,7 @@ mod tests {
|
||||
tonic::Status::internal("operation was canceled"),
|
||||
] {
|
||||
assert!(
|
||||
!Error::Storage(EcstoreError::from(status).clone()).is_recoverable_heal(),
|
||||
!Error::Storage(EcstoreError::from(status)).is_recoverable_heal(),
|
||||
"cancellation text alone must not change application error classification"
|
||||
);
|
||||
}
|
||||
@@ -718,7 +718,7 @@ mod tests {
|
||||
for error in [
|
||||
Error::Disk(DiskError::from(status.clone())),
|
||||
Error::Storage(EcstoreError::from(status.clone())),
|
||||
Error::Io(std::io::Error::from(DiskError::from(status.clone()))),
|
||||
Error::Io(std::io::Error::from(DiskError::from(status))),
|
||||
] {
|
||||
assert!(!error.is_recoverable_heal(), "application RPC errors must stay terminal: {error:?}");
|
||||
}
|
||||
|
||||
@@ -543,7 +543,7 @@ impl HealManager {
|
||||
.filter_map(|target| local_endpoints.get(target).cloned())
|
||||
.collect::<Vec<_>>();
|
||||
if targets.len() != state.replacement_targets.len() {
|
||||
deferred_recovery_sets.insert(state.set_disk_id.clone());
|
||||
deferred_recovery_sets.insert(state.set_disk_id);
|
||||
continue;
|
||||
}
|
||||
// A pending handoff may resolve the predecessor to
|
||||
|
||||
@@ -257,8 +257,6 @@ impl HealManager {
|
||||
let notify_clone = notify.clone();
|
||||
let root_recovery_clone = root_recovery.clone();
|
||||
let manager_cancel_token = cancel_token.clone();
|
||||
let task_type_label_for_spawn = task_type_label.clone();
|
||||
let task_set_label_for_spawn = task_set_label.clone();
|
||||
let config_for_spawn = config.clone();
|
||||
|
||||
// start heal task
|
||||
@@ -270,8 +268,8 @@ impl HealManager {
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
task_id,
|
||||
priority = ?task_priority,
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
heal_type = %task_type_label,
|
||||
set = %task_set_label,
|
||||
state = "task_started",
|
||||
"Heal scheduler task started"
|
||||
);
|
||||
@@ -286,8 +284,8 @@ impl HealManager {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
task_id,
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
heal_type = %task_type_label,
|
||||
set = %task_set_label,
|
||||
state = "task_completed",
|
||||
"Heal scheduler task completed"
|
||||
);
|
||||
@@ -300,8 +298,8 @@ impl HealManager {
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "verification_unavailable",
|
||||
task_id,
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
heal_type = %task_type_label,
|
||||
set = %task_set_label,
|
||||
detail = %e,
|
||||
"Heal attempt completed without independent verification"
|
||||
);
|
||||
@@ -314,8 +312,8 @@ impl HealManager {
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
task_id,
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
heal_type = %task_type_label,
|
||||
set = %task_set_label,
|
||||
state = "task_retrying",
|
||||
retry_attempt = task.retry_attempts.saturating_add(1),
|
||||
error = %e,
|
||||
@@ -328,8 +326,8 @@ impl HealManager {
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
state = "task_failed",
|
||||
task_id,
|
||||
heal_type = %task_type_label_for_spawn,
|
||||
set = %task_set_label_for_spawn,
|
||||
heal_type = %task_type_label,
|
||||
set = %task_set_label,
|
||||
error = %e,
|
||||
"Heal scheduler task failed"
|
||||
});
|
||||
@@ -494,11 +492,29 @@ impl HealManager {
|
||||
}
|
||||
publish_verified_mrf_repair_events(¬ice_targets, &completed_status_for_verified_events);
|
||||
publish_unverified_legacy_mrf_events(¬ice_targets, &completed_status_for_verified_events, &result);
|
||||
// Single-object MRF attempts retain one storage result.
|
||||
// Use the original result even if the report was size
|
||||
// limited, without certifying independent integrity.
|
||||
let unverified_repair_bytes = if completed_status == HealTaskStatus::Completed
|
||||
&& matches!(&result, Err(Error::LegacyMrfUnverified { .. }))
|
||||
{
|
||||
task.result_items
|
||||
.read()
|
||||
.await
|
||||
.iter()
|
||||
.find(|(_, item)| item.drives_healed().is_some_and(|healed| healed > 0))
|
||||
.map(|(_, item)| u64::try_from(item.object_size).unwrap_or(u64::MAX))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
// update statistics
|
||||
let mut stats = statistics_clone.write().await;
|
||||
match completed_status {
|
||||
HealTaskStatus::Completed if matches!(&result, Err(Error::LegacyMrfUnverified { .. })) => {
|
||||
stats.update_unverified_task_completion();
|
||||
if let Some(bytes) = unverified_repair_bytes {
|
||||
stats.add_unverified_repair(bytes);
|
||||
}
|
||||
}
|
||||
HealTaskStatus::Completed => {
|
||||
stats.update_task_completion(true);
|
||||
|
||||
@@ -1106,7 +1106,10 @@ impl HealStorageAPI for MockStorage {
|
||||
if bucket.starts_with("legacy-ambiguity-scheduler-") {
|
||||
return Err(Error::Storage(EcstoreError::DeleteRecoveryLegacyAmbiguous));
|
||||
}
|
||||
if bucket == "legacy-held-pending-scheduler" {
|
||||
if matches!(
|
||||
bucket,
|
||||
"legacy-held-pending-scheduler" | "legacy-held-repaired-scheduler" | "legacy-held-repaired-oversized-scheduler"
|
||||
) {
|
||||
let drives = vec![rustfs_madmin::heal_commands::HealDriveInfo {
|
||||
state: "ok".to_owned(),
|
||||
..Default::default()
|
||||
@@ -1114,8 +1117,25 @@ impl HealStorageAPI for MockStorage {
|
||||
return Ok((
|
||||
HealResultItem {
|
||||
legacy_identity_unverified: true,
|
||||
detail: rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned(),
|
||||
before: rustfs_madmin::heal_commands::Infos { drives: drives.clone() },
|
||||
resolved_version_id: Some([0; 16]),
|
||||
disk_count: 1,
|
||||
object_size: 8192,
|
||||
detail: if bucket == "legacy-held-repaired-oversized-scheduler" {
|
||||
"x".repeat(MAX_COMPLETED_HEAL_RESULT_BYTES)
|
||||
} else {
|
||||
rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned()
|
||||
},
|
||||
before: rustfs_madmin::heal_commands::Infos {
|
||||
drives: vec![rustfs_madmin::heal_commands::HealDriveInfo {
|
||||
state: if bucket.starts_with("legacy-held-repaired-") {
|
||||
"missing"
|
||||
} else {
|
||||
"ok"
|
||||
}
|
||||
.to_owned(),
|
||||
..Default::default()
|
||||
}],
|
||||
},
|
||||
after: rustfs_madmin::heal_commands::Infos { drives },
|
||||
..Default::default()
|
||||
},
|
||||
@@ -2171,11 +2191,32 @@ fn only_typed_mrf_result_can_pause_unverified_legacy() {
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn mrf_legacy_completed_attempt_retains_responsibility_without_failed_statistics() {
|
||||
mrf_legacy_completed_attempt_scenario(false, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn mrf_legacy_repaired_attempt_retains_responsibility_and_physical_statistics() {
|
||||
mrf_legacy_completed_attempt_scenario(true, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn mrf_legacy_repair_statistics_survive_completed_result_truncation() {
|
||||
mrf_legacy_completed_attempt_scenario(true, true).await;
|
||||
}
|
||||
|
||||
async fn mrf_legacy_completed_attempt_scenario(repaired: bool, oversized_result: bool) {
|
||||
use crate::heal::failure_logging::test_support::FailureLogCapture;
|
||||
|
||||
let capture = FailureLogCapture::default();
|
||||
let _subscriber = tracing::subscriber::set_default(capture.clone());
|
||||
let (target, _) = unverified_legacy_fixture("legacy-held-pending-scheduler");
|
||||
let bucket = if oversized_result {
|
||||
"legacy-held-repaired-oversized-scheduler"
|
||||
} else if repaired {
|
||||
"legacy-held-repaired-scheduler"
|
||||
} else {
|
||||
"legacy-held-pending-scheduler"
|
||||
};
|
||||
let (target, _) = unverified_legacy_fixture(bucket);
|
||||
let bucket = target.bucket.clone();
|
||||
let anchor = target.durable_anchor.clone().expect("durable MRF anchor");
|
||||
let manager = HealManager::new_without_root_recovery_for_test(Arc::new(MockStorage), None);
|
||||
@@ -2202,8 +2243,15 @@ async fn mrf_legacy_completed_attempt_retains_responsibility_without_failed_stat
|
||||
assert_eq!(stats.failed_tasks, 0);
|
||||
assert_eq!(stats.successful_tasks, 0);
|
||||
assert_eq!(stats.total_objects_healed, 0);
|
||||
assert_eq!(stats.total_bytes_healed, 0);
|
||||
assert_eq!(stats.total_unverified_objects_repaired, u64::from(repaired));
|
||||
assert_eq!(stats.total_unverified_bytes_repaired, if repaired { 8192 } else { 0 });
|
||||
let report = manager.get_task_report(&receipt.task_id).await.expect("retained task report");
|
||||
assert_eq!(report.status, HealTaskStatus::Completed);
|
||||
if oversized_result {
|
||||
assert!(report.result_items_truncated);
|
||||
assert!(report.result_items.is_empty());
|
||||
}
|
||||
let outcome = report.outcome.expect("retained unknown outcome");
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
|
||||
@@ -50,6 +50,7 @@ use uuid::Uuid;
|
||||
|
||||
use crate::heal::task::{HealOptions, HealPriority, HealRequest, HealType};
|
||||
|
||||
pub mod management;
|
||||
mod partial_write;
|
||||
pub use partial_write::{MrfLegacyResponsibility, MrfOperatorAcceptance};
|
||||
use partial_write::{PartialWrites, ResponsibilityCheckpoint};
|
||||
@@ -64,6 +65,8 @@ static GLOBAL_MRF_LIFECYCLE_SENDER: std::sync::OnceLock<mpsc::Sender<MrfLifecycl
|
||||
pub enum MrfLifecycleControlError {
|
||||
#[error("MRF lifecycle controller is unavailable")]
|
||||
Unavailable,
|
||||
#[error("MRF read query budget is exhausted; retry later")]
|
||||
Busy,
|
||||
#[error("the MRF checkpoint is unreadable; retain its files and restore a compatible reader")]
|
||||
CheckpointUnreadable,
|
||||
#[error("the requested responsibility changed or does not exist")]
|
||||
@@ -94,7 +97,7 @@ pub struct MrfLegacyRiskAcceptanceRequest {
|
||||
pub request_id: Uuid,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MrfLegacyResponsibilitySnapshot {
|
||||
pub contract_version: u32,
|
||||
@@ -119,14 +122,19 @@ pub async fn list_legacy_responsibilities(
|
||||
"cursor must be non-nil and limit must be between 1 and 256".to_string(),
|
||||
));
|
||||
}
|
||||
let permit = management::read_permit()?;
|
||||
let sender = GLOBAL_MRF_LIFECYCLE_SENDER
|
||||
.get()
|
||||
.ok_or(MrfLifecycleControlError::Unavailable)?;
|
||||
let (response, receiver) = tokio::sync::oneshot::channel();
|
||||
sender
|
||||
.send(MrfLifecycleCommand::List { after, limit, response })
|
||||
.await
|
||||
.map_err(|_| MrfLifecycleControlError::Unavailable)?;
|
||||
.try_send(MrfLifecycleCommand::List {
|
||||
after,
|
||||
limit,
|
||||
response,
|
||||
_permit: permit,
|
||||
})
|
||||
.map_err(management::queue_error)?;
|
||||
receiver.await.map_err(|_| MrfLifecycleControlError::Unavailable)?
|
||||
}
|
||||
|
||||
@@ -181,7 +189,13 @@ pub async fn accept_unverified_legacy_risk(request: MrfLegacyRiskAcceptanceReque
|
||||
}
|
||||
|
||||
enum MrfLifecycleCommand {
|
||||
Get {
|
||||
id: Uuid,
|
||||
_permit: tokio::sync::OwnedSemaphorePermit,
|
||||
response: tokio::sync::oneshot::Sender<Result<management::MrfResponsibilityObservation, MrfLifecycleControlError>>,
|
||||
},
|
||||
List {
|
||||
_permit: tokio::sync::OwnedSemaphorePermit,
|
||||
after: Option<Uuid>,
|
||||
limit: usize,
|
||||
response: tokio::sync::oneshot::Sender<Result<MrfLegacyResponsibilitySnapshot, MrfLifecycleControlError>>,
|
||||
@@ -1325,8 +1339,31 @@ impl MrfRuntime {
|
||||
|
||||
async fn handle_lifecycle_command(&mut self, manager: &HealManager, command: MrfLifecycleCommand) {
|
||||
match command {
|
||||
MrfLifecycleCommand::List { after, limit, response } => {
|
||||
let _ = response.send(self.list_unverified_legacy(after, limit));
|
||||
MrfLifecycleCommand::List {
|
||||
after,
|
||||
limit,
|
||||
response,
|
||||
_permit,
|
||||
} => {
|
||||
if !response.is_closed() {
|
||||
let _ = response.send(self.list_unverified_legacy(after, limit));
|
||||
}
|
||||
return;
|
||||
}
|
||||
MrfLifecycleCommand::Get { id, response, _permit } => {
|
||||
if !response.is_closed() {
|
||||
let result = if self.replay_readable {
|
||||
Ok(management::MrfResponsibilityObservation {
|
||||
contract_version: 1,
|
||||
observed_at_ms: management::now_ms(),
|
||||
responsibility: self.partial_writes.responsibility(id),
|
||||
})
|
||||
} else {
|
||||
Err(MrfLifecycleControlError::CheckpointUnreadable)
|
||||
};
|
||||
let _ = response.send(result);
|
||||
}
|
||||
return;
|
||||
}
|
||||
MrfLifecycleCommand::Recheck {
|
||||
responsibility_id,
|
||||
@@ -1826,6 +1863,7 @@ impl MrfRuntime {
|
||||
gauge!("rustfs_heal_mrf_legacy_generation_unknown").set(metric_f64(lifecycle.legacy_generation_unknown_count));
|
||||
gauge!("rustfs_heal_mrf_bucket_incarnation_changed").set(metric_f64(lifecycle.bucket_incarnation_changed_count));
|
||||
gauge!("rustfs_heal_mrf_operator_accepted_unverified").set(metric_f64(lifecycle.operator_accepted_count));
|
||||
management::publish_summary(self, &lifecycle);
|
||||
let now_ms = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map(|duration| u64::try_from(duration.as_millis()).unwrap_or(u64::MAX))
|
||||
@@ -2199,7 +2237,7 @@ async fn replay_into(
|
||||
// runtime dispatches it after publishing the successor, avoiding
|
||||
// a duplicate task in the startup-replay and steady-state paths.
|
||||
if let Ok(Some(anchor)) = manager.durable_mrf_repair_anchor(&intent).await {
|
||||
durable_replay_anchors.push(anchor.clone());
|
||||
durable_replay_anchors.push(anchor);
|
||||
}
|
||||
let restored = intent_digest(&intent).and_then(|digest| take_lifecycle_record(&mut lifecycle, digest));
|
||||
partial_writes.push((intent, restored));
|
||||
@@ -2499,6 +2537,9 @@ async fn finalize_mrf_consumer(
|
||||
lifecycle_receiver.close();
|
||||
while let Some(command) = lifecycle_receiver.recv().await {
|
||||
match command {
|
||||
MrfLifecycleCommand::Get { response, .. } => {
|
||||
let _ = response.send(Err(MrfLifecycleControlError::Unavailable));
|
||||
}
|
||||
MrfLifecycleCommand::List { response, .. } => {
|
||||
let _ = response.send(Err(MrfLifecycleControlError::Unavailable));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Bounded runtime observations. These values never authorize lifecycle changes or prove retirement.
|
||||
|
||||
use super::{GLOBAL_MRF_LIFECYCLE_SENDER, MrfLegacyResponsibility, MrfLifecycleCommand, MrfLifecycleControlError, MrfRuntime};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::{Arc, LazyLock, RwLock};
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot};
|
||||
use uuid::Uuid;
|
||||
|
||||
const READ_CAPACITY: usize = 8;
|
||||
static READ_BUDGET: LazyLock<Arc<Semaphore>> = LazyLock::new(|| Arc::new(Semaphore::new(READ_CAPACITY)));
|
||||
static SUMMARY: RwLock<Option<MrfLifecycleSummary>> = RwLock::new(None);
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MrfLifecycleSummary {
|
||||
pub contract_version: u32,
|
||||
pub sampled_at_ms: u64,
|
||||
pub checkpoint_readable: bool,
|
||||
pub checkpoint_owner: Option<Uuid>,
|
||||
pub checkpoint_sequence: Option<u64>,
|
||||
pub active_responsibility_count: usize,
|
||||
pub held_count: usize,
|
||||
pub legacy_generation_unknown_count: usize,
|
||||
pub bucket_incarnation_changed_count: usize,
|
||||
pub operator_accepted_count: usize,
|
||||
pub best_effort_queue_depth: usize,
|
||||
pub responsibility_bytes: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MrfResponsibilityObservation {
|
||||
pub contract_version: u32,
|
||||
pub observed_at_ms: u64,
|
||||
/// Absence means not observed in this runtime, never verified or discharged.
|
||||
pub responsibility: Option<MrfLegacyResponsibility>,
|
||||
}
|
||||
|
||||
/// Returns the last published sample without scanning or queueing work.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns unavailable until the consumer publishes a sample, or after it stops.
|
||||
pub fn lifecycle_summary() -> Result<MrfLifecycleSummary, MrfLifecycleControlError> {
|
||||
if GLOBAL_MRF_LIFECYCLE_SENDER.get().is_none_or(mpsc::Sender::is_closed) {
|
||||
return Err(MrfLifecycleControlError::Unavailable);
|
||||
}
|
||||
SUMMARY
|
||||
.read()
|
||||
.map_err(|_| MrfLifecycleControlError::Unavailable)?
|
||||
.clone()
|
||||
.ok_or(MrfLifecycleControlError::Unavailable)
|
||||
}
|
||||
|
||||
/// Observes one current responsibility, including Active entries excluded from legacy lists.
|
||||
/// Cancellation skips work that has not started; observations are not durable success receipts.
|
||||
///
|
||||
/// # Errors
|
||||
/// Rejects nil IDs, an exhausted read budget, or an unavailable/unreadable controller.
|
||||
pub async fn responsibility(id: Uuid) -> Result<MrfResponsibilityObservation, MrfLifecycleControlError> {
|
||||
if id.is_nil() {
|
||||
return Err(MrfLifecycleControlError::InvalidAction("responsibility ID must be non-nil".into()));
|
||||
}
|
||||
let permit = read_permit()?;
|
||||
let sender = GLOBAL_MRF_LIFECYCLE_SENDER
|
||||
.get()
|
||||
.ok_or(MrfLifecycleControlError::Unavailable)?;
|
||||
let (response, receiver) = oneshot::channel();
|
||||
sender
|
||||
.try_send(MrfLifecycleCommand::Get {
|
||||
id,
|
||||
response,
|
||||
_permit: permit,
|
||||
})
|
||||
.map_err(queue_error)?;
|
||||
receiver.await.map_err(|_| MrfLifecycleControlError::Unavailable)?
|
||||
}
|
||||
|
||||
pub(super) fn read_permit() -> Result<OwnedSemaphorePermit, MrfLifecycleControlError> {
|
||||
READ_BUDGET
|
||||
.clone()
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| MrfLifecycleControlError::Busy)
|
||||
}
|
||||
|
||||
pub(super) fn queue_error<T>(error: mpsc::error::TrySendError<T>) -> MrfLifecycleControlError {
|
||||
match error {
|
||||
mpsc::error::TrySendError::Full(_) => MrfLifecycleControlError::Busy,
|
||||
mpsc::error::TrySendError::Closed(_) => MrfLifecycleControlError::Unavailable,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn now_ms() -> u64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_or(0, |time| u64::try_from(time.as_millis()).unwrap_or(u64::MAX))
|
||||
}
|
||||
|
||||
pub(super) fn publish_summary(runtime: &MrfRuntime, counts: &super::partial_write::PartialWriteLifecycleSummary) {
|
||||
let unverified = counts.held_count
|
||||
+ counts.legacy_generation_unknown_count
|
||||
+ counts.bucket_incarnation_changed_count
|
||||
+ counts.operator_accepted_count;
|
||||
let summary = MrfLifecycleSummary {
|
||||
contract_version: 1,
|
||||
sampled_at_ms: now_ms(),
|
||||
checkpoint_readable: runtime.replay_readable,
|
||||
checkpoint_owner: runtime.runtime_checkpoint.map(|(owner, _)| owner),
|
||||
checkpoint_sequence: runtime.runtime_checkpoint.map(|(_, sequence)| sequence),
|
||||
active_responsibility_count: runtime.partial_writes.depth().saturating_sub(unverified),
|
||||
held_count: counts.held_count,
|
||||
legacy_generation_unknown_count: counts.legacy_generation_unknown_count,
|
||||
bucket_incarnation_changed_count: counts.bucket_incarnation_changed_count,
|
||||
operator_accepted_count: counts.operator_accepted_count,
|
||||
best_effort_queue_depth: runtime.queue.depth(),
|
||||
responsibility_bytes: runtime.partial_writes.bytes(),
|
||||
};
|
||||
if let Ok(mut published) = SUMMARY.write() {
|
||||
*published = Some(summary);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::{MRF_LIFECYCLE_FORMAT_VERSION, MrfConsumerConfig, MrfQueue, PartialWrites, intent_digest};
|
||||
use super::*;
|
||||
use crate::heal::manager::HealManager;
|
||||
use crate::heal::storage::ECStoreHealStorage;
|
||||
use rustfs_common::mrf_channel::{MrfIntent, MrfKind, MrfScope};
|
||||
|
||||
#[test]
|
||||
fn cancelled_read_holds_budget_until_dequeued() {
|
||||
let budget = Arc::new(Semaphore::new(1));
|
||||
let (response, receiver) = oneshot::channel();
|
||||
let command = MrfLifecycleCommand::Get {
|
||||
id: Uuid::new_v4(),
|
||||
response,
|
||||
_permit: budget.clone().try_acquire_owned().unwrap(),
|
||||
};
|
||||
drop(receiver);
|
||||
assert!(budget.clone().try_acquire_owned().is_err());
|
||||
drop(command);
|
||||
assert!(budget.try_acquire_owned().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn management_queries_preserve_active_records_and_unreadable_checkpoint() {
|
||||
let env = rustfs_test_utils::TestECStoreEnv::builder()
|
||||
.prefix("mrf_management_queries")
|
||||
.build()
|
||||
.await;
|
||||
let manager =
|
||||
HealManager::new_without_root_recovery_for_test(Arc::new(ECStoreHealStorage::new(env.ecstore.clone())), None);
|
||||
let mut runtime = MrfRuntime {
|
||||
queue: MrfQueue::new(4, 8192),
|
||||
partial_writes: PartialWrites::default(),
|
||||
config: MrfConsumerConfig::default(),
|
||||
checkpoint_owner: Uuid::new_v4(),
|
||||
next_checkpoint_sequence: 2,
|
||||
lifecycle_format_version: MRF_LIFECYCLE_FORMAT_VERSION,
|
||||
new_since_flush: 0,
|
||||
dirty: false,
|
||||
journal_on_disk: true,
|
||||
retain_replay_journal: true,
|
||||
replay_readable: true,
|
||||
durable_replay_anchors: Vec::new(),
|
||||
replay_cleanup: None,
|
||||
runtime_checkpoint: Some((Uuid::new_v4(), 1)),
|
||||
backoff_until: None,
|
||||
};
|
||||
for (key, source) in [("active", Some(Uuid::new_v4())), ("legacy", None)] {
|
||||
runtime
|
||||
.admit_partial_write(
|
||||
MrfIntent {
|
||||
bucket: Arc::from("management"),
|
||||
object: Arc::from(key),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
delete_marker_purge: None,
|
||||
scope: Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
}),
|
||||
lease: None,
|
||||
enqueued_at_ms: 1,
|
||||
attempts: 0,
|
||||
},
|
||||
source,
|
||||
)
|
||||
.expect("retain responsibility fixture");
|
||||
}
|
||||
let active = runtime
|
||||
.partial_writes
|
||||
.checkpoint_records(intent_digest)
|
||||
.into_iter()
|
||||
.find(|row| row.source_bucket_incarnation_id.is_some())
|
||||
.expect("source-bound responsibility")
|
||||
.responsibility_id;
|
||||
let before = runtime.snapshot();
|
||||
let (sender, mut receiver) = mpsc::channel(16);
|
||||
assert!(GLOBAL_MRF_LIFECYCLE_SENDER.set(sender).is_ok(), "query test requires an isolated process");
|
||||
assert!(matches!(lifecycle_summary(), Err(MrfLifecycleControlError::Unavailable)));
|
||||
publish_summary(&runtime, &runtime.partial_writes.lifecycle_summary());
|
||||
let mut sample = lifecycle_summary().expect("published summary");
|
||||
assert_eq!(sample.active_responsibility_count, 1);
|
||||
assert_eq!(sample.legacy_generation_unknown_count, 1);
|
||||
assert!(sample.checkpoint_readable);
|
||||
sample.sampled_at_ms = 0;
|
||||
*SUMMARY.write().expect("summary fixture") = Some(sample.clone());
|
||||
|
||||
let permits: Vec<_> = (0..READ_CAPACITY).map(|_| read_permit().expect("read slot")).collect();
|
||||
assert!(matches!(responsibility(active).await, Err(MrfLifecycleControlError::Busy)));
|
||||
assert!(matches!(
|
||||
super::super::list_legacy_responsibilities(None, 1).await,
|
||||
Err(MrfLifecycleControlError::Busy)
|
||||
));
|
||||
drop(permits);
|
||||
|
||||
let (observed, ()) = tokio::join!(responsibility(active), async {
|
||||
let command = receiver.recv().await.expect("queued Get");
|
||||
runtime.handle_lifecycle_command(&manager, command).await;
|
||||
});
|
||||
let row = observed
|
||||
.expect("Get succeeds")
|
||||
.responsibility
|
||||
.expect("Active responsibility remains observable");
|
||||
assert_eq!(row.responsibility_id, active);
|
||||
assert_eq!(row.status, "active");
|
||||
let (listed, ()) = tokio::join!(super::super::list_legacy_responsibilities(None, 2), async {
|
||||
let command = receiver.recv().await.expect("queued List");
|
||||
runtime.handle_lifecycle_command(&manager, command).await;
|
||||
});
|
||||
let listed = listed.expect("legacy list succeeds");
|
||||
assert_eq!(listed.responsibilities.len(), 1);
|
||||
assert_ne!(listed.responsibilities[0].responsibility_id, active);
|
||||
assert_eq!(runtime.snapshot(), before, "observations must preserve the retained payloads");
|
||||
assert_eq!(
|
||||
lifecycle_summary().expect("cached summary").sampled_at_ms,
|
||||
sample.sampled_at_ms,
|
||||
"read commands must not republish summary metrics"
|
||||
);
|
||||
|
||||
for list in [false, true] {
|
||||
let cancelled = tokio::spawn(async move {
|
||||
if list {
|
||||
super::super::list_legacy_responsibilities(None, 1).await.map(|_| ())
|
||||
} else {
|
||||
responsibility(active).await.map(|_| ())
|
||||
}
|
||||
});
|
||||
let command = tokio::time::timeout(std::time::Duration::from_secs(5), receiver.recv())
|
||||
.await
|
||||
.expect("query is enqueued promptly")
|
||||
.expect("queued cancellable query");
|
||||
cancelled.abort();
|
||||
assert!(cancelled.await.expect_err("query task cancelled").is_cancelled());
|
||||
assert_eq!(
|
||||
READ_BUDGET.available_permits(),
|
||||
READ_CAPACITY - 1,
|
||||
"queued work retains its slot after client cancellation"
|
||||
);
|
||||
runtime.handle_lifecycle_command(&manager, command).await;
|
||||
assert_eq!(READ_BUDGET.available_permits(), READ_CAPACITY);
|
||||
assert_eq!(runtime.snapshot(), before);
|
||||
}
|
||||
|
||||
runtime.replay_readable = false;
|
||||
publish_summary(&runtime, &runtime.partial_writes.lifecycle_summary());
|
||||
assert!(
|
||||
!lifecycle_summary()
|
||||
.expect("unreadable summary remains observable")
|
||||
.checkpoint_readable
|
||||
);
|
||||
let (observed, ()) = tokio::join!(responsibility(active), async {
|
||||
let command = receiver.recv().await.expect("Get against unreadable checkpoint");
|
||||
runtime.handle_lifecycle_command(&manager, command).await;
|
||||
});
|
||||
assert!(matches!(observed, Err(MrfLifecycleControlError::CheckpointUnreadable)));
|
||||
let (listed, ()) = tokio::join!(super::super::list_legacy_responsibilities(None, 1), async {
|
||||
let command = receiver.recv().await.expect("List against unreadable checkpoint");
|
||||
runtime.handle_lifecycle_command(&manager, command).await;
|
||||
});
|
||||
assert!(matches!(listed, Err(MrfLifecycleControlError::CheckpointUnreadable)));
|
||||
assert_eq!(runtime.snapshot(), before);
|
||||
receiver.close();
|
||||
assert!(matches!(responsibility(active).await, Err(MrfLifecycleControlError::Unavailable)));
|
||||
assert!(
|
||||
matches!(lifecycle_summary(), Err(MrfLifecycleControlError::Unavailable)),
|
||||
"a stale sample cannot hide shutdown"
|
||||
);
|
||||
assert_eq!(READ_BUDGET.available_permits(), READ_CAPACITY);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn management_query_bounds_and_queue_errors_are_explicit() {
|
||||
assert!(matches!(
|
||||
responsibility(Uuid::nil()).await,
|
||||
Err(MrfLifecycleControlError::InvalidAction(_))
|
||||
));
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
sender.try_send(()).expect("fill queue");
|
||||
assert!(matches!(
|
||||
queue_error(sender.try_send(()).expect_err("full queue")),
|
||||
MrfLifecycleControlError::Busy
|
||||
));
|
||||
drop(receiver);
|
||||
assert!(matches!(
|
||||
queue_error(sender.try_send(()).expect_err("closed queue")),
|
||||
MrfLifecycleControlError::Unavailable
|
||||
));
|
||||
assert_eq!(READ_BUDGET.available_permits(), READ_CAPACITY);
|
||||
}
|
||||
}
|
||||
@@ -776,29 +776,9 @@ impl PartialWrites {
|
||||
) -> (Vec<MrfLegacyResponsibility>, Option<Uuid>) {
|
||||
let mut selected = BTreeMap::new();
|
||||
for entry in self.entries.values() {
|
||||
let (status, bucket_incarnation_id, held_since_ms) = match &entry.state {
|
||||
ResponsibilityState::Active {} if entry.generation_unobserved() => ("legacy_generation_unobserved", None, None),
|
||||
ResponsibilityState::Active {} => continue,
|
||||
ResponsibilityState::SourceAndCurrentGenerationUnknown { since_ms, .. } => {
|
||||
("source_and_current_generation_unknown", None, Some(*since_ms))
|
||||
}
|
||||
ResponsibilityState::HeldUnverifiedLegacy {
|
||||
bucket_incarnation_id,
|
||||
since_ms,
|
||||
} => ("held_unverified_legacy", Some(*bucket_incarnation_id), Some(*since_ms)),
|
||||
ResponsibilityState::LegacyGenerationUnknown {
|
||||
observed_bucket_incarnation_id,
|
||||
detected_at_ms,
|
||||
} => ("legacy_generation_unknown", Some(*observed_bucket_incarnation_id), Some(*detected_at_ms)),
|
||||
ResponsibilityState::BucketIncarnationChanged {
|
||||
observed_bucket_incarnation_id,
|
||||
detected_at_ms,
|
||||
..
|
||||
} => ("bucket_incarnation_changed", Some(*observed_bucket_incarnation_id), Some(*detected_at_ms)),
|
||||
ResponsibilityState::OperatorAcceptedUnverified {
|
||||
bucket_incarnation_id, ..
|
||||
} => ("operator_accepted_unverified", Some(*bucket_incarnation_id), None),
|
||||
};
|
||||
if !entry.is_parked() {
|
||||
continue;
|
||||
}
|
||||
if after.is_some_and(|after| entry.responsibility_id <= after) {
|
||||
continue;
|
||||
}
|
||||
@@ -809,33 +789,7 @@ impl PartialWrites {
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let row = MrfLegacyResponsibility {
|
||||
responsibility_id: entry.responsibility_id,
|
||||
bucket: entry.intent.bucket.to_string(),
|
||||
object: entry.intent.object.to_string(),
|
||||
source_bucket_incarnation_id: entry.source_bucket_incarnation_id,
|
||||
version_id: entry
|
||||
.intent
|
||||
.version_id
|
||||
.and_then(|bytes| Uuid::from_slice(&bytes).ok())
|
||||
.filter(|version| !version.is_nil())
|
||||
.map(|version| version.to_string()),
|
||||
pool_index: entry.intent.scope.map(|scope| scope.pool_index),
|
||||
set_index: entry.intent.scope.map(|scope| scope.set_index),
|
||||
bucket_incarnation_id,
|
||||
enqueued_at_ms: entry.intent.enqueued_at_ms,
|
||||
status,
|
||||
held_since_ms,
|
||||
accepted: entry.last_operator_acceptance.clone(),
|
||||
last_identity_probe_result: entry.last_identity_probe_result,
|
||||
last_identity_probe_at_ms: entry.last_identity_probe_at_ms,
|
||||
next_identity_probe_at_ms: entry.generation_unobserved().then(|| {
|
||||
unix_now_ms().saturating_add(
|
||||
u64::try_from(entry.next_attempt.saturating_duration_since(Instant::now()).as_millis())
|
||||
.unwrap_or(u64::MAX),
|
||||
)
|
||||
}),
|
||||
};
|
||||
let row = Self::responsibility_row(entry);
|
||||
selected.insert(entry.responsibility_id, row);
|
||||
if selected.len() > limit + 1 {
|
||||
selected.pop_last();
|
||||
@@ -852,6 +806,65 @@ impl PartialWrites {
|
||||
(selected.into_values().collect(), next_cursor)
|
||||
}
|
||||
|
||||
pub(super) fn responsibility(&self, id: Uuid) -> Option<MrfLegacyResponsibility> {
|
||||
self.entries
|
||||
.values()
|
||||
.find(|entry| entry.responsibility_id == id)
|
||||
.map(Self::responsibility_row)
|
||||
}
|
||||
|
||||
fn responsibility_row(entry: &Responsibility) -> MrfLegacyResponsibility {
|
||||
let (status, bucket_incarnation_id, held_since_ms) = match &entry.state {
|
||||
ResponsibilityState::Active {} if entry.generation_unobserved() => ("legacy_generation_unobserved", None, None),
|
||||
ResponsibilityState::Active {} => ("active", None, None),
|
||||
ResponsibilityState::SourceAndCurrentGenerationUnknown { since_ms, .. } => {
|
||||
("source_and_current_generation_unknown", None, Some(*since_ms))
|
||||
}
|
||||
ResponsibilityState::HeldUnverifiedLegacy {
|
||||
bucket_incarnation_id,
|
||||
since_ms,
|
||||
} => ("held_unverified_legacy", Some(*bucket_incarnation_id), Some(*since_ms)),
|
||||
ResponsibilityState::LegacyGenerationUnknown {
|
||||
observed_bucket_incarnation_id,
|
||||
detected_at_ms,
|
||||
} => ("legacy_generation_unknown", Some(*observed_bucket_incarnation_id), Some(*detected_at_ms)),
|
||||
ResponsibilityState::BucketIncarnationChanged {
|
||||
observed_bucket_incarnation_id,
|
||||
detected_at_ms,
|
||||
..
|
||||
} => ("bucket_incarnation_changed", Some(*observed_bucket_incarnation_id), Some(*detected_at_ms)),
|
||||
ResponsibilityState::OperatorAcceptedUnverified {
|
||||
bucket_incarnation_id, ..
|
||||
} => ("operator_accepted_unverified", Some(*bucket_incarnation_id), None),
|
||||
};
|
||||
MrfLegacyResponsibility {
|
||||
responsibility_id: entry.responsibility_id,
|
||||
bucket: entry.intent.bucket.to_string(),
|
||||
object: entry.intent.object.to_string(),
|
||||
source_bucket_incarnation_id: entry.source_bucket_incarnation_id,
|
||||
version_id: entry
|
||||
.intent
|
||||
.version_id
|
||||
.and_then(|bytes| Uuid::from_slice(&bytes).ok())
|
||||
.filter(|version| !version.is_nil())
|
||||
.map(|version| version.to_string()),
|
||||
pool_index: entry.intent.scope.map(|scope| scope.pool_index),
|
||||
set_index: entry.intent.scope.map(|scope| scope.set_index),
|
||||
bucket_incarnation_id,
|
||||
enqueued_at_ms: entry.intent.enqueued_at_ms,
|
||||
status: status.to_string(),
|
||||
held_since_ms,
|
||||
accepted: entry.last_operator_acceptance.clone(),
|
||||
last_identity_probe_result: entry.last_identity_probe_result.map(str::to_string),
|
||||
last_identity_probe_at_ms: entry.last_identity_probe_at_ms,
|
||||
next_identity_probe_at_ms: entry.generation_unobserved().then(|| {
|
||||
unix_now_ms().saturating_add(
|
||||
u64::try_from(entry.next_attempt.saturating_duration_since(Instant::now()).as_millis()).unwrap_or(u64::MAX),
|
||||
)
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn replace_observed_incarnation(
|
||||
&mut self,
|
||||
responsibility_id: Uuid,
|
||||
@@ -1154,7 +1167,7 @@ fn validate_operator_audit_fields(actor: &str, reason: &str, reference: &str, re
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct MrfLegacyResponsibility {
|
||||
pub responsibility_id: Uuid,
|
||||
@@ -1166,11 +1179,11 @@ pub struct MrfLegacyResponsibility {
|
||||
pub set_index: Option<u32>,
|
||||
pub bucket_incarnation_id: Option<Uuid>,
|
||||
pub enqueued_at_ms: u64,
|
||||
pub status: &'static str,
|
||||
pub status: String,
|
||||
pub held_since_ms: Option<u64>,
|
||||
pub accepted: Option<MrfOperatorAcceptance>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub last_identity_probe_result: Option<&'static str>,
|
||||
pub last_identity_probe_result: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub last_identity_probe_at_ms: Option<u64>,
|
||||
/// Process-local schedule; replay restarts the conservative probe delay.
|
||||
@@ -1230,6 +1243,71 @@ mod tests {
|
||||
intent
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn management_query_scale_baseline_and_active_lookup() {
|
||||
for budget in [8 * 1024 * 1024, 128 * 1024 * 1024] {
|
||||
let mut writes = PartialWrites::default();
|
||||
let source = Uuid::new_v4();
|
||||
let started = std::time::Instant::now();
|
||||
for index in 0..100_000 {
|
||||
let item = MrfIntent {
|
||||
bucket: Arc::from("management-scale"),
|
||||
object: Arc::from(format!("key-{index:06}")),
|
||||
version_id: None,
|
||||
kind: MrfKind::PartialWrite,
|
||||
scope: Some(MrfScope {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
}),
|
||||
lease: None,
|
||||
enqueued_at_ms: 1,
|
||||
attempts: 0,
|
||||
delete_marker_purge: None,
|
||||
};
|
||||
if writes
|
||||
.admit_with_source_incarnation(item, if index % 10 == 0 { Some(source) } else { None }, 100_000, budget)
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
let admission_ms = started.elapsed().as_millis();
|
||||
assert!(writes.depth() > 1000);
|
||||
let active = writes
|
||||
.entries
|
||||
.values()
|
||||
.find(|entry| !entry.is_parked())
|
||||
.unwrap()
|
||||
.responsibility_id;
|
||||
assert_eq!(writes.responsibility(active).unwrap().status, "active");
|
||||
assert!(writes.responsibility(Uuid::new_v4()).is_none());
|
||||
let mut elapsed = Vec::new();
|
||||
let mut after = None;
|
||||
for _ in 0..32 {
|
||||
let start = std::time::Instant::now();
|
||||
let (rows, next) = writes.list_unverified_legacy(after, 256);
|
||||
let _ = writes.lifecycle_summary();
|
||||
assert!(writes.responsibility(active).is_some());
|
||||
elapsed.push(start.elapsed().as_micros());
|
||||
assert!(rows.len() <= 256);
|
||||
assert!(
|
||||
rows.iter()
|
||||
.all(|row| row.responsibility_id != active && after.is_none_or(|id| row.responsibility_id > id))
|
||||
);
|
||||
after = next;
|
||||
}
|
||||
elapsed.sort_unstable();
|
||||
eprintln!(
|
||||
"MRF query baseline: entries={} bytes={} admission_ms={} page_summary_id_p50_us={} p99_us={}",
|
||||
writes.depth(),
|
||||
writes.bytes(),
|
||||
admission_ms,
|
||||
elapsed[16],
|
||||
elapsed[31]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn risk_acceptance(
|
||||
responsibility_id: Uuid,
|
||||
expected_bucket_incarnation_id: Uuid,
|
||||
|
||||
@@ -179,7 +179,7 @@ async fn unknown_source_deleted_bucket_probes_preserve_errors_and_bound_retries(
|
||||
assert_eq!(row.bucket_incarnation_id, None);
|
||||
assert_eq!(row.source_bucket_incarnation_id, None);
|
||||
assert_eq!(
|
||||
row.last_identity_probe_result,
|
||||
row.last_identity_probe_result.as_deref(),
|
||||
Some(if matches!(outcome, Lookup::Identity(None)) {
|
||||
"unresolved"
|
||||
} else {
|
||||
|
||||
@@ -395,6 +395,77 @@ fn empty_events_keep_dirty_flush_failure_retry_and_cleanup_gates() {
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_park_replay_requires_committed_checkpoint() {
|
||||
if isolated("legacy_park_replay_requires_committed_checkpoint") {
|
||||
return;
|
||||
}
|
||||
tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("runtime")
|
||||
.block_on(async {
|
||||
let (env, manager, mut runtime, anchor) = bound_runtime().await;
|
||||
let original = runtime.partial_writes.checkpoint_records(intent_digest);
|
||||
assert!(runtime.flush().await, "the active responsibility must be durable first");
|
||||
|
||||
// Losing the completed-task notice leaves the original active
|
||||
// owner replayable. No physical result alone can discharge it.
|
||||
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
|
||||
let replay = replay_into(&manager, &mut queue, &mut None).await;
|
||||
assert_eq!(replay.partial_writes.len(), original.len());
|
||||
assert!(replay.partial_writes.iter().all(|(_, record)| {
|
||||
!matches!(
|
||||
record.as_ref().expect("active companion").state,
|
||||
partial_write::ResponsibilityState::HeldUnverifiedLegacy { .. }
|
||||
)
|
||||
}));
|
||||
|
||||
note_mrf_unverified_legacy(MrfUnverifiedLegacyEvent { anchor: anchor.clone() });
|
||||
runtime.discharge_durable_replay_anchors();
|
||||
assert_eq!(runtime.partial_writes.lifecycle_summary().held_count, 1);
|
||||
let paths = env
|
||||
.disk_paths
|
||||
.iter()
|
||||
.map(|disk| (disk.join(".rustfs.sys"), disk.join(".mrf-park-sys-backup")))
|
||||
.collect::<Vec<_>>();
|
||||
for (path, backup) in &paths {
|
||||
fs::rename(path, backup).expect("preserve committed active checkpoint");
|
||||
fs::write(path, b"block held checkpoint publication").expect("inject publication failure");
|
||||
}
|
||||
let persisted = runtime.flush().await;
|
||||
for (path, backup) in &paths {
|
||||
fs::remove_file(path).expect("remove publication blocker");
|
||||
fs::rename(backup, path).expect("restore committed active checkpoint");
|
||||
}
|
||||
assert!(!persisted);
|
||||
assert!(runtime.dirty);
|
||||
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
|
||||
let replay = replay_into(&manager, &mut queue, &mut None).await;
|
||||
let mut restored = replay
|
||||
.partial_writes
|
||||
.into_iter()
|
||||
.map(|(_, record)| record.expect("previous active companion"))
|
||||
.collect::<Vec<_>>();
|
||||
let mut original = original;
|
||||
restored.sort_by_key(|record| record.responsibility_id);
|
||||
original.sort_by_key(|record| record.responsibility_id);
|
||||
assert_eq!(restored, original, "a crash before commit may conservatively repeat Deep");
|
||||
|
||||
assert!(runtime.flush().await, "commit the held state after storage recovers");
|
||||
let mut queue = MrfQueue::new(runtime.config.queue_capacity, runtime.config.journal_max_bytes);
|
||||
let replay = replay_into(&manager, &mut queue, &mut None).await;
|
||||
assert_eq!(replay.partial_writes.len(), original.len());
|
||||
assert!(replay.partial_writes.iter().any(|(_, record)| {
|
||||
matches!(
|
||||
record.as_ref().expect("committed held companion").state,
|
||||
partial_write::ResponsibilityState::HeldUnverifiedLegacy { .. }
|
||||
)
|
||||
}));
|
||||
assert!(runtime.retained_replay_journal(), "parking never releases responsibility");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_events_real_consumer_stop_retains_unresolved_journal() {
|
||||
if isolated("empty_events_real_consumer_stop_retains_unresolved_journal") {
|
||||
@@ -421,6 +492,7 @@ fn empty_events_real_consumer_stop_retains_unresolved_journal() {
|
||||
after: None,
|
||||
limit: 10,
|
||||
response,
|
||||
_permit: crate::heal::mrf_queue::management::read_permit().expect("MRF management read permit"),
|
||||
})
|
||||
.await
|
||||
.expect("query the actual consumer");
|
||||
|
||||
@@ -230,6 +230,7 @@ async fn assert_unreadable_lifecycle_is_retained(corrupt: fn(&mut Value)) {
|
||||
after: None,
|
||||
limit: 10,
|
||||
response,
|
||||
_permit: crate::heal::mrf_queue::management::read_permit().expect("MRF management read permit"),
|
||||
})
|
||||
.await
|
||||
.expect("query the actual consumer");
|
||||
|
||||
@@ -234,6 +234,7 @@ async fn check_reader_restart(version: u8) {
|
||||
after: None,
|
||||
limit: 3,
|
||||
response,
|
||||
_permit: crate::heal::mrf_queue::management::read_permit().expect("MRF management read permit"),
|
||||
})
|
||||
.await
|
||||
.expect("query the real consumer after its startup replay");
|
||||
|
||||
@@ -72,6 +72,7 @@ async fn owned_mrf_shutdown_drains_and_persists_before_ack_and_join() {
|
||||
after: None,
|
||||
limit: 1,
|
||||
response,
|
||||
_permit: crate::heal::mrf_queue::management::read_permit().expect("MRF management read permit"),
|
||||
})
|
||||
.await
|
||||
.expect("queued lifecycle query");
|
||||
|
||||
@@ -509,6 +509,12 @@ pub struct HealStatistics {
|
||||
/// Completed attempts without independent verification; not the current durable MRF backlog.
|
||||
#[serde(default)]
|
||||
pub unverified_tasks: u64,
|
||||
/// Physical repair attempts for unverified legacy objects, not verified outcomes or unique objects.
|
||||
#[serde(default)]
|
||||
pub total_unverified_objects_repaired: u64,
|
||||
/// Logical object bytes in those repair attempts, not physical disk or network bytes.
|
||||
#[serde(default)]
|
||||
pub total_unverified_bytes_repaired: u64,
|
||||
/// Running tasks
|
||||
pub running_tasks: u64,
|
||||
/// Total healed objects
|
||||
@@ -532,6 +538,8 @@ impl HealStatistics {
|
||||
successful_tasks: 0,
|
||||
failed_tasks: 0,
|
||||
unverified_tasks: 0,
|
||||
total_unverified_objects_repaired: 0,
|
||||
total_unverified_bytes_repaired: 0,
|
||||
running_tasks: 0,
|
||||
total_objects_healed: 0,
|
||||
total_bytes_healed: 0,
|
||||
@@ -558,6 +566,11 @@ impl HealStatistics {
|
||||
self.last_update_time = SystemTime::now();
|
||||
}
|
||||
|
||||
pub(crate) fn add_unverified_repair(&mut self, bytes: u64) {
|
||||
self.total_unverified_objects_repaired = self.total_unverified_objects_repaired.saturating_add(1);
|
||||
self.total_unverified_bytes_repaired = self.total_unverified_bytes_repaired.saturating_add(bytes);
|
||||
}
|
||||
|
||||
pub fn add_healed_objects(&mut self, count: u64, bytes: u64) {
|
||||
self.total_objects_healed += count;
|
||||
self.total_bytes_healed += bytes;
|
||||
@@ -928,15 +941,26 @@ mod tests {
|
||||
fn unverified_statistics_preserve_old_snapshot_compatibility() {
|
||||
let mut stats = HealStatistics::new();
|
||||
stats.update_unverified_task_completion();
|
||||
stats.add_unverified_repair(8192);
|
||||
assert_eq!(stats.unverified_tasks, 1);
|
||||
assert_eq!(stats.successful_tasks, 0);
|
||||
assert_eq!(stats.failed_tasks, 0);
|
||||
let mut wire = serde_json::to_value(&stats).expect("serialize task statistics");
|
||||
let current: HealStatistics = serde_json::from_value(wire.clone()).expect("read new statistics");
|
||||
assert_eq!(current.unverified_tasks, 1);
|
||||
wire.as_object_mut().expect("statistics object").remove("unverified_tasks");
|
||||
assert_eq!(current.total_unverified_objects_repaired, 1);
|
||||
assert_eq!(current.total_unverified_bytes_repaired, 8192);
|
||||
for field in [
|
||||
"unverified_tasks",
|
||||
"total_unverified_objects_repaired",
|
||||
"total_unverified_bytes_repaired",
|
||||
] {
|
||||
wire.as_object_mut().expect("statistics object").remove(field);
|
||||
}
|
||||
let old: HealStatistics = serde_json::from_value(wire).expect("read statistics without new field");
|
||||
assert_eq!(old.unverified_tasks, 0);
|
||||
assert_eq!(old.total_unverified_objects_repaired, 0);
|
||||
assert_eq!(old.total_unverified_bytes_repaired, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -348,10 +348,7 @@ mod tests {
|
||||
let enabled = std::env::var(ENABLE_ENV)
|
||||
.ok()
|
||||
.is_some_and(|value| matches!(value.as_str(), "1" | "true" | "TRUE" | "yes" | "YES"));
|
||||
if !enabled {
|
||||
return Ok(false);
|
||||
}
|
||||
Ok(true)
|
||||
Ok(enabled)
|
||||
}
|
||||
|
||||
fn run_current_test_in_mount_namespace() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
|
||||
@@ -514,17 +514,36 @@ impl HealTask {
|
||||
let receipt_missing = storage_result.receipt.is_none();
|
||||
if !self.record_verified_storage_receipt(&expected, storage_result.receipt).await {
|
||||
let ok_drive_state = DriveState::Ok.to_string();
|
||||
let healthy_legacy_without_repair = receipt_missing
|
||||
let selected_version_matches = storage_result.item.resolved_version_id.is_some_and(|resolved| {
|
||||
let requested = version_id.filter(|version| !version.is_empty());
|
||||
match requested {
|
||||
Some(requested) => Uuid::parse_str(requested).is_ok_and(|version| version.as_bytes() == &resolved),
|
||||
None => resolved == [0; 16],
|
||||
}
|
||||
});
|
||||
// A completed legacy repair can restore redundancy without proving
|
||||
// original content. Keep the same durable responsibility as an
|
||||
// already-healthy Deep result, including when this attempt repaired
|
||||
// a shard. Storage errors and invalid receipts never qualify.
|
||||
let healthy_legacy = receipt_missing
|
||||
&& storage_result.item.legacy_identity_unverified
|
||||
&& storage_result.item.drives_healed() == Some(0)
|
||||
&& !storage_result.item.after.drives.is_empty()
|
||||
&& selected_version_matches
|
||||
&& storage_result.item.drives_healed().is_some()
|
||||
&& storage_result.item.after.drives.len() == storage_result.item.disk_count
|
||||
&& storage_result
|
||||
.item
|
||||
.before
|
||||
.drives
|
||||
.iter()
|
||||
.zip(&storage_result.item.after.drives)
|
||||
.all(|(before, after)| before.endpoint == after.endpoint)
|
||||
&& storage_result
|
||||
.item
|
||||
.after
|
||||
.drives
|
||||
.iter()
|
||||
.all(|drive| drive.state == ok_drive_state);
|
||||
if healthy_legacy_without_repair {
|
||||
if healthy_legacy {
|
||||
self.outcome.write().await.record(HealObjectOutcome {
|
||||
deep_check_observed: false,
|
||||
quorum_selected_data_dir: None,
|
||||
@@ -534,7 +553,7 @@ impl HealTask {
|
||||
});
|
||||
}
|
||||
self.record_result_item(storage_result.item).await;
|
||||
return Err(if healthy_legacy_without_repair {
|
||||
return Err(if healthy_legacy {
|
||||
Error::LegacyMrfUnverified {
|
||||
target: format!("{bucket}/{object}"),
|
||||
}
|
||||
|
||||
@@ -3227,6 +3227,8 @@ fn healthy_legacy_heal_item() -> HealResultItem {
|
||||
}];
|
||||
HealResultItem {
|
||||
legacy_identity_unverified: true,
|
||||
resolved_version_id: Some([0; 16]),
|
||||
disk_count: 1,
|
||||
detail: rustfs_heal_contracts::heal_channel::LEGACY_OBJECT_IDENTITY_UNVERIFIED_DETAIL.to_owned(),
|
||||
before: Infos { drives: drives.clone() },
|
||||
after: Infos { drives },
|
||||
@@ -3234,6 +3236,34 @@ fn healthy_legacy_heal_item() -> HealResultItem {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mrf_legacy_repair_completes_unverified_without_second_deep() {
|
||||
let mut item = healthy_legacy_heal_item();
|
||||
item.before.drives[0].state = "missing".to_owned();
|
||||
item.object_size = 8192;
|
||||
let (task, storage) = mrf_task_for_item("legacy-repaired", item);
|
||||
|
||||
let error = task
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("physical repair cannot certify legacy identity");
|
||||
assert!(matches!(error, Error::LegacyMrfUnverified { .. }), "{error:?}");
|
||||
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.execution, HealExecutionOutcome::Completed);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.failed, 0);
|
||||
assert!(matches!(outcome.objects[0].disposition, HealObjectDisposition::Unknown));
|
||||
let items = task.get_seqed_result_items().await;
|
||||
assert_eq!(items.len(), 1);
|
||||
assert_eq!(items[0].1.drives_healed(), Some(1));
|
||||
assert_eq!(items[0].1.object_size, 8192);
|
||||
let opts = storage.object_heal_opts.lock().expect("physical repair options");
|
||||
assert_eq!(opts.len(), 1, "the first completed repair must supply the terminal classification");
|
||||
assert_eq!(opts[0].scan_mode, HealScanMode::Deep);
|
||||
}
|
||||
|
||||
fn mrf_task_for_item(object: &str, item: HealResultItem) -> (HealTask, Arc<MockStorage>) {
|
||||
let incarnation = Uuid::new_v4();
|
||||
let storage = Arc::new(MockStorage {
|
||||
@@ -3343,11 +3373,18 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
|
||||
"empty",
|
||||
"unhealthy",
|
||||
"unaligned",
|
||||
"wrong_endpoint",
|
||||
"wrong_disk_count",
|
||||
"wrong_version",
|
||||
"unknown_version",
|
||||
"versioned",
|
||||
"versioned_mismatch",
|
||||
"receipt",
|
||||
"storage_error",
|
||||
"dry_run",
|
||||
] {
|
||||
let mut item = healthy_legacy_heal_item();
|
||||
let requested_version = Uuid::new_v4();
|
||||
match case {
|
||||
"qualified" => item.detail = "updated diagnostic wording".to_owned(),
|
||||
"text_only" => item.legacy_identity_unverified = false,
|
||||
@@ -3361,9 +3398,20 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
|
||||
item.after.drives[0].state = "missing".to_owned();
|
||||
}
|
||||
"unaligned" => item.before.drives.clear(),
|
||||
"wrong_endpoint" => item.before.drives[0].endpoint = "another-disk".to_owned(),
|
||||
"wrong_disk_count" => item.disk_count = 4,
|
||||
"wrong_version" => item.resolved_version_id = Some(*Uuid::new_v4().as_bytes()),
|
||||
"unknown_version" => item.resolved_version_id = None,
|
||||
"versioned" => item.resolved_version_id = Some(*requested_version.as_bytes()),
|
||||
"versioned_mismatch" => item.resolved_version_id = Some(*Uuid::new_v4().as_bytes()),
|
||||
_ => {}
|
||||
}
|
||||
let (mut task, storage) = mrf_task_for_item("object", item);
|
||||
if matches!(case, "versioned" | "versioned_mismatch")
|
||||
&& let HealType::Object { version_id, .. } = &mut task.heal_type
|
||||
{
|
||||
*version_id = Some(requested_version.to_string());
|
||||
}
|
||||
if case == "receipt" {
|
||||
storage.heal_object_receipts.lock().expect("receipt fixture").insert(
|
||||
"object".to_owned(),
|
||||
@@ -3383,13 +3431,16 @@ async fn mrf_legacy_pause_requires_storage_owned_healthy_deep_result() {
|
||||
.execute()
|
||||
.await
|
||||
.expect_err("legacy result cannot discharge a durable responsibility");
|
||||
assert_eq!(matches!(error, Error::LegacyMrfUnverified { .. }), case == "qualified", "case={case}");
|
||||
let qualifies = matches!(case, "qualified" | "repaired" | "versioned");
|
||||
assert_eq!(matches!(error, Error::LegacyMrfUnverified { .. }), qualifies, "case={case}");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
if case == "qualified" {
|
||||
if qualifies {
|
||||
assert_eq!(task.get_status().await, HealTaskStatus::Completed);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(outcome.objects[0].detail.as_deref(), Some("updated diagnostic wording"));
|
||||
if case == "qualified" {
|
||||
assert_eq!(outcome.objects[0].detail.as_deref(), Some("updated diagnostic wording"));
|
||||
}
|
||||
let opts = storage.object_heal_opts.lock().expect("storage options");
|
||||
assert_eq!(opts.len(), 1);
|
||||
assert_eq!(opts[0].scan_mode, HealScanMode::Deep);
|
||||
|
||||
@@ -1004,6 +1004,15 @@ async fn partial_write_sigkill_replay_scenario(protected: bool) {
|
||||
.await,
|
||||
"legacy replay attempts must finish"
|
||||
);
|
||||
let stats = manager.get_statistics().await;
|
||||
assert_eq!(stats.failed_tasks, 0, "the first physical legacy repair must not be a failed attempt");
|
||||
assert_eq!(
|
||||
stats.unverified_tasks, 1,
|
||||
"no second Deep is needed to classify the repaired legacy object"
|
||||
);
|
||||
assert_eq!(stats.total_unverified_objects_repaired, 1);
|
||||
assert_eq!(stats.total_unverified_bytes_repaired, 36);
|
||||
assert_eq!(stats.total_objects_healed, 0, "physical repair does not prove legacy identity");
|
||||
// Cover two MRF admission backoff periods so a missed hold notice
|
||||
// cannot pass merely because its task finished between polls.
|
||||
let retry_window = tokio::time::Instant::now() + Duration::from_secs(12);
|
||||
|
||||
@@ -379,8 +379,8 @@ mod tests {
|
||||
let mut stored = generated.stored;
|
||||
assert_eq!(remaining(&stored), RECOVERY_CODE_COUNT as u32);
|
||||
|
||||
for index in 0..3 {
|
||||
consume(&mut stored, &generated.plaintext[index], now());
|
||||
for code in generated.plaintext.iter().take(3) {
|
||||
consume(&mut stored, code, now());
|
||||
}
|
||||
assert_eq!(remaining(&stored), (RECOVERY_CODE_COUNT - 3) as u32);
|
||||
}
|
||||
|
||||
@@ -181,7 +181,7 @@ where
|
||||
);
|
||||
|
||||
let body: BoxBody = Full::new(Bytes::from(error_xml))
|
||||
.map_err(|e| -> BoxError { Box::new(e) })
|
||||
.map_err(|never| match never {})
|
||||
.boxed_unsync();
|
||||
|
||||
let response = Response::builder()
|
||||
|
||||
@@ -1347,7 +1347,7 @@ mod tests {
|
||||
let summary = KmsConfigSummary::from(&config);
|
||||
let response = KmsStatusResponse {
|
||||
status: KmsServiceStatus::Configured,
|
||||
backend_type: Some(config.backend.clone()),
|
||||
backend_type: Some(config.backend),
|
||||
healthy: None,
|
||||
config_summary: Some(summary),
|
||||
};
|
||||
|
||||
@@ -977,7 +977,7 @@ async fn enumerate_target_conflicts(target: &Path, manifest: &BackupManifest) ->
|
||||
(stem.to_string(), format!("target already has key file {entry:?}"))
|
||||
} else if entry == LOCAL_KMS_MASTER_KEY_SALT_FILE {
|
||||
(
|
||||
entry.clone(),
|
||||
entry,
|
||||
"target holds an orphan master-key KDF salt; it would poison every key created after the restore".to_string(),
|
||||
)
|
||||
} else {
|
||||
|
||||
@@ -692,7 +692,7 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
|
||||
let predicted = Event {
|
||||
action: IlmAction::DeleteAction,
|
||||
rule_id: rule.id.clone().unwrap_or_default(),
|
||||
rule_id: rule.id.unwrap_or_default(),
|
||||
due: Some(due),
|
||||
noncurrent_days: 0,
|
||||
newer_noncurrent_versions: 0,
|
||||
|
||||
@@ -487,7 +487,7 @@ mod fast_lock_tests {
|
||||
// Waiter: starts a touch later so the holder wins the lock first, then
|
||||
// must survive the whole hold and acquire promptly after the release.
|
||||
let waiter_mgr = manager.clone();
|
||||
let waiter_key = key.clone();
|
||||
let waiter_key = key;
|
||||
handles.push(tokio::spawn(async move {
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
let request = ObjectLockRequest::new_write(waiter_key, "waiter").with_acquire_timeout(Duration::from_secs(5));
|
||||
|
||||
@@ -1537,7 +1537,7 @@ impl RealtimeMetrics {
|
||||
self.hosts.sort();
|
||||
|
||||
for (k, v) in other.by_disk.into_iter() {
|
||||
self.by_disk.entry(k.to_string()).and_modify(|h| *h = v.clone()).or_insert(v);
|
||||
self.by_disk.entry(k).and_modify(|h| *h = v.clone()).or_insert(v);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,8 +120,8 @@ pub fn collect_erasure_set_metrics(stats: &[ErasureSetStats]) -> Vec<PrometheusM
|
||||
);
|
||||
metrics.push(
|
||||
PrometheusMetric::from_descriptor(&ERASURE_SET_WRITE_HEALTH_MD, stat.write_health as f64)
|
||||
.with_label_owned(POOL_ID_L, pool_id_label.clone())
|
||||
.with_label_owned(SET_ID_L, set_id_label.clone()),
|
||||
.with_label_owned(POOL_ID_L, pool_id_label)
|
||||
.with_label_owned(SET_ID_L, set_id_label),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -102,7 +102,7 @@ fn check() -> Result<(), OpaConfigError> {
|
||||
let prefix = format!("{ENV_PREFIX}{POLICY_PLUGIN_SUB_SYS}").to_uppercase();
|
||||
for (key, value) in env_list {
|
||||
if key.starts_with(&prefix) {
|
||||
candidate.insert(key.to_string(), value);
|
||||
candidate.insert(key, value);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user