mirror of
https://github.com/rustfs/rustfs.git
synced 2026-10-31 16:22:06 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 65f23894fb | |||
| 2c7f154809 | |||
| 933af7c8b9 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=7c705942d3a9e583ebaa85845daa21c4f81fdbf36def24db931aaef38636c53e
|
||||
sha256-linux=5c33dc395233671cb77452f8fb12900ecdbeb368632222b96835d310bce1aab8
|
||||
sha256-darwin=5d4509d69c3106c1b8a6c353d123440d6b4dbe5dd42612e42dcb864946beae6c
|
||||
sha256-linux=414e1dae735e6d14821a992cebc09a545576e9a77e10a5e0ed671ab07b89df98
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=00f8a56b75990b487a594b97c36610b94476a6944ba00473e37187ea654d42bf
|
||||
sha256=e9dc896774122a304e2b8b90f28c82d4d4e0af8ded36a82b147b30945ec994e3
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"plan_id": "rustfs-release-validation",
|
||||
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"targets": [
|
||||
{"os": "linux", "arch": "x86_64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
|
||||
{"os": "macos", "arch": "aarch64", "abi": "none"},
|
||||
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
|
||||
],
|
||||
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
|
||||
"features": ["pyroscope"],
|
||||
"trusted_workflows": [
|
||||
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
|
||||
],
|
||||
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
|
||||
"lanes": [
|
||||
{
|
||||
"lane": "identity",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
|
||||
"timeout_seconds": 1800,
|
||||
"environment_capabilities": ["process-exec"],
|
||||
"cases": ["artifact-identity", "startup"],
|
||||
"applies_to": {
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"platforms": "all"
|
||||
},
|
||||
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
|
||||
},
|
||||
{
|
||||
"lane": "upgrade",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
|
||||
"timeout_seconds": 5400,
|
||||
"environment_capabilities": ["process-exec", "previous-release-artifact"],
|
||||
"cases": [
|
||||
"direct-upgrade",
|
||||
"mixed-version-upgrade",
|
||||
"bucket-config-upgrade",
|
||||
"bucket-config-rollback",
|
||||
"multipart-layout-upgrade",
|
||||
"multipart-layout-baseline"
|
||||
],
|
||||
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
|
||||
"applies_to": {
|
||||
"release_types": ["prerelease", "stable"],
|
||||
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
|
||||
},
|
||||
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.1.zip",
|
||||
"artifact_sha256": "9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5"
|
||||
},
|
||||
{
|
||||
"version": "1.0.0-rc.5",
|
||||
"role": "N-2",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.0-rc.5.zip",
|
||||
"artifact_sha256": "3ee8df71e8edcfada533be452c4135868f697bc515460ae97b027313eade7a3d"
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
+5
-311
@@ -814,18 +814,13 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Determine upload path based on build type. Release and prerelease
|
||||
# builds only stage their bytes here: the official
|
||||
# artifacts/rustfs/release/ path is written solely by the
|
||||
# certification-gated promote-release-artifacts job, from these same
|
||||
# staged objects. Development builds keep the uncertified dev channel
|
||||
# and never touch official or latest channels.
|
||||
# Determine upload path based on build type
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/dev/"
|
||||
echo "📤 Uploading development build to R2 dev directory"
|
||||
else
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
echo "📤 Staging release build for certification-gated promotion"
|
||||
R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
echo "📤 Uploading release build to R2 release directory"
|
||||
fi
|
||||
|
||||
# Upload all rustfs zip files to R2 using S3-compatible API
|
||||
@@ -874,221 +869,6 @@ jobs:
|
||||
echo "This run does not trigger Docker publishing."
|
||||
fi
|
||||
|
||||
# Aggregate the parent candidate batch. This job always collects results and
|
||||
# retains its evidence, including on failure, and every publication job
|
||||
# below requires its success. Missing, stale, substituted or non-PASS
|
||||
# evidence fails closed here: only the trusted verification sources
|
||||
# (rustfs/s3chaos release-gate envelopes, rustfs/backlog#2658) can satisfy
|
||||
# it, so a PR-uploaded JSON set never gains release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: [ build-check, build-rustfs ]
|
||||
if: ${{ !cancelled() && needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') }}
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download all build artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Write candidate and collect trusted validation results
|
||||
env:
|
||||
# Cross-repo read of the trusted verification run. Falls back to the
|
||||
# workflow token, which cannot read rustfs/s3chaos; a missing token
|
||||
# therefore fail-closes instead of certifying.
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
PRODUCT_COMMIT: ${{ github.sha }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
BUILD_TYPE: ${{ needs.build-check.outputs.build_type }}
|
||||
FEATURES: ${{ needs.build-check.outputs.build_type == 'development' && '' || 'pyroscope' }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
mkdir -p results release-gate-output
|
||||
PLAN_SHA256=$(python3 -c "import hashlib,json,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('.config/release-validation-plan.json'))))")
|
||||
|
||||
python3 - "$PLAN_SHA256" > candidate.json <<'PY'
|
||||
import hashlib, json, os, sys, zipfile
|
||||
|
||||
plan_sha256 = sys.argv[1]
|
||||
profile = {"release": "stable", "prerelease": "prerelease", "preview": "preview"}.get(os.environ["BUILD_TYPE"])
|
||||
if profile is None:
|
||||
raise SystemExit("release certification only covers preview, prerelease and stable builds")
|
||||
|
||||
def sha256(path):
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
deliverables = []
|
||||
for name in sorted(os.listdir("artifacts")):
|
||||
if not name.startswith("rustfs-") or not name.endswith(".zip"):
|
||||
continue
|
||||
path = os.path.join("artifacts", name)
|
||||
members = zipfile.ZipFile(path).namelist()
|
||||
binary_name = "rustfs.exe" if "rustfs.exe" in members else "rustfs"
|
||||
with zipfile.ZipFile(path) as archive:
|
||||
binary_bytes = archive.read(binary_name)
|
||||
import tempfile
|
||||
with tempfile.NamedTemporaryFile() as handle:
|
||||
handle.write(binary_bytes)
|
||||
handle.flush()
|
||||
binary_sha = sha256(handle.name)
|
||||
deliverables.append({
|
||||
"kind": "zip",
|
||||
"name": name,
|
||||
"location": "https://github.com/rustfs/rustfs/releases/download/{}/{}".format(os.environ["VERSION"], name),
|
||||
"sha256": sha256(path),
|
||||
"size": os.path.getsize(path),
|
||||
"binary_sha256": binary_sha,
|
||||
})
|
||||
if not deliverables:
|
||||
raise SystemExit("no built deliverables found to certify")
|
||||
candidate = {
|
||||
"schema_version": 1,
|
||||
"product_commit": os.environ["PRODUCT_COMMIT"],
|
||||
"version": os.environ["VERSION"],
|
||||
"release_profile": profile,
|
||||
"features": [f for f in os.environ.get("FEATURES", "").split(",") if f],
|
||||
"plan_sha256": plan_sha256,
|
||||
"validation_batch": {"run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": int(os.environ["GITHUB_RUN_ATTEMPT"])},
|
||||
"deliverables": deliverables,
|
||||
}
|
||||
json.dump(candidate, sys.stdout, indent=2, sort_keys=True)
|
||||
print()
|
||||
PY
|
||||
|
||||
# Trusted verification evidence only: the release-gate envelopes of
|
||||
# rustfs/s3chaos for this exact product commit. Absence blocks below.
|
||||
TRUSTED_REPO="rustfs/s3chaos"
|
||||
TRUSTED_WORKFLOW="release-gate.yml"
|
||||
TRUSTED_RUN=$(gh api "repos/${TRUSTED_REPO}/actions/workflows/${TRUSTED_WORKFLOW}/runs?head_sha=${PRODUCT_COMMIT}&status=success&per_page=1" --jq '.workflow_runs[0].id // empty' || true)
|
||||
if [[ -z "$TRUSTED_RUN" ]]; then
|
||||
echo "❌ No trusted verification run of ${TRUSTED_REPO}/${TRUSTED_WORKFLOW} for ${PRODUCT_COMMIT}; refusing to certify" >&2
|
||||
else
|
||||
gh run download "$TRUSTED_RUN" -R "$TRUSTED_REPO" -n release-validation-evidence -D results || \
|
||||
echo "❌ Trusted verification run ${TRUSTED_RUN} published no release-validation-evidence artifact" >&2
|
||||
fi
|
||||
|
||||
python3 scripts/release/release_gate.py validate \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate candidate.json \
|
||||
--results results \
|
||||
--output release-gate-output
|
||||
|
||||
- name: Upload certification evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: |
|
||||
candidate.json
|
||||
results/
|
||||
release-gate-output/
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
# The only writer of the official R2 release path. Promotion re-checks the
|
||||
# exact bytes about to be published against the passed certification list
|
||||
# and copies the staged objects byte-for-byte: no rebuild-after-verify.
|
||||
promote-release-artifacts:
|
||||
name: Promote Release Artifacts
|
||||
needs: [ build-check, build-rustfs, release-certification ]
|
||||
# Target-channel lock for the official R2 release path.
|
||||
concurrency:
|
||||
group: release-channel-r2-release
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download build and certification artifacts
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./artifacts
|
||||
pattern: rustfs-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged release artifacts
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$name"
|
||||
done
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
STAGING_PATH="s3://${R2_BUCKET}/artifacts/rustfs/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"
|
||||
for zip_file in artifacts/rustfs-*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
name="$(basename "$zip_file")"
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PATH}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$zip_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged object ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
|
||||
# Create GitHub Release for every valid release tag, including previews
|
||||
create-release:
|
||||
name: Create GitHub Release
|
||||
@@ -1248,10 +1028,6 @@ jobs:
|
||||
update-latest-version:
|
||||
name: Update Latest Version
|
||||
needs: [ build-check, publish-release ]
|
||||
# Target-channel lock for the latest.json version pointer.
|
||||
concurrency:
|
||||
group: release-channel-oss-latest
|
||||
cancel-in-progress: false
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
@@ -1292,45 +1068,6 @@ jobs:
|
||||
rm -rf "$OSSUTIL_DIR" "$OSSUTIL_ZIP"
|
||||
chmod +x "$OSSUTIL_BIN"
|
||||
|
||||
# Candidate/pointer ordering: an older candidate must never overwrite
|
||||
# a newer pointer, even if its promotion job finishes later.
|
||||
CURRENT_LATEST="${RUNNER_TEMP}/current-latest.json"
|
||||
if "$OSSUTIL_BIN" cp oss://rustfs-version/latest.json "$CURRENT_LATEST" 2>/dev/null; then
|
||||
python3 - "$CURRENT_LATEST" "$VERSION" <<'PY'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
def key(version):
|
||||
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$", version)
|
||||
if not match:
|
||||
raise SystemExit(3)
|
||||
core = tuple(int(part) for part in match.groups()[:3])
|
||||
prerelease = match.group(4)
|
||||
if prerelease is None:
|
||||
return core + (1,)
|
||||
return core + (0, tuple(int(p) if p.isdigit() else p for p in re.split(r"[.-]", prerelease)))
|
||||
|
||||
try:
|
||||
current = json.load(open(sys.argv[1], encoding="utf-8")).get("version", "")
|
||||
proposed = sys.argv[2]
|
||||
if current and key(current) > key(proposed):
|
||||
raise SystemExit(1)
|
||||
except SystemExit:
|
||||
raise
|
||||
except (OSError, ValueError, TypeError):
|
||||
raise SystemExit(3)
|
||||
PY
|
||||
guard_status=$?
|
||||
if [[ "$guard_status" -eq 1 ]]; then
|
||||
echo "Current latest.json points at a newer version; refusing to downgrade" >&2
|
||||
exit 0
|
||||
elif [[ "$guard_status" -ne 0 ]]; then
|
||||
echo "Existing latest.json is unparsable; refusing to overwrite it blindly" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
# Create latest.json
|
||||
cat > latest.json << EOF
|
||||
{
|
||||
@@ -1347,59 +1084,16 @@ jobs:
|
||||
|
||||
echo "✅ Updated latest.json for ${RELEASE_TYPE} release $VERSION"
|
||||
|
||||
# Publish release (remove draft status). Requires the release-certification
|
||||
# aggregate: without a complete legal success there is no public write.
|
||||
# Publish release (remove draft status)
|
||||
publish-release:
|
||||
name: Publish Release
|
||||
needs: [ build-check, create-release, upload-release-assets, release-certification ]
|
||||
# Target-channel lock: concurrent releases serialize on this channel so an
|
||||
# older candidate cannot interleave its publish with a newer one.
|
||||
concurrency:
|
||||
group: release-channel-github-release
|
||||
cancel-in-progress: false
|
||||
needs: [ build-check, create-release, upload-release-assets ]
|
||||
if: needs.build-check.outputs.should_publish == 'true' && startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify release assets against certification
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
TAG="${{ needs.build-check.outputs.version }}"
|
||||
mkdir -p "$RUNNER_TEMP/promotion-assets"
|
||||
gh release download "$TAG" --dir "$RUNNER_TEMP/promotion-assets" --pattern '*.zip'
|
||||
found=0
|
||||
for zip_file in "$RUNNER_TEMP"/promotion-assets/*.zip; do
|
||||
[[ -f "$zip_file" ]] || continue
|
||||
found=1
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$zip_file" \
|
||||
--artifact-name "$(basename "$zip_file")"
|
||||
done
|
||||
if [[ "$found" -ne 1 ]]; then
|
||||
echo "No release zip assets to verify against the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Publish release
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
|
||||
@@ -96,7 +96,7 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
|
||||
# Readers: test-and-lint (workspace and ILM), build-rustfs-debug-binary,
|
||||
# e2e-tests, e2e-full.
|
||||
warm-ci-dev:
|
||||
name: Warm ci-dev
|
||||
@@ -146,7 +146,7 @@ jobs:
|
||||
if-no-files-found: error
|
||||
|
||||
# --all-targets covers the test binaries nextest builds, including
|
||||
# e2e_test, which the workspace run excludes. The second build adds
|
||||
# e2e_test, which test-and-lint's own run excludes. The second build adds
|
||||
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
|
||||
# and that no lint lane enables.
|
||||
- name: Build ci-dev superset
|
||||
|
||||
+34
-36
@@ -113,7 +113,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=2$' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^rehearse_shards=' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -160,7 +160,7 @@ jobs:
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
workspace-lint-and-build:
|
||||
test-and-lint:
|
||||
name: Workspace Lint and Build
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
@@ -193,13 +193,13 @@ jobs:
|
||||
|
||||
- name: Prepare test evidence
|
||||
run: |
|
||||
mkdir -p artifacts/workspace
|
||||
mkdir -p artifacts/test-and-lint
|
||||
{
|
||||
echo "run_id=${GITHUB_RUN_ID}"
|
||||
echo "job=${GITHUB_JOB}"
|
||||
echo "runner=${RUNNER_NAME}"
|
||||
echo "started_at=$(date --utc --iso-8601=seconds)"
|
||||
} > artifacts/workspace/run-metadata.txt
|
||||
} > artifacts/test-and-lint/run-metadata.txt
|
||||
|
||||
# Clippy gates every event, pull requests included. When it ran only after
|
||||
# merge, a lint regression merged freely and then failed every main run
|
||||
@@ -233,11 +233,11 @@ jobs:
|
||||
|
||||
- name: Run documentation tests
|
||||
run: |
|
||||
mkdir -p artifacts/workspace
|
||||
mkdir -p artifacts/test-and-lint
|
||||
set +e
|
||||
timeout --verbose --signal=TERM --kill-after=30s 15m \
|
||||
python3 scripts/ci_test_artifact.py cargo doc --configuration workspace \
|
||||
2>&1 | tee artifacts/workspace/doctest.log
|
||||
2>&1 | tee artifacts/test-and-lint/doctest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=cargo test --all --exclude e2e_test --doc"
|
||||
@@ -246,7 +246,7 @@ jobs:
|
||||
echo
|
||||
echo "Remaining test-related processes:"
|
||||
pgrep -af 'cargo|rustdoc|target/.*/deps/' || true
|
||||
} > artifacts/workspace/doctest-diagnostics.txt
|
||||
} > artifacts/test-and-lint/doctest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Upload test reports and diagnostics
|
||||
@@ -255,7 +255,7 @@ jobs:
|
||||
with:
|
||||
name: junit-test-and-lint-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
artifacts/workspace
|
||||
artifacts/test-and-lint
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
@@ -268,7 +268,7 @@ jobs:
|
||||
|
||||
workspace-tests:
|
||||
name: Workspace tests
|
||||
needs: [ workspace-lint-and-build, classify-changes ]
|
||||
needs: [ test-and-lint, classify-changes ]
|
||||
# The archive retains compile-time fixture paths and the large test set
|
||||
# requires the same runner image until the hosted-runtime pilot passes.
|
||||
runs-on: sm-standard-4
|
||||
@@ -293,14 +293,14 @@ jobs:
|
||||
|
||||
- name: Extract and list workspace tests
|
||||
run: |
|
||||
mkdir -p artifacts/workspace
|
||||
mkdir -p artifacts/test-and-lint
|
||||
cargo nextest list --profile ci --archive-file "$RUNNER_TEMP/workspace-tests.tar.zst" --workspace-remap "$GITHUB_WORKSPACE" --extract-to "$GITHUB_WORKSPACE" --extract-overwrite --message-format json \
|
||||
> artifacts/workspace/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/workspace/core-test-listing.json
|
||||
> artifacts/test-and-lint/core-test-listing.json
|
||||
python3 scripts/check_test_wiring.py --check-core artifacts/test-and-lint/core-test-listing.json
|
||||
|
||||
- name: Run nextest tests
|
||||
run: |
|
||||
mkdir -p artifacts/workspace
|
||||
mkdir -p artifacts/test-and-lint
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
@@ -309,7 +309,7 @@ jobs:
|
||||
cargo nextest run --profile ci --cargo-metadata target/nextest/cargo-metadata.json --binaries-metadata target/nextest/binaries-metadata.json \
|
||||
--workspace-remap "$GITHUB_WORKSPACE" --target-dir-remap "$GITHUB_WORKSPACE/target" \
|
||||
--status-level all --final-status-level all \
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
status=${PIPESTATUS[0]}
|
||||
{
|
||||
echo "command=workspace archive execution"
|
||||
@@ -321,13 +321,13 @@ jobs:
|
||||
echo
|
||||
echo "Kernel OOM / kill events:"
|
||||
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
|
||||
} > artifacts/workspace/nextest-diagnostics.txt
|
||||
} > artifacts/test-and-lint/nextest-diagnostics.txt
|
||||
exit "${status}"
|
||||
|
||||
- name: Verify workspace execution completeness
|
||||
run: |
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/workspace/core-test-listing.json target/nextest/ci/junit.xml
|
||||
python3 scripts/ci_test_artifact.py evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
./scripts/check_migration_gate_count.sh evidence artifacts/test-and-lint/core-test-listing.json target/nextest/ci/junit.xml
|
||||
|
||||
- name: Run ignored ILM integration tests serially
|
||||
if: needs.classify-changes.outputs.mode == 'full'
|
||||
@@ -358,7 +358,7 @@ jobs:
|
||||
name: workspace-test-evidence-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/workspace
|
||||
artifacts/test-and-lint
|
||||
artifacts/ilm-integration
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
@@ -368,7 +368,7 @@ jobs:
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
run: |
|
||||
python3 scripts/ci_test_shards.py prepare \
|
||||
--listing artifacts/workspace/core-test-listing.json \
|
||||
--listing artifacts/test-and-lint/core-test-listing.json \
|
||||
--junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
|
||||
- name: Upload same-run shard plan
|
||||
if: needs.classify-changes.outputs.rehearse_shards == 'true'
|
||||
@@ -449,7 +449,7 @@ jobs:
|
||||
|
||||
connect-feature-boundaries:
|
||||
name: Connect Feature Boundaries
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect != 'false' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -499,13 +499,11 @@ jobs:
|
||||
|
||||
test-and-lint-rio-v2:
|
||||
name: Test and Lint (rio-v2)
|
||||
# Specialized boundary, hardware and alternate-feature lanes run only on
|
||||
# the nightly schedule and manual dispatch. They do not gate a merge: the
|
||||
# nightly run reports a regression within a day, and repeating these builds
|
||||
# on every PR revision and merge group bought no earlier signal worth the
|
||||
# runner occupancy. Workspace lint and tests, the core contract, the debug
|
||||
# build, E2E smoke and the S3 lanes remain the pull-request gate.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
# Alternate feature configurations run on the nightly schedule, merge
|
||||
# groups and manual dispatch. The default workspace remains the PR gate and
|
||||
# the per-push main baseline: repeating four feature builds on every merge
|
||||
# bought nothing the nightly run does not report within a day.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -542,12 +540,12 @@ jobs:
|
||||
# --profile ci so the quarantine list (and its junit flaky markers)
|
||||
# covers this leg too; the default profile is the local no-retry
|
||||
# profile and silently ignored quarantined flakes here (rustfs#6703).
|
||||
mkdir -p artifacts/workspace
|
||||
mkdir -p artifacts/test-and-lint
|
||||
rm -f target/nextest/ci/junit.xml
|
||||
./scripts/ci/resource_sampler.sh start nextest-rio-v2
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
cargo nextest run --profile ci -p rustfs -p rustfs-ecstore --features rio-v2 \
|
||||
2>&1 | tee artifacts/workspace/nextest.log
|
||||
2>&1 | tee artifacts/test-and-lint/nextest.log
|
||||
cargo test -p rustfs --doc --features rio-v2
|
||||
|
||||
- name: Upload rio-v2 test reports and diagnostics
|
||||
@@ -557,14 +555,14 @@ jobs:
|
||||
name: junit-test-and-lint-rio-v2-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/workspace
|
||||
artifacts/test-and-lint
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
test-and-lint-protocols:
|
||||
name: "Test and Lint (${{ matrix.features.name }})"
|
||||
# Same cadence as test-and-lint-rio-v2 above.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["merge_group", "schedule", "workflow_dispatch"]'), github.event_name)
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
@@ -633,7 +631,7 @@ jobs:
|
||||
path: |
|
||||
target/nextest/ci/junit.xml
|
||||
artifacts/protocol-tests
|
||||
artifacts/workspace
|
||||
artifacts/test-and-lint
|
||||
retention-days: 3
|
||||
if-no-files-found: warn
|
||||
|
||||
@@ -785,7 +783,7 @@ jobs:
|
||||
# job had neither, so each closed/merged PR really ran the whole io_uring
|
||||
# suite (measured 4m17s / 7m19s / 7m31s on runs 30678272341 / 30678117601 /
|
||||
# 30662728539) and kept the cancellation run in progress for minutes.
|
||||
if: needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('["schedule", "workflow_dispatch"]'), github.event_name)
|
||||
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# GitHub-hosted ubuntu-latest runs a recent kernel with io_uring and, unlike
|
||||
# a container, applies no seccomp filter that would block io_uring_setup — so
|
||||
@@ -980,7 +978,7 @@ jobs:
|
||||
path: |
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-logs/
|
||||
${{ runner.temp }}/rustfs-e2e-smoke-list.json
|
||||
artifacts/workspace
|
||||
artifacts/test-and-lint
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload e2e smoke JUnit report
|
||||
@@ -1391,7 +1389,7 @@ jobs:
|
||||
- classify-changes
|
||||
- typos
|
||||
- quick-checks
|
||||
- workspace-lint-and-build
|
||||
- test-and-lint
|
||||
- workspace-tests
|
||||
- workspace-shards
|
||||
- workspace-shard-evidence
|
||||
@@ -1427,7 +1425,7 @@ jobs:
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
- workspace-lint-and-build
|
||||
- test-and-lint
|
||||
- workspace-tests
|
||||
- core-contract
|
||||
- test-and-lint-rio-v2
|
||||
|
||||
@@ -329,75 +329,13 @@ jobs:
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Consume the trusted release certification for the exact version this
|
||||
# workflow wraps and verify the certified input digests. A manual dispatch
|
||||
# without that certification fails here and pushes no official tag; only
|
||||
# results from the trusted verification sources carry release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$run_id" ]]; then
|
||||
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$run_id" -n release-certification -D certification; then
|
||||
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Input digest binding: every zip this image wraps must be the
|
||||
# certified byte-for-byte content, never a mutable latest pointer.
|
||||
while IFS=$'\t' read -r name location expected; do
|
||||
echo "Verifying certified input ${name}"
|
||||
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "Certified input ${name} does not match its recorded digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
# Supports both release and dev channel binaries based on build context
|
||||
# Only runs when should_build is true (which includes workflow success check)
|
||||
build-docker:
|
||||
name: Build Docker Images
|
||||
needs: [ build-check, release-certification ]
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
@@ -505,14 +443,6 @@ jobs:
|
||||
# Output tags
|
||||
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Staging identity for this run's promotion: official tags are only
|
||||
# created after the staged image verifies against the certification,
|
||||
# and they point at this same digest (no rebuild-after-verify).
|
||||
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
|
||||
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
|
||||
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
|
||||
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Generate labels
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
@@ -537,9 +467,7 @@ jobs:
|
||||
file: ${{ matrix.file }}
|
||||
platforms: ${{ env.DOCKER_PLATFORMS }}
|
||||
push: ${{ needs.build-check.outputs.should_push == 'true' }}
|
||||
# Push to staging tags only: official tags are created by the
|
||||
# certification-gated promotion step below from this same digest.
|
||||
tags: ${{ steps.meta.outputs.staging_tags }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# No layer cache. This build compiles nothing — it downloads a
|
||||
# release zip and runs apk/apt — so the cache could only save the
|
||||
@@ -565,114 +493,6 @@ jobs:
|
||||
no-cache: false
|
||||
pull: true
|
||||
|
||||
# A later-built wrapper: the staged image must complete this platform's
|
||||
# own binary identity verification before any official tag is written.
|
||||
# It may not inherit an unverified package's conclusion.
|
||||
- name: Verify staged image against certification
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
case "${{ matrix.variant }}" in
|
||||
musl) libc="musl" ;;
|
||||
glibc) libc="gnu" ;;
|
||||
esac
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
case "$platform" in
|
||||
linux/amd64) arch_sub="x86_64" ;;
|
||||
linux/arm64) arch_sub="aarch64" ;;
|
||||
esac
|
||||
docker pull --platform "$platform" "$STAGING_REF"
|
||||
container="$(docker create --platform "$platform" "$STAGING_REF")"
|
||||
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
|
||||
docker rm "$container" >/dev/null
|
||||
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
|
||||
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
|
||||
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/staged-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Promote staged image to official tags
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
|
||||
for tag in "${official_tags[@]}"; do
|
||||
docker buildx imagetools create -t "$tag" "$STAGING_REF"
|
||||
echo "Promoted $tag -> $STAGING_REF"
|
||||
done
|
||||
|
||||
# Downstream consumers (Helm) verify this record instead of inheriting
|
||||
# an unverified wrapper's conclusion.
|
||||
- name: Record image promotion
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
jq -n \
|
||||
--arg version "$VERSION" \
|
||||
--arg digest "$digest" \
|
||||
--arg staging_ref "$STAGING_REF" \
|
||||
--arg candidate_sha256 "$candidate_sha" \
|
||||
--arg tags "${{ steps.meta.outputs.tags }}" \
|
||||
--slurpfile candidate certification/candidate.json \
|
||||
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
|
||||
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
|
||||
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
|
||||
> image-promotion-record.json
|
||||
|
||||
- name: Upload image promotion record
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: image-promotion-record${{ matrix.suffix }}
|
||||
path: image-promotion-record.json
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
# Note: Manifest creation is no longer needed as we only build one variant
|
||||
# Multi-arch manifests are automatically created by docker/build-push-action
|
||||
|
||||
|
||||
@@ -30,102 +30,9 @@ permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
# Consume the same trusted release certification as every other publication
|
||||
# exit and verify the image promotion record of the Docker workflow. The
|
||||
# chart wraps no binary of its own, so it may only reference an image whose
|
||||
# platform verification already passed; it must not inherit an unverified
|
||||
# package's conclusion. A run without that evidence fails closed here and
|
||||
# never reaches the chart repository.
|
||||
release-certification:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.raw_tag }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Normalize release version
|
||||
id: version
|
||||
env:
|
||||
RAW_INPUT: ${{ github.event.inputs.version }}
|
||||
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
RAW="$RAW_INPUT"
|
||||
else
|
||||
RAW="$RAW_BRANCH"
|
||||
fi
|
||||
case "$RAW" in
|
||||
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
|
||||
*) RAW_TAG="$RAW" ;;
|
||||
esac
|
||||
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download and verify release certification and image record
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification image-record
|
||||
|
||||
build_run="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$build_run" ]]; then
|
||||
build_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$build_run" ]]; then
|
||||
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$build_run" -n release-certification -D certification; then
|
||||
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
docker_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
|
||||
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
|
||||
exit 1
|
||||
fi
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
|
||||
if [[ "$record_sha" != "$candidate_sha" ]]; then
|
||||
echo "Image promotion record does not belong to this certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
|
||||
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
|
||||
image-record/image-promotion-record.json)"
|
||||
if [[ "$overlap" -ne 0 ]]; then
|
||||
echo "Image promotion record carries binary digests outside the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build-helm-package:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ release-certification ]
|
||||
if: |
|
||||
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
|
||||
(
|
||||
|
||||
+14
-210
@@ -225,57 +225,10 @@ jobs:
|
||||
echo " Build run ID: $BUILD_RUN_ID"
|
||||
echo " Build run number: $RUN_NUMBER"
|
||||
|
||||
# Consume the trusted release certification of the resolved build run and
|
||||
# verify the certified input digests before any package is wrapped. A run
|
||||
# without that certification fails closed here; packages built later must
|
||||
# complete their own verification and cannot inherit an unverified
|
||||
# conclusion.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: resolve
|
||||
if: needs.resolve.result == 'success'
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
|
||||
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
# Development artifacts are explicitly uncertified and never touch
|
||||
# official or latest channels; only the staging/dev paths remain.
|
||||
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
|
||||
> certification/development-uncertified.json
|
||||
exit 0
|
||||
fi
|
||||
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
|
||||
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Build DEB and RPM packages for each architecture and libc variant
|
||||
package:
|
||||
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
|
||||
needs: [ resolve, release-certification ]
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
@@ -541,55 +494,6 @@ jobs:
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
- name: Download certification evidence
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
# Later-built wrapper verification: the binary inside each package must
|
||||
# match the certified binary digest for this platform before anything is
|
||||
# uploaded, and the package itself stays on staging until promotion.
|
||||
- name: Verify wrapped binary against certification
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
shell: bash
|
||||
env:
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
ARTIFACT_NAME: ${{ matrix.artifact_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" ]]; then
|
||||
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y cpio
|
||||
fi
|
||||
for package_file in "$DEB_FILE" "$RPM_FILE"; do
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Upload packages to artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
@@ -627,14 +531,10 @@ jobs:
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Build-phase writes stay on staging for release types: the official
|
||||
# packages/release and packages/latest paths are written only by the
|
||||
# certification-gated promote-packages job, from these same staged
|
||||
# objects. Development packages keep the uncertified dev channel.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PREFIX="artifacts/rustfs/packages/dev"
|
||||
else
|
||||
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
|
||||
R2_PREFIX="artifacts/rustfs/packages/release"
|
||||
fi
|
||||
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
|
||||
|
||||
@@ -649,6 +549,18 @@ jobs:
|
||||
|
||||
echo "✅ Upload complete"
|
||||
|
||||
# Also upload as latest for release/prerelease
|
||||
if [[ "$BUILD_TYPE" == "release" || "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading latest: $(basename "$f")"
|
||||
aws s3 cp "$f" "$LATEST_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
echo "✅ Latest packages updated"
|
||||
fi
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
@@ -702,114 +614,6 @@ jobs:
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# The only writer of the official packages/release and packages/latest R2
|
||||
# paths. Promotion re-checks each package's wrapped binary against the
|
||||
# passed certification list and copies the staged objects byte-for-byte:
|
||||
# no rebuild-after-verify, no mutable latest resolution downstream.
|
||||
promote-packages:
|
||||
name: Promote Packages
|
||||
needs: [ resolve, package, release-certification ]
|
||||
# Target-channel lock for the official package paths.
|
||||
concurrency:
|
||||
group: release-channel-r2-packages
|
||||
cancel-in-progress: false
|
||||
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download packages and certification
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./packages
|
||||
pattern: packages-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged packages
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y cpio
|
||||
fi
|
||||
|
||||
promoted=0
|
||||
for package_file in packages/*.deb packages/*.rpm; do
|
||||
[[ -f "$package_file" ]] || continue
|
||||
name="$(basename "$package_file")"
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged package ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
promoted=$((promoted + 1))
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
if [[ "$promoted" -eq 0 ]]; then
|
||||
echo "No staged packages found to promote" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
|
||||
@@ -16,193 +16,8 @@
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::http::HttpResponse;
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
|
||||
fn assert_not_found<E: ProvideErrorMetadata + std::fmt::Debug>(error: &SdkError<E, HttpResponse>) {
|
||||
assert_eq!(error.raw_response().map(|r| r.status().as_u16()), Some(404), "{error:?}");
|
||||
assert_eq!(
|
||||
error.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("NoSuchKey"),
|
||||
"{error:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_writes_cannot_replace_delete_marker_with_if_match() {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await.expect("create test environment");
|
||||
env.start_rustfs_server(vec![]).await.expect("start RustFS");
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "conditional-delete-marker-proof";
|
||||
create_versioned_bucket(&client, bucket).await;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("source")
|
||||
.body(ByteStream::from_static(b"source"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write copy source");
|
||||
|
||||
for operation in ["put", "copy", "multipart"] {
|
||||
let old = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.body(ByteStream::from_static(b"old"))
|
||||
.send()
|
||||
.await
|
||||
.expect("write historical version");
|
||||
let old_etag = old.e_tag().expect("historical ETag");
|
||||
let marker = client
|
||||
.delete_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("create delete marker");
|
||||
let marker_id = marker.version_id().expect("delete marker version");
|
||||
|
||||
let (upload_id, completed) = if operation == "multipart" {
|
||||
let upload = client
|
||||
.create_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("initiate multipart upload");
|
||||
let upload_id = upload.upload_id().expect("upload ID").to_owned();
|
||||
let part = client
|
||||
.upload_part()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(1)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("upload part");
|
||||
let completed = CompletedMultipartUpload::builder()
|
||||
.parts(
|
||||
CompletedPart::builder()
|
||||
.part_number(1)
|
||||
.set_e_tag(part.e_tag().map(str::to_owned))
|
||||
.build(),
|
||||
)
|
||||
.build();
|
||||
(Some(upload_id), Some(completed))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
for etag in [old_etag, "\"00000000000000000000000000000000\""] {
|
||||
match operation {
|
||||
"put" => {
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_match(etag)
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
"copy" => {
|
||||
let err = client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match copy must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
_ => {
|
||||
let err = client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.clone().expect("completed parts"))
|
||||
.if_match(etag)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("If-Match completion must reject marker");
|
||||
assert_not_found(&err);
|
||||
}
|
||||
}
|
||||
let versions = client
|
||||
.list_object_versions()
|
||||
.bucket(bucket)
|
||||
.prefix(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("list versions after rejected write");
|
||||
assert_eq!(versions.versions().len(), 1, "{operation}: failed write created a version");
|
||||
assert_eq!(versions.delete_markers().len(), 1);
|
||||
assert_eq!(versions.delete_markers()[0].version_id(), Some(marker_id));
|
||||
assert_eq!(versions.delete_markers()[0].is_latest(), Some(true));
|
||||
}
|
||||
|
||||
match operation {
|
||||
"put" => {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.if_none_match("*")
|
||||
.body(ByteStream::from_static(b"new"))
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match PUT over marker");
|
||||
}
|
||||
"copy" => {
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.copy_source(format!("{bucket}/source"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match copy over marker");
|
||||
}
|
||||
_ => {
|
||||
client
|
||||
.complete_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.upload_id(upload_id.as_deref().expect("upload ID"))
|
||||
.multipart_upload(completed.expect("completed parts"))
|
||||
.if_none_match("*")
|
||||
.send()
|
||||
.await
|
||||
.expect("If-None-Match completion over marker");
|
||||
}
|
||||
}
|
||||
let current = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key(operation)
|
||||
.send()
|
||||
.await
|
||||
.expect("read successful conditional write")
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.expect("read object body")
|
||||
.into_bytes();
|
||||
let expected: &[u8] = if operation == "copy" { b"source" } else { b"new" };
|
||||
assert_eq!(current.as_ref(), expected, "{operation}: unexpected current object");
|
||||
}
|
||||
}
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
|
||||
async fn create_versioned_bucket(client: &Client, bucket: &str) {
|
||||
client
|
||||
|
||||
@@ -2146,4 +2146,3 @@ mod tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
// event-drill: isolated harness-only change
|
||||
|
||||
@@ -683,6 +683,7 @@ pub(crate) async fn update_validated_in<F>(
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
@@ -690,7 +691,7 @@ where
|
||||
{
|
||||
Box::pin(async {
|
||||
let sys = require_bucket_metadata_sys_in(ctx)?;
|
||||
let guard = acquire_config_write_guard(sys.clone(), bucket).await?;
|
||||
let guard = acquire_config_write_guard_for_incarnation(sys.clone(), bucket, expected_incarnation_id).await?;
|
||||
validate()?;
|
||||
update_under_config_write_guard(sys, &guard, config_file, data, None).await
|
||||
})
|
||||
@@ -1901,15 +1902,6 @@ impl BucketMetadataSys {
|
||||
self.api.upgrade()
|
||||
}
|
||||
|
||||
/// Whether the backing `ECStore` has been dropped.
|
||||
///
|
||||
/// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
|
||||
/// `ECStore::drop` so a shared bootstrap context does not retain a
|
||||
/// `BucketMetadataSys` whose weak handle is dangling.
|
||||
pub(crate) fn is_store_stale(&self) -> bool {
|
||||
self.api.upgrade().is_none()
|
||||
}
|
||||
|
||||
fn metadata_publish_lock(&self, bucket: &str) -> Arc<Mutex<MetadataPublishLockState>> {
|
||||
Self::bucket_lock_in(&self.metadata_publish_locks, bucket)
|
||||
}
|
||||
@@ -5209,6 +5201,61 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validated_config_update_rejects_a_different_bucket_incarnation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = "catalog-cleanup-incarnation";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let incarnation = store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("read bucket incarnation");
|
||||
let called = std::sync::atomic::AtomicBool::new(false);
|
||||
let result = store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
Uuid::new_v4(),
|
||||
|| {
|
||||
called.store(true, std::sync::atomic::Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(result, Err(Error::BucketNotFound(name)) if name == bucket));
|
||||
assert!(
|
||||
!called.load(std::sync::atomic::Ordering::SeqCst),
|
||||
"incarnation must be checked before the external fence"
|
||||
);
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read unchanged bucket metadata");
|
||||
assert!(
|
||||
metadata.quota_config_json.is_empty(),
|
||||
"a stale cleanup must not modify the replacement bucket"
|
||||
);
|
||||
store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
"quota.json",
|
||||
br#"{"quota":1}"#.to_vec(),
|
||||
incarnation,
|
||||
|| Ok(()),
|
||||
)
|
||||
.await
|
||||
.expect("the current incarnation permits the validated update");
|
||||
let (metadata, _) = get_config_from_disk_with_presence_in(&store.ctx, bucket)
|
||||
.await
|
||||
.expect("read updated bucket metadata");
|
||||
assert_eq!(metadata.quota_config_json, br#"{"quota":1}"#);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_admission_rejects_stale_cached_no_quota() {
|
||||
|
||||
@@ -941,32 +941,6 @@ impl InstanceContext {
|
||||
.clone()
|
||||
}
|
||||
|
||||
/// Remove the `BucketMetadataSys` when its backing `ECStore` is gone.
|
||||
///
|
||||
/// Called from `ECStore::drop`. When test fixtures share the bootstrap
|
||||
/// context, the `BucketMetadataSys` stores a `Weak<ECStore>` to prevent a
|
||||
/// reference cycle; once the store drops the weak is dangling and any
|
||||
/// `object_store()` call panics. Clearing the slot here lets the next
|
||||
/// fixture install a fresh `BucketMetadataSys` without hitting the stale one.
|
||||
pub fn clear_bucket_metadata_sys_if_stale(&self) {
|
||||
let mut guard = self.bucket_metadata_sys.lock().expect("bucket_metadata_sys lock poisoned");
|
||||
if let Some(sys) = guard.as_ref() {
|
||||
// `read()` may block briefly; the async RwLock is acceptable here
|
||||
// because Drop is sync and the lock is held only long enough to
|
||||
// check the weak reference.
|
||||
let is_stale = {
|
||||
let s = sys.try_read();
|
||||
match s {
|
||||
Ok(s) => s.is_store_stale(),
|
||||
Err(_) => true, // contended: treat as stale-safe to clear
|
||||
}
|
||||
};
|
||||
if is_stale {
|
||||
*guard = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Set this instance's background-services cancellation token (once).
|
||||
pub fn init_background_cancel_token(&self, token: CancellationToken) -> Result<(), CancellationToken> {
|
||||
self.background_cancel_token.set(token)
|
||||
|
||||
@@ -6737,7 +6737,9 @@ impl SetDisks {
|
||||
object,
|
||||
);
|
||||
}
|
||||
if should_prevent_write(&oi, http_preconditions.if_none_match_value(), http_preconditions.if_match_value())
|
||||
let if_none_match = http_preconditions.if_none_match_value().map(str::to_owned);
|
||||
let if_match = http_preconditions.if_match_value().map(str::to_owned);
|
||||
if should_prevent_write(&oi, if_none_match, if_match)
|
||||
&& !crate::data_movement::can_replace_stale_data_movement_target(&oi, opts)
|
||||
{
|
||||
return Some(StorageError::PreconditionFailed);
|
||||
|
||||
@@ -161,7 +161,7 @@ impl SetDisks {
|
||||
)
|
||||
});
|
||||
let mut metadata = FileMeta::new();
|
||||
for parts in versions.into_values() {
|
||||
for (version_id, parts) in versions {
|
||||
let errors = parts
|
||||
.iter()
|
||||
.zip(&missing)
|
||||
@@ -173,15 +173,25 @@ impl SetDisks {
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let read_quorum = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((quorum, _)) => usize::try_from(quorum).map_err(|_| DiskError::FileCorrupt)?,
|
||||
let (read_quorum, write_quorum) = match Self::object_quorum_from_meta(&parts, &errors, default_parity) {
|
||||
Ok((read, write)) => (
|
||||
usize::try_from(read).map_err(|_| DiskError::FileCorrupt)?,
|
||||
usize::try_from(write).map_err(|_| DiskError::FileCorrupt)?,
|
||||
),
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) if complete => continue,
|
||||
Err(DiskError::FileNotFound | DiskError::FileVersionNotFound) => return Err(DiskError::ErasureReadQuorum),
|
||||
Err(DiskError::ErasureReadQuorum) if complete => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
let selected = Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?;
|
||||
let selected = if crate::bucket::utils::is_meta_bucketname(bucket) && version_id.is_none() {
|
||||
// A null-version system record must carry the same commit proof
|
||||
// as GET. Listing only needs the selected metadata, so no disk
|
||||
// handles are retained from the shared read selector.
|
||||
Self::select_committed_fileinfo(&vec![None; disk_count], &parts, &errors, "", read_quorum, write_quorum)?.1
|
||||
} else {
|
||||
let mod_time = Self::common_time(&Self::list_object_modtimes(&parts, &errors), read_quorum);
|
||||
Self::pick_valid_fileinfo(&parts, mod_time, None, read_quorum)?
|
||||
};
|
||||
metadata.add_version(selected)?;
|
||||
}
|
||||
if metadata.versions.is_empty() {
|
||||
|
||||
@@ -120,6 +120,7 @@ use bytesize::ByteSize;
|
||||
use futures::future::join_all;
|
||||
use http::HeaderMap;
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
use regex::Regex;
|
||||
use rustfs_config::MI_B;
|
||||
use rustfs_filemeta::metadata_keys;
|
||||
use rustfs_filemeta::{
|
||||
@@ -8035,29 +8036,16 @@ fn parts_after_marker(part_numbers: &[usize], part_number_marker: usize) -> Opti
|
||||
Some(&part_numbers[index..])
|
||||
}
|
||||
|
||||
fn canonicalized_etag(etag: &str) -> Cow<'_, str> {
|
||||
// Preserve the legacy suffix-matching behavior for embedded quotes.
|
||||
let trimmed = etag.trim_end_matches('"');
|
||||
let Some(last_quote) = trimmed.rfind('"') else {
|
||||
return Cow::Borrowed(trimmed);
|
||||
};
|
||||
let prefix = trimmed[..last_quote].trim_end_matches('"');
|
||||
let suffix = &trimmed[last_quote + 1..];
|
||||
if prefix.is_empty() {
|
||||
Cow::Borrowed(suffix)
|
||||
} else {
|
||||
let mut result = String::with_capacity(prefix.len() + suffix.len());
|
||||
result.push_str(prefix);
|
||||
result.push_str(suffix);
|
||||
Cow::Owned(result)
|
||||
}
|
||||
pub fn canonicalize_etag(etag: &str) -> String {
|
||||
let re = Regex::new("\"*?([^\"]*?)\"*?$").unwrap();
|
||||
re.replace_all(etag, "$1").to_string()
|
||||
}
|
||||
|
||||
pub fn e_tag_matches(etag: &str, condition: &str) -> bool {
|
||||
if condition.trim() == "*" {
|
||||
return true;
|
||||
}
|
||||
canonicalized_etag(etag) == canonicalized_etag(condition)
|
||||
canonicalize_etag(etag) == canonicalize_etag(condition)
|
||||
}
|
||||
|
||||
fn weak_etag_matches(etag: &str, condition: &str) -> bool {
|
||||
@@ -8072,12 +8060,15 @@ fn weak_etag_matches(etag: &str, condition: &str) -> bool {
|
||||
.or_else(|| trimmed.strip_prefix("w/"))
|
||||
.unwrap_or(trimmed)
|
||||
}
|
||||
canonicalized_etag(strip_weak(etag)) == canonicalized_etag(strip_weak(condition))
|
||||
canonicalize_etag(strip_weak(etag)) == canonicalize_etag(strip_weak(condition))
|
||||
}
|
||||
|
||||
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<&str>, if_match: Option<&str>) -> bool {
|
||||
let if_none_match = if_none_match.map(str::trim).filter(|condition| !condition.is_empty());
|
||||
let if_match = if_match.map(str::trim).filter(|condition| !condition.is_empty());
|
||||
pub fn should_prevent_write(oi: &ObjectInfo, if_none_match: Option<String>, if_match: Option<String>) -> bool {
|
||||
let if_none_match = if_none_match
|
||||
.as_deref()
|
||||
.map(str::trim)
|
||||
.filter(|condition| !condition.is_empty());
|
||||
let if_match = if_match.as_deref().map(str::trim).filter(|condition| !condition.is_empty());
|
||||
|
||||
match &oi.etag {
|
||||
Some(etag) => {
|
||||
@@ -13781,22 +13772,6 @@ mod tests {
|
||||
assert!(result2.iter().all(|d| d.is_none()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn canonicalized_etag_preserves_legacy_regex_semantics() {
|
||||
let regex = regex::Regex::new("\"*?([^\"]*?)\"*?$").expect("valid legacy ETag pattern");
|
||||
let alphabet = ['a', '"', '\n', 'é', '\0'];
|
||||
for length in 0..=6_u32 {
|
||||
for mut number in 0..alphabet.len().pow(length) {
|
||||
let mut input = String::new();
|
||||
for _ in 0..length {
|
||||
input.push(alphabet[number % alphabet.len()]);
|
||||
number /= alphabet.len();
|
||||
}
|
||||
assert_eq!(canonicalized_etag(&input), regex.replace_all(&input, "$1"), "input: {input:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_etag_matches() {
|
||||
assert!(e_tag_matches("abc", "abc"));
|
||||
@@ -14372,61 +14347,61 @@ mod tests {
|
||||
etag: Some("abc".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let if_none_match = Some("abc");
|
||||
let if_none_match = Some("abc".to_string());
|
||||
let if_match = None;
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("*");
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_match = None;
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = Some("def");
|
||||
let if_match = Some("def".to_string());
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = Some("*");
|
||||
let if_match = Some("*".to_string());
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def");
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_match = None;
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def");
|
||||
let if_match = Some("*");
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_match = Some("*".to_string());
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("def");
|
||||
let if_match = Some("\"abc\"");
|
||||
let if_none_match = Some("def".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("*");
|
||||
let if_match = Some("\"abc\"");
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let oi = ObjectInfo {
|
||||
etag: None,
|
||||
..Default::default()
|
||||
};
|
||||
let if_none_match = Some("*");
|
||||
let if_match = Some("\"abc\"");
|
||||
let if_none_match = Some("*".to_string());
|
||||
let if_match = Some("\"abc\"".to_string());
|
||||
assert!(should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = None;
|
||||
let if_match = None;
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let if_none_match = Some("");
|
||||
let if_match = Some(" ");
|
||||
let if_none_match = Some(String::new());
|
||||
let if_match = Some(" ".to_string());
|
||||
assert!(!should_prevent_write(&oi, if_none_match, if_match));
|
||||
|
||||
let oi = ObjectInfo {
|
||||
etag: Some("abc".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(should_prevent_write(&oi, Some("W/\"abc\""), None));
|
||||
assert!(!should_prevent_write(&oi, Some("W/\"def\""), None));
|
||||
assert!(should_prevent_write(&oi, None, Some("W/\"abc\"")));
|
||||
assert!(should_prevent_write(&oi, Some("W/\"abc\"".to_string()), None));
|
||||
assert!(!should_prevent_write(&oi, Some("W/\"def\"".to_string()), None));
|
||||
assert!(should_prevent_write(&oi, None, Some("W/\"abc\"".to_string())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -977,7 +977,7 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
|
||||
if !root_prefix.is_empty() {
|
||||
if !root_prefix.is_empty() && root_prefix != DATA_MOVEMENT_MULTIPART_PREFIX {
|
||||
let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
@@ -1002,6 +1002,13 @@ async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_pref
|
||||
.into_iter()
|
||||
.map(|sha_dir| sha_dir.trim_end_matches('/').to_owned())
|
||||
.filter(|sha_dir| sha_dir.len() == 64 && sha_dir.bytes().all(|byte| byte.is_ascii_hexdigit()))
|
||||
.map(|sha_dir| {
|
||||
if root_prefix.is_empty() {
|
||||
sha_dir
|
||||
} else {
|
||||
format!("{root_prefix}/{sha_dir}")
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let listed_upload_dirs = stream::iter(sha_dirs)
|
||||
.map(|sha_dir| {
|
||||
@@ -1124,6 +1131,7 @@ impl SetDisks {
|
||||
orig_bucket: &str,
|
||||
error_path: &str,
|
||||
root_prefix: &str,
|
||||
require_all_disks: bool,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize, bool)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
if disks.is_empty() {
|
||||
@@ -1165,7 +1173,8 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, discovery_quorum) {
|
||||
let required_discoveries = if require_all_disks { disks.len() } else { discovery_quorum };
|
||||
if let Some(err) = reduce_read_quorum_errs(&discovery_errors, OBJECT_OP_IGNORED_ERRS, required_discoveries) {
|
||||
return Err(to_object_err(err.into(), vec![orig_bucket, error_path]));
|
||||
}
|
||||
|
||||
@@ -1187,7 +1196,7 @@ impl SetDisks {
|
||||
|
||||
pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result<Option<String>> {
|
||||
let (_, paths, _, _) = self
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "")
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "", false)
|
||||
.await?;
|
||||
Ok(paths.into_iter().next())
|
||||
}
|
||||
@@ -1200,8 +1209,9 @@ impl SetDisks {
|
||||
upload_identity: &str,
|
||||
) -> Result<Vec<String>> {
|
||||
let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object));
|
||||
let (_, candidate_paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?;
|
||||
let (_, candidate_paths, _, has_minority_candidate) = self
|
||||
.discover_multipart_upload_paths(bucket, object, &expected_parent, false)
|
||||
.await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::DecommissionCapacityBlocked {
|
||||
message: "data movement multipart cleanup found an upload path on fewer than the discovery quorum of disks"
|
||||
@@ -1750,6 +1760,54 @@ impl SetDisks {
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
// Every disk must confirm absence in both namespaces before cleanup can
|
||||
// remove catalog protection from the bucket.
|
||||
for root_prefix in ["", DATA_MOVEMENT_MULTIPART_PREFIX] {
|
||||
let (disks, paths, _, has_minority_candidate) =
|
||||
self.discover_multipart_upload_paths(bucket, "", root_prefix, true).await?;
|
||||
if has_minority_candidate {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
for upload_path in paths {
|
||||
let (sha_dir, _) = upload_path
|
||||
.rsplit_once('/')
|
||||
.filter(|(directory, upload_id)| !directory.is_empty() && !upload_id.is_empty())
|
||||
.ok_or(DiskError::CorruptedFormat)?;
|
||||
let (metadata, errors) =
|
||||
Self::read_all_fileinfo(&disks, bucket, RUSTFS_META_MULTIPART_BUCKET, &upload_path, "", false, false, false)
|
||||
.await?;
|
||||
let (read_quorum, _) = Self::object_quorum_from_meta(&metadata, &errors, self.default_parity_count)?;
|
||||
let read_quorum = usize::try_from(read_quorum).map_err(|_| DiskError::ErasureReadQuorum)?;
|
||||
if let Some(error) = reduce_read_quorum_errs(&errors, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(to_object_err(error.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
let (_, modified, etag) = Self::list_online_disks(&disks, &metadata, &errors, read_quorum);
|
||||
let file = Self::pick_valid_fileinfo(&metadata, modified, etag, read_quorum)?;
|
||||
let (Some(owner_bucket), Some(owner_object)) = (
|
||||
file.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY),
|
||||
file.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY),
|
||||
) else {
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
};
|
||||
if owner_bucket.is_empty()
|
||||
|| owner_object.is_empty()
|
||||
|| Self::get_multipart_sha_dir(owner_bucket, owner_object)
|
||||
!= sha_dir
|
||||
.strip_prefix(DATA_MOVEMENT_MULTIPART_PREFIX)
|
||||
.and_then(|suffix| suffix.strip_prefix('/'))
|
||||
.unwrap_or(sha_dir)
|
||||
{
|
||||
return Err(to_object_err(DiskError::CorruptedFormat.into(), vec![bucket, &upload_path]));
|
||||
}
|
||||
if owner_bucket == bucket {
|
||||
return Err(Error::BucketNotEmpty(bucket.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn list_multipart_uploads_for_incarnation(
|
||||
&self,
|
||||
@@ -1761,7 +1819,8 @@ impl SetDisks {
|
||||
max_uploads: usize,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
let (disks, candidate_paths, discovery_quorum, _) = self.discover_multipart_upload_paths(bucket, prefix, "").await?;
|
||||
let (disks, candidate_paths, discovery_quorum, _) =
|
||||
self.discover_multipart_upload_paths(bucket, prefix, "", false).await?;
|
||||
let listed_uploads = stream::iter(candidate_paths)
|
||||
.map(|upload_path| {
|
||||
let disks = &disks;
|
||||
@@ -11194,6 +11253,163 @@ mod tests {
|
||||
assert_eq!(upload_uuid_suffix(&recovered[0]), upload_uuid_suffix(&upload.upload_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_distinguishes_bucket_ownership() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-target";
|
||||
let other = "multipart-cleanup-other";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create target bucket");
|
||||
disk.make_volume(other).await.expect("create other bucket");
|
||||
}
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an empty upload namespace permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(other, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in other bucket");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("an upload owned by another bucket permits cleanup");
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create upload in target bucket");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_requires_every_disk_online() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-offline";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks.disks.write().await[3] = None;
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing preserves its discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("an offline disk must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the offline disk must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
set_disks.disks.write().await[3] = Some(disk_stores[3].clone());
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after recovery");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_a_failed_disk_enumeration() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-list-error";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let sha_dir = SetDisks::get_multipart_sha_dir(bucket, "object.bin");
|
||||
let multipart_dir = temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET);
|
||||
tokio::fs::create_dir_all(&multipart_dir)
|
||||
.await
|
||||
.expect("create multipart staging directory");
|
||||
tokio::fs::write(multipart_dir.join(&sha_dir), b"not a directory")
|
||||
.await
|
||||
.expect("replace one upload parent directory with a file");
|
||||
multipart_upload_paths_on_disk(disk_stores[0].clone(), bucket, "")
|
||||
.await
|
||||
.expect_err("the affected disk must fail directory enumeration");
|
||||
let listed = set_disks
|
||||
.list_multipart_uploads_for_incarnation(bucket, "", None, None, None, 1000, None)
|
||||
.await
|
||||
.expect("ordinary listing still accepts the remaining discovery quorum");
|
||||
assert!(listed.uploads.is_empty());
|
||||
let error = set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("a failed disk enumeration must prevent confirmation of an empty multipart namespace");
|
||||
assert!(
|
||||
matches!(&error, Error::InsufficientReadQuorum(error_bucket, error_path) if error_bucket == bucket && error_path.is_empty()),
|
||||
"the failed enumeration must produce a bucket-scoped read quorum error: {error:?}"
|
||||
);
|
||||
|
||||
tokio::fs::remove_file(temp_dirs[0].path().join(RUSTFS_META_MULTIPART_BUCKET).join(sha_dir))
|
||||
.await
|
||||
.expect("remove the invalid directory entry");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect("all disks can confirm the empty namespace after repair");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_data_movement_uploads() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-data-movement";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
let mut metadata = HashMap::new();
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut metadata,
|
||||
rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD,
|
||||
format!("v1:{}:{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp_nanos()),
|
||||
);
|
||||
set_disks
|
||||
.new_multipart_upload(
|
||||
bucket,
|
||||
"object.bin",
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
user_defined: metadata,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create data movement upload");
|
||||
assert!(
|
||||
matches!(set_disks.ensure_no_multipart_uploads(bucket).await, Err(Error::BucketNotEmpty(name)) if name == bucket)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_absence_inspection_rejects_unreadable_ownership() {
|
||||
let (temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "multipart-cleanup-corrupt";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("create bucket");
|
||||
}
|
||||
set_disks
|
||||
.new_multipart_upload(bucket, "object.bin", &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload");
|
||||
let mut corrupted = 0;
|
||||
for temp_dir in &temp_dirs {
|
||||
for path in multipart_meta_files_on_disk(temp_dir, "xl.meta").await {
|
||||
tokio::fs::write(path, b"invalid upload metadata")
|
||||
.await
|
||||
.expect("corrupt upload metadata");
|
||||
corrupted += 1;
|
||||
}
|
||||
}
|
||||
assert!(corrupted > 0, "upload metadata must exist before corruption");
|
||||
set_disks
|
||||
.ensure_no_multipart_uploads(bucket)
|
||||
.await
|
||||
.expect_err("unreadable upload ownership must block cleanup");
|
||||
}
|
||||
|
||||
/// Regression (issue #5716): a single upload directory whose `xl.meta` was
|
||||
/// destroyed (crash mid-write, torn disk state) must degrade to that upload
|
||||
/// alone. Failing the whole ListMultipartUploads turns one piece of stale
|
||||
|
||||
@@ -44,15 +44,16 @@ pub(crate) mod lifecycle {
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker, WalkOptions as StorageWalkOptions,
|
||||
WalkVersionsSortOrder,
|
||||
AuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, VersionMarker,
|
||||
WalkOptions as StorageWalkOptions, WalkVersionsSortOrder,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use rustfs_storage_api::{
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadResult, PartInfo,
|
||||
CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartOperations, MultipartUploadInspection,
|
||||
MultipartUploadResult, PartInfo,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -523,7 +523,22 @@ impl ECStore {
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, validate).await
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, None, validate).await
|
||||
}
|
||||
|
||||
/// Validate an external fence and the bucket incarnation in one metadata transaction.
|
||||
pub async fn update_bucket_metadata_config_for_incarnation_validated<F>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
config_file: &str,
|
||||
data: Vec<u8>,
|
||||
expected_incarnation_id: Uuid,
|
||||
validate: F,
|
||||
) -> Result<OffsetDateTime>
|
||||
where
|
||||
F: FnOnce() -> Result<()> + Send,
|
||||
{
|
||||
metadata_sys::update_validated_in(&self.ctx, bucket, config_file, data, Some(expected_incarnation_id), validate).await
|
||||
}
|
||||
|
||||
pub async fn bucket_incarnation_id(&self, bucket: &str) -> Result<Uuid> {
|
||||
|
||||
@@ -5340,203 +5340,6 @@ mod tests {
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[cfg(feature = "test-util")]
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn conditional_multipart_must_not_commit_against_an_older_readable_pool() {
|
||||
use crate::storage_api_contracts::multipart::CompletePart;
|
||||
use crate::storage_api_contracts::object::HTTPPreconditions;
|
||||
|
||||
let temp_dir = tempfile::tempdir().expect("create conditional multipart test directory");
|
||||
let (_ctx, store, shutdown) =
|
||||
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "conditional-mpu-unreadable", &[4, 4])).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
let bucket = format!("conditional-mpu-{}", uuid::Uuid::new_v4().simple());
|
||||
let object = "object";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create conditional multipart test bucket");
|
||||
let mut old_body = PutObjReader::from_vec(b"older-pool".to_vec());
|
||||
let old = store.pools[0]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut old_body,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write old object to staging pool");
|
||||
let upload = store.pools[0]
|
||||
.new_multipart_upload(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("create multipart upload in pool 0");
|
||||
let mut part_body = PutObjReader::from_vec(b"stale-writer".to_vec());
|
||||
let part = store.pools[0]
|
||||
.put_object_part(&bucket, object, &upload.upload_id, 1, &mut part_body, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("stage multipart part in pool 0");
|
||||
let mut new_body = PutObjReader::from_vec(b"newer-pool".to_vec());
|
||||
let new = store.pools[1]
|
||||
.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut new_body,
|
||||
&ObjectOptions {
|
||||
mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("write newer object to pool 1");
|
||||
let old_etag = old.etag.clone().expect("old object must have an ETag");
|
||||
let before = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
|
||||
|
||||
let mut metadata = Vec::new();
|
||||
for disk_index in 0..4 {
|
||||
let path = temp_dir
|
||||
.path()
|
||||
.join(format!("pool1/set0/disk{disk_index}/{bucket}/{object}/xl.meta"));
|
||||
let original = tokio::fs::read(&path)
|
||||
.await
|
||||
.expect("save original object metadata before fault injection");
|
||||
metadata.push((path, original));
|
||||
}
|
||||
let mut fault_error = None;
|
||||
for (path, _) in &metadata {
|
||||
if let Err(err) = tokio::fs::write(path, b"invalid object metadata").await {
|
||||
fault_error = Some(err);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let mut fault_contents = Vec::new();
|
||||
if fault_error.is_none() {
|
||||
for (path, _) in &metadata {
|
||||
fault_contents.push(tokio::fs::read(path).await);
|
||||
}
|
||||
store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.invalidate_get_object_metadata_cache(&bucket, object)
|
||||
.await;
|
||||
}
|
||||
let unreadable = if fault_error.is_none() {
|
||||
Some(
|
||||
store.pools[1]
|
||||
.get_object_info(
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
metadata_cache_safe: false,
|
||||
suppress_read_repair: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let completed = if fault_error.is_none() {
|
||||
let completion_store = Arc::clone(&store);
|
||||
let completion_bucket = bucket.clone();
|
||||
let upload_id = upload.upload_id.clone();
|
||||
let part_etag = part.etag.clone();
|
||||
Some(
|
||||
tokio::spawn(async move {
|
||||
completion_store
|
||||
.complete_multipart_upload(
|
||||
&completion_bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: part_etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(old_etag),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut restore_error = None;
|
||||
for (path, original) in &metadata {
|
||||
if let Err(err) = tokio::fs::write(path, original).await {
|
||||
restore_error = Some(err);
|
||||
}
|
||||
}
|
||||
store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.invalidate_get_object_metadata_cache(&bucket, object)
|
||||
.await;
|
||||
let after = store.get_object_info(&bucket, object, &ObjectOptions::default()).await;
|
||||
let recovered_body = match store
|
||||
.get_object_reader(&bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
{
|
||||
Ok(mut reader) => {
|
||||
let mut body = Vec::new();
|
||||
let result = reader
|
||||
.read_to_end(&mut body)
|
||||
.await
|
||||
.map(|_| body)
|
||||
.map_err(|err| err.to_string());
|
||||
drop(reader);
|
||||
result
|
||||
}
|
||||
Err(err) => Err(err.to_string()),
|
||||
};
|
||||
let parts = store
|
||||
.list_object_parts(&bucket, object, &upload.upload_id, None, 100, &ObjectOptions::default())
|
||||
.await;
|
||||
shutdown.cancel();
|
||||
|
||||
assert!(fault_error.is_none(), "failed to inject metadata fault: {fault_error:?}");
|
||||
assert!(restore_error.is_none(), "failed to restore metadata: {restore_error:?}");
|
||||
assert_eq!(fault_contents.len(), 4, "all four pool 1 metadata files must be inspected");
|
||||
for observed in fault_contents {
|
||||
assert_eq!(
|
||||
observed.expect("read injected metadata"),
|
||||
b"invalid object metadata",
|
||||
"each pool 1 metadata file must contain the injected corruption"
|
||||
);
|
||||
}
|
||||
assert_eq!(before.expect("latest object before fault").etag, new.etag);
|
||||
assert_ne!(old.etag, new.etag, "fixture bodies must produce different ETags");
|
||||
let read_error = unreadable
|
||||
.expect("metadata fault should have been injected")
|
||||
.expect_err("the newer pool must fail its object metadata read during the fault");
|
||||
assert!(
|
||||
!crate::error::is_err_object_not_found(&read_error) && !crate::error::is_err_version_not_found(&read_error),
|
||||
"corrupt metadata must produce a non-not-found read error: {read_error:?}"
|
||||
);
|
||||
let completed = completed
|
||||
.expect("completion should have been attempted")
|
||||
.expect("completion task should not panic");
|
||||
let Err(completion_error) = completed else {
|
||||
panic!("stale If-Match must not commit while the newer object's metadata is unreadable");
|
||||
};
|
||||
assert_eq!(completion_error, read_error, "completion must propagate the unreadable pool's error");
|
||||
assert_eq!(after.expect("newer object after recovery").etag, new.etag);
|
||||
assert_eq!(recovered_body.expect("newer body after recovery"), b"newer-pool");
|
||||
let parts = parts.expect("failed completion must leave the upload retryable");
|
||||
assert_eq!(parts.parts.len(), 1);
|
||||
assert_eq!(parts.parts[0].etag, part.etag);
|
||||
}
|
||||
|
||||
async fn migrate_versioned_decommission_test_object(
|
||||
store: &Arc<crate::store::ECStore>,
|
||||
bucket: &str,
|
||||
|
||||
@@ -37,8 +37,8 @@ use crate::error::{
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions};
|
||||
use crate::set_disk::SetDisks;
|
||||
use crate::storage_api_contracts::list::{
|
||||
ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr,
|
||||
StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
AuthoritativeListOperations, ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo,
|
||||
StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
@@ -80,6 +80,21 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl AuthoritativeListOperations for ECStore {
|
||||
async fn walk_authoritative(
|
||||
self: Arc<Self>,
|
||||
rx: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
check_list_objs_args(bucket, prefix, &None)?;
|
||||
self.walk_authoritatively(rx, bucket, prefix, result, opts).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the versioned listing options used by recursive object walks.
|
||||
///
|
||||
/// `list_path_result` normally derives these scan bounds before dispatching to
|
||||
@@ -1519,6 +1534,57 @@ impl ECStore {
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
async fn walk_authoritatively(
|
||||
self: Arc<Self>,
|
||||
cancellation: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Sender<ObjectInfoOrErr>,
|
||||
opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
let cancel = cancellation.child_token();
|
||||
let _cancel_on_drop = cancel.clone().drop_guard();
|
||||
let mut scans = Vec::new();
|
||||
let mut inputs = Vec::new();
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
inputs.push(receiver);
|
||||
let mut scan = recursive_walk_list_path_options(bucket.to_owned(), prefix, &opts);
|
||||
// Internal absence checks must include reserved object prefixes.
|
||||
scan.skip_hidden_prefix_check = true;
|
||||
scans.push(set.list_versions_authoritatively(cancel.clone(), scan, sender));
|
||||
}
|
||||
}
|
||||
if scans.is_empty() {
|
||||
return Err(StorageError::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
let (sender, receiver) = mpsc::channel(100);
|
||||
let forwarder = spawn_walk_entry_forwarder(cancel.clone(), receiver, result, opts, bucket.to_owned());
|
||||
let merge = spawn_listing_merge(cancel.clone(), inputs, sender);
|
||||
let scan_results = join_all(scans).await;
|
||||
let merge_result = merge.await.map_err(Error::from)?;
|
||||
let forward_result = forwarder.await.map_err(Error::from);
|
||||
|
||||
// An unavailable set cannot establish absence, even when another set
|
||||
// completed. Only an explicitly missing directory or volume is empty.
|
||||
for scan in scan_results {
|
||||
match scan {
|
||||
Ok(outcome) if !outcome.limit_reached => {}
|
||||
Ok(_) => return Err(StorageError::ErasureReadQuorum),
|
||||
Err(StorageError::FileNotFound | StorageError::VolumeNotFound) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
merge_result?;
|
||||
forward_result?;
|
||||
if cancellation.is_cancelled() {
|
||||
return Err(Error::other("authoritative object walk was cancelled"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Forwards merged raw entries to a walk consumer as `ObjectInfoOrErr`,
|
||||
@@ -2489,7 +2555,8 @@ impl SetDisks {
|
||||
/// Version scan for set-scoped callers that act on absence: set walks
|
||||
/// (tier reference proofs, tier-delete journals, pool maintenance) and the
|
||||
/// recursive prefix-delete scan. Every online drive is read and each
|
||||
/// version is resolved at the read quorum of its own erasure layout; a
|
||||
/// version uses the read quorum of its own erasure layout. Null versions
|
||||
/// in the system bucket also retain GET's commit-proof selection. A
|
||||
/// version that cannot be ruled out fails the scan instead of being
|
||||
/// omitted. Merged `ListObjectVersions` pages go through `list_path`.
|
||||
async fn list_versions_authoritatively(
|
||||
@@ -2828,6 +2895,7 @@ mod test {
|
||||
use crate::error::{Result, StorageError};
|
||||
use crate::object_api::{ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::{SetDisks, hermetic_set_disks_isolated};
|
||||
use crate::storage_api_contracts::list::AuthoritativeListOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::storage_api_contracts::object::ObjectIO as _;
|
||||
use bytes::Bytes;
|
||||
@@ -5688,6 +5756,126 @@ mod test {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_rejects_uncommitted_null_version_split() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
for new_etag in ["new", "old"] {
|
||||
let new =
|
||||
test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), new_etag, Uuid::from_u128(2));
|
||||
let entries = MetaCacheEntries(vec![Some(old.clone()), Some(old.clone()), Some(new.clone()), Some(new)]);
|
||||
|
||||
assert!(matches!(
|
||||
SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2),
|
||||
Err(DiskError::ErasureReadQuorum)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_selects_committed_null_version() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (new_copies, expected) in [(1, "old"), (3, "new"), (4, "new")] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| Some(if slot < new_copies { new.clone() } else { old.clone() }))
|
||||
.collect(),
|
||||
);
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &[None, None, None, None], 4, 2)
|
||||
.expect("committed generation should resolve")
|
||||
.expect("committed record should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_matches_degraded_null_version_reads() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for (old_copies, new_copies, expected) in [
|
||||
(2, 1, Some("old")),
|
||||
(1, 2, None),
|
||||
(0, 3, Some("new")),
|
||||
(0, 2, Some("new")),
|
||||
(0, 1, None),
|
||||
] {
|
||||
let entries = MetaCacheEntries(
|
||||
(0..4)
|
||||
.map(|slot| {
|
||||
if slot < old_copies {
|
||||
Some(old.clone())
|
||||
} else if slot < old_copies + new_copies {
|
||||
Some(new.clone())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
let errors = (0..4)
|
||||
.map(|slot| (slot >= old_copies + new_copies).then_some(DiskError::DiskNotFound))
|
||||
.collect::<Vec<_>>();
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2);
|
||||
|
||||
if let Some(expected) = expected {
|
||||
let info = resolved
|
||||
.expect("degraded committed record should resolve")
|
||||
.expect("degraded committed record should be present")
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some(expected));
|
||||
} else {
|
||||
assert!(matches!(resolved, Err(DiskError::ErasureReadQuorum)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_user_bucket_scan_keeps_null_version_read_quorum_selection() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let old = test_unversioned_object_meta_entry("entry.json", time, "old", Uuid::from_u128(1));
|
||||
let new = test_unversioned_object_meta_entry("entry.json", time + time::Duration::seconds(1), "new", Uuid::from_u128(2));
|
||||
for old_copies in [1, 2] {
|
||||
let entries = MetaCacheEntries(vec![
|
||||
Some(new.clone()),
|
||||
Some(new.clone()),
|
||||
Some(old.clone()),
|
||||
(old_copies == 2).then(|| old.clone()),
|
||||
]);
|
||||
let errors = [None, None, None, (old_copies == 1).then_some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions("bucket", entries, &errors, 4, 2)
|
||||
.expect("user bucket read quorum should resolve")
|
||||
.expect("user bucket record should be present");
|
||||
let info = resolved.to_fileinfo("bucket").expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("new"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_system_bucket_scan_keeps_explicit_version_read_quorum() {
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_700_000_000).expect("valid fixture time");
|
||||
let entry = test_object_meta_entry_with_erasure_versions("entry.json", &[(time, "version", 2, 2)]);
|
||||
let entries = MetaCacheEntries(vec![Some(entry.clone()), Some(entry), None, None]);
|
||||
let errors = [None, None, Some(DiskError::DiskNotFound), Some(DiskError::DiskNotFound)];
|
||||
let resolved = SetDisks::resolve_listed_versions(RUSTFS_META_BUCKET, entries, &errors, 4, 2)
|
||||
.expect("explicit version read quorum should resolve")
|
||||
.expect("explicit version should be present");
|
||||
let info = resolved
|
||||
.to_fileinfo(RUSTFS_META_BUCKET)
|
||||
.expect("resolved metadata should decode");
|
||||
|
||||
assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
|
||||
assert_eq!(info.metadata.get("etag").map(String::as_str), Some("version"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authoritative_version_scan_tolerates_corrupt_replicas_only_with_version_quorum() {
|
||||
let entry = test_object_meta_entry_with_erasure_versions(
|
||||
@@ -6029,6 +6217,49 @@ mod test {
|
||||
Bytes::from(meta.marshal_msg().expect("encode fixture xl.meta"))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_store_walk_includes_reserved_prefix_versions() {
|
||||
let bucket = "catalog-cleanup-versions";
|
||||
let name = ".rustfs-table/namespaces/ns/tables/table/metadata/metadata.json";
|
||||
let (_dirs, store) = single_set_store(16).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
for (slot, disk) in set.disks.read().await.iter().enumerate() {
|
||||
let disk = disk.as_ref().expect("fixture drive is online");
|
||||
disk.make_volume(bucket).await.expect("create bucket volume");
|
||||
disk.write_all(
|
||||
bucket,
|
||||
&format!("{name}/{STORAGE_FORMAT_FILE}"),
|
||||
versioned_object_metadata(bucket, name, &[(2, 1_700_000_002), (1, 1_700_000_001)], slot),
|
||||
)
|
||||
.await
|
||||
.expect("persist reserved metadata versions");
|
||||
}
|
||||
let (sender, mut receiver) = mpsc::channel(1);
|
||||
let walk = store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let collect = async {
|
||||
let mut versions = Vec::new();
|
||||
while let Some(item) = receiver.recv().await {
|
||||
assert!(item.err.is_none(), "reserved metadata must decode: {:?}", item.err);
|
||||
let item = item.item.expect("walk must return an object");
|
||||
versions.push((item.name, item.version_id));
|
||||
}
|
||||
versions
|
||||
};
|
||||
let (walk, versions) = timeout(Duration::from_secs(10), async { tokio::join!(walk, collect) })
|
||||
.await
|
||||
.expect("authoritative reserved-prefix walk must terminate");
|
||||
walk.expect("authoritative walk must succeed");
|
||||
assert_eq!(
|
||||
versions,
|
||||
[
|
||||
(name.to_string(), Some(Uuid::from_u128(1))),
|
||||
(name.to_string(), Some(Uuid::from_u128(2)))
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
// A versioned DeleteObject(versionId) can reach write quorum while three
|
||||
// drives are offline; they keep the deleted version after they return. A
|
||||
// different drive going offline later must not turn that sub-quorum
|
||||
@@ -6099,6 +6330,16 @@ mod test {
|
||||
.await
|
||||
.expect("set walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let (sender, mut receiver) = mpsc::channel(16);
|
||||
let walk =
|
||||
store
|
||||
.clone()
|
||||
.walk_authoritative(CancellationToken::new(), bucket, "", sender, super::WalkOptions::default());
|
||||
let drain = async { while receiver.recv().await.is_some() {} };
|
||||
let (walk, ()) = timeout(Duration::from_secs(10), async { tokio::join!(walk, drain) })
|
||||
.await
|
||||
.expect("authoritative store walk must terminate");
|
||||
assert!(matches!(walk, Err(StorageError::ErasureReadQuorum)), "{walk:?}");
|
||||
let recursive_delete_scan = set
|
||||
.clone()
|
||||
.inner_list_object_versions_for_recursive_delete(bucket, "", None, None, 1000)
|
||||
|
||||
@@ -498,17 +498,6 @@ pub struct ECStore {
|
||||
pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
|
||||
}
|
||||
|
||||
impl Drop for ECStore {
|
||||
/// Clear the instance context's `BucketMetadataSys` when its backing store
|
||||
/// is dropped. Without this, the shared bootstrap context retains a
|
||||
/// `BucketMetadataSys` whose `Weak<ECStore>` is dangling after the store
|
||||
/// goes away (e.g. between sequential test fixtures that share the
|
||||
/// bootstrap), and the next `object_store()` call panics.
|
||||
fn drop(&mut self) {
|
||||
self.ctx.clear_bucket_metadata_sys_if_stale();
|
||||
}
|
||||
}
|
||||
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
|
||||
|
||||
@@ -23,6 +23,23 @@ use std::collections::HashSet;
|
||||
|
||||
const MULTIPART_LIST_SET_CONCURRENCY: usize = 4;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::multipart::MultipartUploadInspection for ECStore {
|
||||
type Error = Error;
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<()> {
|
||||
let (_, lifecycle_guard) = self
|
||||
.guard_multipart_bucket_incarnation(bucket, &ObjectOptions::default())
|
||||
.await?;
|
||||
for pool in &self.pools {
|
||||
for set in &pool.disk_set {
|
||||
set.ensure_no_multipart_uploads(bucket).await?;
|
||||
}
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_guard_held(lifecycle_guard.as_ref(), bucket, "")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DATA_MOVEMENT_MULTIPART_DISCOVERY_COUNTS: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, usize>>> =
|
||||
std::sync::OnceLock::new();
|
||||
@@ -612,7 +629,7 @@ impl ECStore {
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& !opts.data_movement
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, &opts, false).await
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, &opts).await
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
@@ -1154,33 +1171,12 @@ impl ECStore {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn check_write_precondition(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
require_all_pools: bool,
|
||||
) -> Option<StorageError> {
|
||||
async fn check_write_precondition(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Option<StorageError> {
|
||||
let mut lookup_opts = opts.clone();
|
||||
let http_preconditions = lookup_opts.http_preconditions?;
|
||||
lookup_opts.http_preconditions = None;
|
||||
lookup_opts.no_lock = true;
|
||||
let oi = if require_all_pools {
|
||||
lookup_opts.metadata_cache_safe = false;
|
||||
lookup_opts.suppress_read_repair = true;
|
||||
if let Err(err) = check_object_args(bucket, object) {
|
||||
return Some(err);
|
||||
}
|
||||
let encoded_object = encode_dir_object(object);
|
||||
self.get_latest_object_info_for_conditional_write(bucket, &encoded_object, &lookup_opts)
|
||||
.await
|
||||
.and_then(|(info, _)| {
|
||||
lookup_opts.precondition_check(&info)?;
|
||||
Ok(info)
|
||||
})
|
||||
} else {
|
||||
self.get_object_info_for_delete(bucket, object, &lookup_opts).await
|
||||
};
|
||||
let oi = self.get_object_info_for_delete(bucket, object, &lookup_opts).await;
|
||||
crate::set_disk::SetDisks::evaluate_write_precondition(oi, &http_preconditions, bucket, object, opts)
|
||||
}
|
||||
|
||||
@@ -1207,13 +1203,11 @@ impl ECStore {
|
||||
.await;
|
||||
return enqueue_transition_after_write(self.as_ref(), result, LcEventSrc::S3CompleteMultipartUpload, opts).await;
|
||||
}
|
||||
if opts.http_preconditions.is_some() && !opts.data_movement {
|
||||
// Preserve explicitly scoped internal reads and borrowed-lock callers.
|
||||
let require_all_pools =
|
||||
!opts.replication_request && !opts.no_lock && !opts.skip_decommissioned && !opts.skip_rebalancing;
|
||||
if let Some(err) = self.check_write_precondition(bucket, object, opts, require_all_pools).await {
|
||||
return Err(err);
|
||||
}
|
||||
if opts.http_preconditions.is_some()
|
||||
&& !opts.data_movement
|
||||
&& let Some(err) = self.check_write_precondition(bucket, object, opts).await
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let result = match self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await {
|
||||
|
||||
@@ -837,29 +837,7 @@ impl ECStore {
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
self.get_latest_object_info_with_pool_policy(bucket, object, opts, false)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Observe every pool for a conditional-write preflight. This does not
|
||||
/// hold the namespace locks needed to prove a condition at publication.
|
||||
pub(super) async fn get_latest_object_info_for_conditional_write(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
self.get_latest_object_info_with_pool_policy(bucket, object, opts, true).await
|
||||
}
|
||||
|
||||
async fn get_latest_object_info_with_pool_policy(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
require_all_pools: bool,
|
||||
) -> Result<(ObjectInfo, usize)> {
|
||||
let suspended_pools = if opts.skip_decommissioned && !require_all_pools {
|
||||
let suspended_pools = if opts.skip_decommissioned {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
Some(
|
||||
(0..self.pools.len())
|
||||
@@ -875,7 +853,7 @@ impl ECStore {
|
||||
continue;
|
||||
}
|
||||
|
||||
if opts.skip_rebalancing && !require_all_pools && self.is_pool_rebalancing(idx).await {
|
||||
if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -895,9 +873,6 @@ impl ECStore {
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
if require_all_pools && !is_err_object_not_found(&e) && !is_err_version_not_found(&e) {
|
||||
return Err(e);
|
||||
}
|
||||
candidates.push(LatestObjectInfoCandidate {
|
||||
info: None,
|
||||
idx,
|
||||
|
||||
@@ -90,7 +90,9 @@ pub use bucket::BUCKET_LIFECYCLE_LOCK_OBJECT;
|
||||
pub use bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions, SRBucketDeleteOp};
|
||||
pub use capability::{CapabilitySnapshotError, CapabilityState, CapabilityStatus};
|
||||
pub use error::{StorageErrorCode, StorageResult};
|
||||
pub use multipart::MultipartUploadInspection;
|
||||
pub use multipart::{CompletePart, ListMultipartsInfo, ListPartsInfo, MultipartInfo, MultipartUploadResult, PartInfo};
|
||||
pub use object::AuthoritativeListOperations;
|
||||
pub use object::DeleteAccounting;
|
||||
pub use object::ObjectLockDeleteOptions;
|
||||
pub use object::ObjectToDeleteIdentity;
|
||||
|
||||
@@ -15,6 +15,15 @@
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
/// Checks upload absence before removing a bucket's data protections.
|
||||
#[async_trait::async_trait]
|
||||
pub trait MultipartUploadInspection: Send + Sync {
|
||||
type Error: std::error::Error + Send + Sync + 'static;
|
||||
|
||||
/// Unknown or unreadable upload ownership must fail this check.
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct MultipartUploadResult {
|
||||
pub upload_id: String,
|
||||
|
||||
@@ -566,6 +566,29 @@ pub trait ListOperations: Send + Sync + fmt::Debug {
|
||||
) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
/// Quorum-backed enumeration for decisions that require proving object absence.
|
||||
///
|
||||
/// Consumers must drain the result stream and await successful completion before
|
||||
/// trusting the scan. An error or cancellation can follow partial results and
|
||||
/// leaves absence unproven. Callers coordinate concurrent mutations separately;
|
||||
/// this capability does not provide a snapshot or retain a mutation lock.
|
||||
#[async_trait::async_trait]
|
||||
pub trait AuthoritativeListOperations: ListOperations {
|
||||
/// Resolves every set at its version read quorum, including reserved prefixes.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error when a set cannot establish quorum, metadata cannot be
|
||||
/// resolved, or enumeration is interrupted. Retry starts a new scan.
|
||||
async fn walk_authoritative(
|
||||
self: Arc<Self>,
|
||||
rx: Self::WalkCancellation,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
result: Self::WalkResultSender,
|
||||
opts: Self::WalkOptions,
|
||||
) -> Result<(), Self::Error>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub trait MultipartOperations: Send + Sync + fmt::Debug {
|
||||
|
||||
@@ -91,6 +91,26 @@ Use `GET /iceberg/v1/buckets/{warehouse}` to check `enabled` and `disable-suppor
|
||||
|
||||
Review the bucket lifecycle configuration before disabling: clearing table-bucket protection allows existing object expiration rules to run again. Objects are preserved by the disable operation itself. Catalog disablement requires no namespaces, retained table/view resources, or objects under `.rustfs-table/`; an active rename or backing migration also blocks it. Clean up through supported catalog operations; do not manually delete internal metadata to bypass these checks.
|
||||
|
||||
The server persists an inactive catalog entry before clearing the bucket marker. If marker publication fails, retry the same DELETE; the marker retains object protections until the retry succeeds. A subsequent PUT explicitly enables the bucket again. Stale requests cannot initialize an inactive catalog entry. No table metadata or object data is deleted by either state change.
|
||||
The server persists an inactive catalog entry before clearing the bucket marker. If marker publication fails, retry the same DELETE; the marker retains object protections until the retry succeeds. A subsequent PUT explicitly enables the bucket again. Stale requests cannot initialize an inactive catalog entry. A plain DELETE and a subsequent PUT preserve table metadata and object data.
|
||||
|
||||
### Clean Up Deleted Empty Tables
|
||||
|
||||
For the object-backed catalog, `DELETE /iceberg/v1/buckets/{warehouse}?cleanup=true` removes the metadata and internal records left by deleted empty tables, then disables the table catalog. Ordinary table deletion continues to preserve files. Cleanup preserves existing table record formats and can process empty tables deleted before the server upgrade.
|
||||
|
||||
Complete the server upgrade across all serving nodes and set the existing `RUSTFS_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED=true` setting before requesting cleanup. Delete the empty tables and their namespaces through the catalog APIs, then send the cleanup request with `admin:SetTableBucket` and `s3:GetObject`/`s3:DeleteObject` permissions for the target metadata objects. The `/_iceberg/v1` alias accepts the same option. Omitting `cleanup` or setting it to `false` keeps the ordinary disable behavior; repeated or invalid values are rejected.
|
||||
|
||||
Cleanup accepts only recognized empty-table metadata in the default managed warehouse. Active catalog resources, snapshots or other data references, user objects, object versions, incomplete multipart uploads, corrupt or unknown metadata, and unfinished catalog operations block cleanup. The server validates the complete cleanup plan and object permissions before deleting any object. Durable-strong catalogs retain their existing disable procedure.
|
||||
|
||||
If a storage operation fails during cleanup, retry the same request. The server checks the remaining objects again and keeps the bucket's table protection enabled until cleanup and catalog disablement complete. No separate cleanup task file is created. After a successful response with `enabled: false`, delete the empty bucket through the normal S3 `DeleteBucket` operation.
|
||||
|
||||
## Object-Backed Registration Reads
|
||||
|
||||
Table registration checks table ID uniqueness and warehouse directory overlap using the existing catalog and warehouse index records. Each registration obtains a complete inventory under the existing publication and migration guards. System-bucket inventory records must satisfy the same metadata commit checks as ordinary catalog reads. An incomplete or failed inventory rejects registration.
|
||||
|
||||
Each server keeps a bounded memory cache of validated ownership records. A record can be reused only when its object identity and ETag match the current inventory. New or changed records are read and validated again. Cache eviction and process restarts cause additional reads; catalog state continues to come from the existing persisted records. The cache is automatic and requires no additional configuration or stored objects.
|
||||
|
||||
Registration still enumerates the namespace and warehouse index prefixes, so enumeration work grows with catalog size. Cache hits reduce object-body reads and JSON parsing. A cache hit verifies the current metadata identity and reuses previously validated content; it does not re-read the current body to check its physical readability. Conditional writes, rollback, and lock-loss fencing continue to control publication.
|
||||
|
||||
Existing catalog formats and Console and Iceberg client request formats are preserved. Concurrent writers must follow the existing publication, migration, and storage fencing protocol; the memory cache does not provide synchronization for writers that bypass those guards.
|
||||
|
||||
Before rolling back to a server version without this endpoint, re-enable disabled catalogs with PUT while the supporting version is running. Older enable handlers do not reactivate inactive catalog entries. Complete the server upgrade across the cluster before using disablement.
|
||||
|
||||
@@ -33,8 +33,8 @@ Run URLs are `https://github.com/rustfs/rustfs/actions/runs/<id>`. Re-open the r
|
||||
| `workflow_dispatch` | manual full validation | [41152382212](https://github.com/rustfs/rustfs/actions/runs/41152382212) | full suite set ran and the aggregate accepted the complete plan (85 min) |
|
||||
| `workflow_run` | `Scheduled Validation Watchdog` | [41161431618](https://github.com/rustfs/rustfs/actions/runs/41161431618) | independent filter/monitor path executed against the registry |
|
||||
| `merge_group` | first queued candidate after queue enablement | see below | live evidence attached after the first queue entry completes |
|
||||
| Documentation-only PR | PR #9088, this document's own PR | [41162871854](https://github.com/rustfs/rustfs/actions/runs/41162871854) | `docs` mode: `Select CI scope` recorded `mode: docs`; `Test and Lint` accepted in 8 s with every expensive job required **skipped** |
|
||||
| Isolated E2E PR | PR #9089, comment-only change under `crates/e2e_test/src/` | [41163203959](https://github.com/rustfs/rustfs/actions/runs/41163203959) | `e2e` mode: Connect boundary, serial ILM, rio-v2/protocol lanes and real io_uring required **skipped**; workspace build, E2E and both S3 lanes retained |
|
||||
| Documentation-only PR | this document's own PR | see below | `docs` mode: expensive jobs required **skipped** |
|
||||
| Isolated E2E PR | drill branch confined to `crates/e2e_test/src/` | see below | `e2e` mode: Connect/ILM/protocol lanes required **skipped** |
|
||||
|
||||
## Repeating a drill
|
||||
|
||||
|
||||
+13
-35
@@ -18,7 +18,7 @@ Every PR enters `ci.yml`. The `classify-changes` job uses the base revision of `
|
||||
|
||||
PRs changing only Rust sources under `crates/e2e_test/src/` and known E2E selection digests (optionally with those documentation paths) use the `e2e` scope. They keep workspace validation, the server build, E2E smoke, and both S3 lanes; independent workflows retain their path selection. Connect boundary, serial ILM, optional protocol/rio-v2, and real io_uring jobs are skipped because their sources and test inputs are unchanged. Manifests, fixtures, shared nextest configuration, scripts, production sources, and unknown paths still select `full`. A dependency on `e2e_test` from another workspace member or reachable local dependency disables this shortcut. All non-PR events retain their full selection. The structured planner enforces affected-package and Connect selection from the trusted base policy, retaining fixed core coverage and full validation for shared or unknown inputs. See [verification plans and artifacts](ci-verification-pipeline.md) for identity, rollout and queue-switch contracts.
|
||||
|
||||
Specialized boundary, hardware and alternate-feature lanes run only on the scheduled (nightly) and manual full-validation runs: `Connect Feature Boundaries`, `io_uring Integration (real)`, and the rio-v2, Swift, SFTP and HTTP/3 feature lanes. They do not gate a merge. The nightly run reports their regressions within a day, and repeating these builds on every PR revision and merge group bought no earlier signal worth the runner occupancy (Connect alone measured 53.5 median minutes per run and occasionally set the wall). PRs, merge groups and main pushes run the default workspace and retain ILM, S3, E2E smoke, the short core contract and the debug build. The aggregate requires the specialized lanes to be skipped on pull-request, merge-group and push events and successful on scheduled and manual events. For changes to these features, dispatch `ci.yml` against the candidate branch before merging when that signal is needed, including PRs targeting `gateway/integration`, which has no automatic push trigger in this workflow. Scheduled results supplement the main baseline and do not replace it.
|
||||
Alternate-feature unit tests and Clippy for rio-v2, Swift, SFTP, and HTTP/3 run on merge groups and on scheduled (nightly) or manual validation. PRs and main pushes run the default workspace and retain ILM, S3, real io_uring, and Connect security boundaries in their applicable scopes. The aggregate requires the alternate-feature jobs to be skipped on PRs and main pushes and successful on merge-group, scheduled, and manual events. This avoids repeating four feature builds on every PR revision and on every merge; feature-specific regressions are detected by the nightly run when the merge queue is not used. For changes to these features, dispatch `ci.yml` against the candidate branch before merging when that signal is needed, including PRs targeting `gateway/integration`, which has no automatic push trigger in this workflow. Scheduled results supplement the main baseline and do not replace it.
|
||||
|
||||
Workspace Clippy runs on every event with a selected workspace Rust job, pull requests included, in `Workspace Lint and Build`. That job also creates the exact test archive and runs doctests; `Workspace tests` consumes the archive without rebuilding it. `cache-warm.yml` warms Clippy's check artifacts under `ci-dev` with the same invocation to reuse compatible dependency artifacts. `make pre-commit` does not run Clippy. Select local Clippy targets, features, and documentation exemptions from the root `AGENTS.md` verification tier; the workspace command below reproduces the CI lane and is not a universal pre-push requirement.
|
||||
|
||||
@@ -50,13 +50,13 @@ The four site-replication state-writer concurrency proofs reserve the available
|
||||
| Event | Check name | Workflow / job | Merge status | Reproduce |
|
||||
|---|---|---|---|---|
|
||||
| PR, non-doc change | `Quick Checks` | `ci.yml` `quick-checks` | Required | `make pre-commit` |
|
||||
| PR, non-doc change, and every non-PR event | `Workspace Lint and Build`, `Workspace tests` | `ci.yml` `workspace-lint-and-build` / `workspace-tests` | Via aggregate | `cargo clippy --all-targets -- -D warnings`; `cargo nextest run --profile ci --all --exclude e2e_test`; `cargo test --all --exclude e2e_test --doc`; migration evidence check described below |
|
||||
| PR, non-doc change, and every non-PR event | `Workspace Lint and Build`, `Workspace tests` | `ci.yml` `test-and-lint` / `workspace-tests` | Via aggregate | `cargo clippy --all-targets -- -D warnings`; `cargo nextest run --profile ci --all --exclude e2e_test`; `cargo test --all --exclude e2e_test --doc`; migration evidence check described below |
|
||||
| PR, non-doc change | `Typos` | `ci.yml` `typos` | Via aggregate | `typos` |
|
||||
| Full selection | `Workspace tests` (serial ILM step) | `ci.yml` `workspace-tests` | Via aggregate | exact `Run ignored ILM integration tests serially` command in the job |
|
||||
| Nightly schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo nextest run` with the job's `--features` |
|
||||
| Nightly schedule, manual dispatch, full selection | `Connect Feature Boundaries` | `ci.yml` `connect-feature-boundaries` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail; `scripts/check_offline_enrollment_e2e.sh` |
|
||||
| Merge group, nightly schedule, manual dispatch | `Test and Lint (rio-v2)`, `Test and Lint (swift)`, `Test and Lint (sftp)`, `Test and Lint (http3)` | `ci.yml` `test-and-lint-rio-v2`, `test-and-lint-protocols` | Via aggregate on these events; skipped on PRs and main pushes | `cargo nextest run` with the job's `--features` |
|
||||
| PR, full selection | `Connect Feature Boundaries` | `ci.yml` `connect-feature-boundaries` | Via aggregate | `cargo test -p rustfs --test connect_registration --features connect-e2e-short-credentials`; `cargo check -p rustfs --release --features connect-e2e-short-credentials` must fail; `scripts/check_offline_enrollment_e2e.sh` |
|
||||
| PR, non-doc change | `Build RustFS Debug Binary` | `ci.yml` `build-rustfs-debug-binary` | Via aggregate; prerequisite for black-box jobs | `python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks` (binary plus its `rustfs.e2e.json` sidecar) |
|
||||
| Nightly schedule, manual dispatch, full selection | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Via aggregate on these events; skipped on PRs, merge groups and main pushes | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
|
||||
| PR, full selection | `io_uring Integration (real)` | `ci.yml` `uring-integration` | Via aggregate | `cargo test -p rustfs-ecstore --lib uring_ -- --test-threads=1 --nocapture` |
|
||||
| PR, non-doc change | `End-to-End Tests` | `ci.yml` `e2e-tests` | Via aggregate | `python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-smoke -p e2e_test`, then `./scripts/e2e-run.sh ./target/debug/rustfs <data-dir>` under the same wrapper; membership guards `scripts/check_test_wiring.py --check-profile e2e-smoke <listing.json>` and `scripts/check_security_smoke_count.sh check <listing.json>` |
|
||||
| PR, non-doc change | `Core contract` | `ci.yml` `core-contract` | Via aggregate | existing write/read, anonymous denial and restart-byte tests via `core-contract` profile and the E2E archive |
|
||||
| PR, non-doc change | `S3 Implemented Tests` | `ci.yml` `s3-implemented-tests` | Via aggregate | build `rustfs`, then `scripts/s3-tests/run.sh` with the job's `DEPLOY_MODE` / `TEST_MODE` / `MAXFAIL` env |
|
||||
@@ -117,40 +117,18 @@ Functional-chain health runs on a hosted runner independently of the lab fleet.
|
||||
|
||||
## Release validation
|
||||
|
||||
Post-merge and tag-driven; not a substitute for a PR gate. Every public publication exit consumes the release certification described here; this certification path is separate from the reusable CI artifact contract in [verification plans](ci-verification-pipeline.md), whose archives expire after one day and are never used as release certification.
|
||||
|
||||
The versioned plan is `.config/release-validation-plan.json` (`schema_version=1`): release types, target OS/arch/ABI, supported features, a fixed harness SHA per lane, required case IDs, environment capabilities, timeouts, applicability conditions, and an explicit `upgrade_sources` list carrying the real artifact digests of the supported N-1/N-2 sources plus a stated rollback basis — compatibility and rollback are never invented or defaulted. `scripts/release/release_gate.py validate` expands and freezes `plan_sha256` and the required case set before reading results, then binds that plan, a `candidate.json` (product 40-character SHA, version, release profile, features, immutable deliverable locations with SHA256 and wrapped-binary SHA256, image digest) and the shared G01 evidence-envelope set (rustfs/backlog#2658) of one parent candidate batch, and writes machine JSON, JUnit and a summary. Within a batch the unique key is `(lane, platform{os,arch,abi}, case_id)`; the top-level run/attempt is the parent batch and the nested workflow run/attempt is the child workflow identity, and both are validated separately (workflow SHA and product build SHA are bound independently and need not be equal). Missing, duplicate or extra cases, stale attempts, wrong digests, wrong platforms, wrong harness SHAs, zero tests, cancellation, timeout, untrusted sources and any non-PASS required item block with classified errors; only a complete legal success returns 0. `verify-promotion` re-checks the exact bytes about to be published against the passed list and fails on a swapped package or an edited plan after verification. preview, prerelease and stable all require an explicit plan; preview may use the plan's smaller pre-declared set but never bypasses artifact identity and startup checks. development artifacts are explicitly uncertified and never touch official or latest channels.
|
||||
Post-merge and tag-driven; not a substitute for a PR gate.
|
||||
|
||||
| Trigger | Workflow / job | Result |
|
||||
|---|---|---|
|
||||
| Nightly schedule, dispatch | `build.yml` `build-rustfs` (development build over the full platform matrix) | build artifacts and the uncertified dev download channel; no release publication |
|
||||
| Valid release or preview tag | `build.yml` `build-rustfs`, `release-certification`, `create-release`, `upload-release-assets`, `publish-release`, `promote-release-artifacts` | staged binaries, draft release, checksummed assets, then certified publish and promotion of the same bytes |
|
||||
| Successful non-preview release-tag build (`workflow_run`) | `docker.yml` `release-certification`, `build-docker`, `scan-docker-image` | multi-architecture images staged, verified against the certification and promoted, plus vulnerability report |
|
||||
| Successful release-tag build (`workflow_run`) | `package.yml` `release-certification`, `package`, `promote-packages` | DEB/RPM packages with wrapped-binary verification, staged then promoted with checksums |
|
||||
| Successful non-preview release-tag build (`workflow_run`) | `helm-package.yml` `release-certification`, `build-helm-package`, `publish-helm-package` | versioned chart and repository index, only over a certified image promotion record |
|
||||
| Nightly schedule, dispatch | `build.yml` `build-rustfs` (development build over the full platform matrix) | build artifacts and the dev download channel; no release publication |
|
||||
| Valid release or preview tag | `build.yml` `build-rustfs`, `create-release`, `upload-release-assets`, `publish-release` | draft release, checksummed assets, publish |
|
||||
| Successful non-preview release-tag build (`workflow_run`) | `docker.yml` `build-docker`, `scan-docker-image` | multi-architecture images and vulnerability report |
|
||||
| Successful release-tag build (`workflow_run`) | `package.yml` `package` | DEB/RPM packages and checksums uploaded to the release |
|
||||
| Successful non-preview release-tag build (`workflow_run`) | `helm-package.yml` `build-helm-package`, `publish-helm-package` | versioned chart and repository index |
|
||||
| Final tag's release published | `build.yml` `cleanup-preview-releases` | deletes every `<target>-preview.<N>` Release for that target; the tags are kept |
|
||||
|
||||
Publication exits and their gate:
|
||||
|
||||
| Exit | Writer | Certification consumed |
|
||||
|---|---|---|
|
||||
| GitHub Release draft → public | `build.yml` `publish-release` | `release-certification` success and per-asset `verify-promotion` |
|
||||
| Official R2 binary path `artifacts/rustfs/release/` | `build.yml` `promote-release-artifacts` | `release-certification` plus `verify-promotion`; the build job itself only writes `artifacts/rustfs/staging/<run>/<attempt>/` |
|
||||
| `latest.json` version pointer | `build.yml` `update-latest-version` | transitively through `publish-release` |
|
||||
| Image channels (Docker Hub, GHCR, Quay tags) | `docker.yml` `build-docker` promotion step | certified input digests, staged image binary identity and `verify-promotion`, then promotion of the same digest with `imagetools create` |
|
||||
| Official R2 package paths `packages/release` and `packages/latest` | `package.yml` `promote-packages` | `release-certification` and wrapped-binary `verify-promotion`; the package job only stages |
|
||||
| GitHub release package assets | `package.yml` `package` | `release-certification` and wrapped-binary `verify-promotion` before upload |
|
||||
| Helm chart repository (`rustfs/helm` push) | `helm-package.yml` `publish-helm-package` | `release-certification` plus the Docker workflow's image-promotion record bound to the same candidate |
|
||||
|
||||
Downstream workflows fail closed when the trusted certification is absent: a manual dispatch without certification fails and pushes no official tag. Only results from the trusted verification sources listed in the plan (currently the `rustfs/s3chaos` `release-gate.yml` envelope runs) carry release authority; a PR-uploaded JSON set cannot certify a release. Retries reuse the same candidate, only the staged bytes are promoted (no rebuild-after-verify), and no downstream step re-resolves a mutable `latest`.
|
||||
|
||||
Use an exact preview tag for an end-to-end release rehearsal. Manual dispatches are backfill/debug paths and do not prove the automatic `workflow_run` chain. The committed fixtures under `testdata/release-gate/` and the commands below are contract rehearsals of this orchestration; they are explicitly not product release evidence and not live verifier results.
|
||||
|
||||
```bash
|
||||
python3 -m unittest discover -s scripts/release -p 'test_release_gate.py'
|
||||
python3 scripts/release/release_gate.py validate --plan testdata/release-gate/plan.json --candidate testdata/release-gate/candidate.json --results testdata/release-gate/pass --output target/release-gate-check
|
||||
bash scripts/security/check_preview_release_workflow.sh
|
||||
```
|
||||
Use an exact preview tag for an end-to-end release rehearsal. Manual dispatches are backfill/debug paths and do not prove the automatic `workflow_run` chain.
|
||||
|
||||
## Change checklist
|
||||
|
||||
@@ -158,7 +136,7 @@ Update this file in the same PR when a job or check name changes, a workflow gai
|
||||
|
||||
## ECStore invariant selection
|
||||
|
||||
The existing `ci.yml` `workspace-tests` job runs the ordinary ECStore and filemeta tests. After that run, `scripts/check_test_wiring.py --check-core` checks the same nextest profile and package selection against `.config/ecstore-required-tests.json`. Every named test must exist, match the filter, and be non-ignored; the job also requires a nonempty JUnit report. This checks membership without running the tests twice. `core-test-listing.json`, JUnit, and the run log are retained in the existing `artifacts/workspace` evidence directory.
|
||||
The existing `ci.yml` test-and-lint job runs the ordinary ECStore and filemeta tests. After that run, `scripts/check_test_wiring.py --check-core` checks the same nextest profile and package selection against `.config/ecstore-required-tests.json`. Every named test must exist, match the filter, and be non-ignored; the job also requires a nonempty JUnit report. This checks membership without running the tests twice. `core-test-listing.json`, JUnit, and the run log are retained in the existing test-and-lint artifact.
|
||||
|
||||
The manifest records a minimum set of invariants: write quorum, metadata rollback, stale-writer lock loss, plaintext Range content, multipart cancellation, hiding uncommitted LIST versions, real MinIO metadata, corrupt part arrays, and the shared on-demand-migration source-backend contract for each provider dialect (S3, Azure, native GCS). The three contract entries live in the `rustfs` suite and reach the lane through that package's default features, so dropping `gcs` from `rustfs`'s defaults fails this check instead of silently deselecting the GCS contract (rustfs/backlog#2323). Renaming or moving a required test must update the manifest in the same change after checking the compiled listing. Extend this list as new deterministic regressions land; it is not a claim that all storage invariants are covered.
|
||||
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
# CI timing sample 2026-10-31 (backlog #2963 CI01 measurement)
|
||||
|
||||
Operational measurement of ordinary-code PR feedback and runner occupancy against the CI01 experimental targets (feedback median 15–20 min, P95 ≤ 30 min, paid occupancy median 50–60 runner minutes). Methodology: [ci-timing.md](ci-timing.md) and [ci-resource-accounting.md](ci-resource-accounting.md). Raw snapshots: `/tmp/ci-timing.json` (limit 30 reproduction), `/tmp/ci-timing-200.json` (analysis sample), `/tmp/ci-acc-w{1,3}.json`. Nothing in this report is an acceptance result; the targets are **not met** by the measured sample.
|
||||
|
||||
## Sample window and n
|
||||
|
||||
| Item | Value |
|
||||
|---|---|
|
||||
| Collection time | 2026-10-31 ~02:50–03:15 UTC |
|
||||
| Window | completed `pull_request` runs of `ci.yml` created in the last 7 days |
|
||||
| Runs sampled | 30 (reproduction) and 200 (analysis, script max) |
|
||||
| Successful-code runs (classifier) | 1 of 30; **39 of 200** |
|
||||
| Other outcomes in 200-run sample | 68 cancelled (34 %), 36 failed, remainder non-code greens/closure handlers |
|
||||
| Lane shapes in sample | 24 combined-lane runs (`Workspace Test and Lint`), 15 split-lane runs (`Workspace Lint and Build` + `Workspace tests` + `Core contract`) |
|
||||
|
||||
The limit-30 reproduction reports median wall 76.85 min and median job-sum 133.33 runner-min on n=1 (per methodology, an absent/small median is not zero). All medians/P95 below are from the n=39 analysis sample. Two of the 39 are rerun continuations (run_attempt > 1, wall 7.0 and 22.6 min re-executing only failed jobs under the rerun-start convention); they do not move the median.
|
||||
|
||||
## Measured feedback and occupancy (ordinary successful code runs, n=39)
|
||||
|
||||
| Metric | Median | P90 | P95 | Max |
|
||||
|---|---|---|---|---|
|
||||
| Wall feedback (created/rerun-start → last job end) | **76.3 min** | 94.5 | **97.0** | 103.2 |
|
||||
| Runner-minute job sum per run | **179.5** | 206.5 | **213.6** | 220.4 |
|
||||
|
||||
Independent accounting cross-check (`scripts/ci/actions_cost_report.py`, 2026-10-30T00–12Z window, 547 attempts, collection **incomplete** — see gaps): successful-code PR feedback median 88.34 min (n=12), runner_sum median 199.68 min (n=12). A complete half-day window (2026-10-31T00–12Z, 78 attempts) shows total known runner occupancy 1379.72 min with 42 jobs of unknown occupancy; queue wait summed 28.43 min, so queueing is not the dominant cost.
|
||||
|
||||
## Per-job critical path breakdown
|
||||
|
||||
Execution minutes across the 39 runs (known only):
|
||||
|
||||
| Job | n | Median | P95 |
|
||||
|---|---|---|---|
|
||||
| Workspace Test and Lint (combined lane) | 24 | 63.6 | 73.9 |
|
||||
| Connect Feature Boundaries | 38 | 53.5 | 70.4 |
|
||||
| Workspace Lint and Build (split lane) | 15 | 43.3 | 56.5 |
|
||||
| Workspace tests (split lane) | 15 | 29.2 | 43.0 |
|
||||
| Build RustFS Debug Binary | 39 | 18.5 | 24.9 |
|
||||
| End-to-End Tests | 39 | 11.4 | 16.7 |
|
||||
| S3 Implemented Tests | 39 | 9.8 | 11.1 |
|
||||
| io_uring Integration (real) | 39 | 7.7 | 8.8 |
|
||||
| Quick Checks | 39 | 7.2 | 9.2 |
|
||||
|
||||
Per-run chain timeline (start = attempt start; see `/tmp/per-run-jobs.json`):
|
||||
|
||||
- Split lane (15 runs): `Quick Checks` (~7–10) → `Workspace Lint and Build` (ends ~46–67) → `Workspace tests` (ends ~71–103) → shard evidence → aggregator `Test and Lint`/`required-checks`. The lint→tests chain sets the wall in most split runs.
|
||||
- Combined lane (24 runs): `Quick Checks` → `Workspace Test and Lint` alone runs 63–86 min and sets the wall.
|
||||
- `Connect Feature Boundaries` (median 53.5, P95 70.4) runs parallel to the lint chain and occasionally becomes the tail (e.g. runs 41021787700, 40750122159: connect end within 0.2 min of wall).
|
||||
- `Build RustFS Debug Binary` → E2E/Core-contract/S3 chain finishes with 30–60 min of slack in every sampled run.
|
||||
- The required-check aggregator `Test and Lint` (`required-checks`, `ci_gate.py verify`) finishes last by construction (~0.2 min).
|
||||
|
||||
## Against the CI01 targets
|
||||
|
||||
| Target | Measured | Verdict |
|
||||
|---|---|---|
|
||||
| Feedback median 15–20 min | 76.3 min (n=39) | **not met** (~3.8–5× over) |
|
||||
| Feedback P95 ≤ 30 min | 97.0 min | **not met** (~3.2× over) |
|
||||
| Occupancy median 50–60 runner-min | 179.5 runner-min | **not met** (~3–3.6× over) |
|
||||
|
||||
These are the experimental-target measurements. The full merge-lifecycle cost is separate and was not established here (the accounting PR-allocation buckets were empty in the partial window; see gaps).
|
||||
|
||||
## Bottlenecks (honest list)
|
||||
|
||||
1. **Workspace lint+test compile/test chain (critical path):** combined `Workspace Test and Lint` median 63.6 min, or split `Workspace Lint and Build` 43.3 + `Workspace tests` 29.2 ≈ 72.5 min after Quick Checks. This is the wall clock.
|
||||
2. **Connect Feature Boundaries (parallel near-critical):** median 53.5 min on its own lane with `CARGO_BUILD_JOBS: "2"` limiting build concurrency on the credential-test step; becomes the tail when the lint chain is fast.
|
||||
3. **Duplicate compilation across lanes:** `Workspace Lint and Build` builds workspace test binaries; `Build RustFS Debug Binary` separately builds the debug binary + E2E binaries on another runner; Connect builds again. `cache-save-if: 'false'` on the Connect lane means that lane never warms the shared `ci-dev` cache.
|
||||
4. Runner queueing is small (28.43 queue-min vs 1379.72 runner-min in the complete half-day window) — not the bottleneck.
|
||||
|
||||
## Gaps and caveats
|
||||
|
||||
- Accounting windows: 2026-10-30T00–12Z collected 547 attempts but is flagged incomplete; 2026-10-30T12–24Z collection did not finish (timeout) and has no report; 2026-10-31T12–24Z was empty at collection time. Occupancy figures are per-window partial sums and must not be summed as a bill.
|
||||
- No CI workflow or selection change was made in this pass; strict gate semantics unchanged. No savings are claimed. Candidate fixes (build-artifact reuse between lanes, enabling cache save on the Connect lane, finer affected-selection granularity in shadow mode per ci-timing.md) are **unmeasured proposals**, not results.
|
||||
- Timing data alone does not establish functional coverage or escaped-regression rate; the 15–20 min target remains an experiment that currently misses by a wide margin.
|
||||
@@ -32,7 +32,7 @@ Periodic and manual suites use the responsibility registry and retain the existi
|
||||
|
||||
## Merge-queue preparation and rollback
|
||||
|
||||
The merge-group event plans and validates its actual combined SHA/tree with the default workspace matrix: workspace lint and tests, the short core contract, the debug build, E2E smoke, both S3 lanes and the full merge-gate E2E. The specialized boundary, hardware and alternate-feature lanes (Connect Feature Boundaries, real io_uring, rio-v2, Swift, SFTP, HTTP/3) run only in the scheduled and manual full-validation runs and are required to be skipped here. Main pushes keep complete validation; no reuse is inferred from tree equality because commit identity and build inputs can matter. CLA remains an independent queue-entry requirement.
|
||||
The merge-group event plans and validates its actual combined SHA/tree with the full Linux and feature configuration matrix. Main pushes keep complete validation; no reuse is inferred from tree equality because commit identity and build inputs can matter. CLA remains an independent queue-entry requirement.
|
||||
|
||||
`scripts/ci_merge_queue.py` prepares a bounded queue proposal from a live ruleset snapshot without calling a mutation API. It preserves all existing checks, reviews, bypass actors and branch conditions, adds an `ALLGREEN` squash queue with at most two concurrent builds/two merged entries, and produces the original rollback payload. Its timeout accommodates the existing full gate while measurements are collected. API fields follow [GitHub's ruleset contract](https://docs.github.com/en/rest/repos/rules).
|
||||
|
||||
|
||||
@@ -142,7 +142,7 @@ filtered and malformed listing cases, and missing or altered required fixtures.
|
||||
Those cases must fail; an optional legacy fixture test that returns early is
|
||||
not evidence for a required fixture.
|
||||
|
||||
The `artifacts/workspace` evidence directory contains the native log, listing, JUnit and
|
||||
The `test-and-lint` artifact contains the native log, listing, JUnit and
|
||||
execution receipt. A native test that reopens a disk in the same process does
|
||||
not establish process-crash or power-loss durability.
|
||||
|
||||
|
||||
@@ -50,17 +50,23 @@ pub struct DisableTableBucketHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for DisableTableBucketHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
async fn call(&self, mut req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let warehouse = warehouse_from_params(¶ms)?;
|
||||
let resource = TableCatalogResource::warehouse(&warehouse);
|
||||
authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableBucketAction).await?;
|
||||
let principal = authorize_table_catalog_resource_request(&req, &resource, AdminAction::SetTableBucketAction).await?;
|
||||
let cleanup = table_bucket_cleanup_requested_from_query(&req.uri)?;
|
||||
let app_context = runtime_sources::app_context_from_extensions(&req.extensions);
|
||||
let backend = table_catalog_backend_from_extensions(&req.extensions)?;
|
||||
let object_store = runtime_sources::object_store_from_req(&req)
|
||||
.ok_or_else(|| table_catalog_internal_error("request object store is not initialized"))?;
|
||||
let store = table_catalog_store_from_backend(backend.clone())?;
|
||||
let publication = TableCommitObjectBackend::preauthorized(backend);
|
||||
let response = disable_table_bucket_response(&store, &publication, object_store.as_ref(), &warehouse).await?;
|
||||
install_table_catalog_s3_request_info(&mut req, &principal)?;
|
||||
let publication = TableCommitObjectBackend::for_request(backend, req);
|
||||
let response = if cleanup {
|
||||
cleanup_empty_table_bucket_response(&store, &publication, object_store.as_ref(), &warehouse).await?
|
||||
} else {
|
||||
disable_table_bucket_response(&store, &publication, object_store.as_ref(), &warehouse).await?
|
||||
};
|
||||
crate::app::bucket_usecase::notify_table_bucket_metadata_reload(warehouse.clone(), "disable table bucket", app_context)
|
||||
.await;
|
||||
build_json_response(StatusCode::OK, &response)
|
||||
|
||||
@@ -2092,17 +2092,25 @@ fn warehouse_from_config_query(uri: &http::Uri) -> S3Result<Option<String>> {
|
||||
}
|
||||
|
||||
fn rest_purge_requested_from_query(uri: &http::Uri) -> S3Result<bool> {
|
||||
let mut purge_requested = None;
|
||||
table_catalog_boolean_query(uri, "purgeRequested")
|
||||
}
|
||||
|
||||
fn table_bucket_cleanup_requested_from_query(uri: &http::Uri) -> S3Result<bool> {
|
||||
table_catalog_boolean_query(uri, "cleanup")
|
||||
}
|
||||
|
||||
fn table_catalog_boolean_query(uri: &http::Uri, parameter: &str) -> S3Result<bool> {
|
||||
let mut requested = None;
|
||||
if let Some(query) = uri.query() {
|
||||
for (key, value) in url::form_urlencoded::parse(query.as_bytes()) {
|
||||
if key != "purgeRequested" {
|
||||
if key != parameter {
|
||||
continue;
|
||||
}
|
||||
if purge_requested.is_some() {
|
||||
if requested.is_some() {
|
||||
return Err(iceberg_rest_error(
|
||||
ICEBERG_ERROR_BAD_REQUEST,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"purgeRequested query parameter must not be repeated",
|
||||
format!("{parameter} query parameter must not be repeated"),
|
||||
));
|
||||
}
|
||||
let value = if value.eq_ignore_ascii_case("true") {
|
||||
@@ -2113,13 +2121,13 @@ fn rest_purge_requested_from_query(uri: &http::Uri) -> S3Result<bool> {
|
||||
return Err(iceberg_rest_error(
|
||||
ICEBERG_ERROR_BAD_REQUEST,
|
||||
StatusCode::BAD_REQUEST,
|
||||
"purgeRequested query parameter must be true or false",
|
||||
format!("{parameter} query parameter must be true or false"),
|
||||
));
|
||||
};
|
||||
purge_requested = Some(value);
|
||||
requested = Some(value);
|
||||
}
|
||||
}
|
||||
Ok(purge_requested.unwrap_or(false))
|
||||
Ok(requested.unwrap_or(false))
|
||||
}
|
||||
|
||||
fn rest_table_snapshot_selection_from_query(uri: &http::Uri) -> S3Result<RestTableSnapshotSelection> {
|
||||
@@ -2417,9 +2425,10 @@ fn table_catalog_backend_from_extensions(
|
||||
) -> S3Result<crate::table_catalog::EcStoreTableCatalogObjectBackend<ECStore>> {
|
||||
let context = runtime_sources::app_context_from_extensions(extensions)
|
||||
.ok_or_else(|| table_catalog_internal_error("request application context is not initialized"))?;
|
||||
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
Ok(crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_runtimes(
|
||||
context.object_store(),
|
||||
context.table_catalog_strong_runtime(),
|
||||
context.table_catalog_registration_cache(),
|
||||
))
|
||||
}
|
||||
|
||||
@@ -2572,6 +2581,61 @@ where
|
||||
table_bucket_response(store, bucket, true).await
|
||||
}
|
||||
|
||||
async fn cleanup_empty_table_bucket_response<B>(
|
||||
store: &crate::table_catalog::ConfiguredTableCatalogStore<B>,
|
||||
publication: &TableCommitObjectBackend<B>,
|
||||
object_store: &ECStore,
|
||||
bucket: &str,
|
||||
) -> S3Result<TableBucketResponse>
|
||||
where
|
||||
B: crate::table_catalog::TableCatalogObjectBackend,
|
||||
{
|
||||
if !publication.publication_fence_fleet_confirmed {
|
||||
return Err(catalog_store_error(crate::table_catalog::TableCatalogStoreError::Conflict(
|
||||
"empty table bucket cleanup requires publication fences on every cluster node".to_string(),
|
||||
)));
|
||||
}
|
||||
crate::table_catalog::TableCommitPublication::begin_table_bucket(publication, bucket)
|
||||
.await
|
||||
.map_err(catalog_store_error)?;
|
||||
let _publication_completion = crate::table_catalog::TableCommitPublicationCompletion::new(publication);
|
||||
let incarnation = object_store
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.map_err(|err| ApiError::other(format!("failed to read table bucket identity: {err}")))?;
|
||||
let prepared = store
|
||||
.prepare_empty_table_bucket_cleanup(table_bucket_entry_from_metadata_marker(bucket), publication)
|
||||
.await
|
||||
.map_err(catalog_store_error);
|
||||
let mut cleanup = publication.finish(prepared).await?;
|
||||
for object in cleanup.metadata_objects() {
|
||||
if let Err(err) = publication.authorize(bucket, object, S3Action::DeleteObjectAction).await {
|
||||
return publication.finish(Err(catalog_store_error(err))).await;
|
||||
}
|
||||
}
|
||||
store
|
||||
.execute_empty_table_bucket_cleanup(&mut cleanup, incarnation)
|
||||
.await
|
||||
.map_err(catalog_store_error)?;
|
||||
let fence = cleanup.mutation_fence();
|
||||
object_store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
bucket,
|
||||
crate::table_catalog::TABLE_BUCKET_MARKER_CONFIG,
|
||||
Vec::new(),
|
||||
incarnation,
|
||||
|| {
|
||||
fence
|
||||
.ensure_held()
|
||||
.map_err(|err| crate::admin::storage_api::error::StorageError::other(err.to_string()))
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|err| ApiError::other(format!("failed to disable table bucket: {err}")))?;
|
||||
drop(cleanup);
|
||||
table_bucket_response(store, bucket, false).await
|
||||
}
|
||||
|
||||
async fn disable_table_bucket_response<S, B>(
|
||||
store: &S,
|
||||
publication: &TableCommitObjectBackend<B>,
|
||||
|
||||
@@ -87,6 +87,15 @@ async fn call_load_table_handler(request: S3Request<Body>) -> S3Result<S3Respons
|
||||
RestLoadTableHandler {}.call(request, matched.params).await
|
||||
}
|
||||
|
||||
async fn call_disable_table_bucket_handler(request: S3Request<Body>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let mut router = matchit::Router::new();
|
||||
router.insert("/iceberg/v1/buckets/{warehouse}", ()).unwrap();
|
||||
router.insert("/_iceberg/v1/buckets/{warehouse}", ()).unwrap();
|
||||
let path = request.uri.path().to_string();
|
||||
let matched = router.at(&path).unwrap();
|
||||
DisableTableBucketHandler {}.call(request, matched.params).await
|
||||
}
|
||||
|
||||
async fn call_maintenance_worker_handler(request: S3Request<Body>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let mut router = matchit::Router::new();
|
||||
router
|
||||
@@ -1255,6 +1264,28 @@ fn drop_table_purge_query_is_explicit_and_strict() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_bucket_cleanup_query_is_explicit_and_strict() {
|
||||
for (uri, expected) in [
|
||||
("/iceberg/v1/buckets/analytics", false),
|
||||
("/iceberg/v1/buckets/analytics?cleanup=false", false),
|
||||
("/iceberg/v1/buckets/analytics?cleanup=true", true),
|
||||
("/_iceberg/v1/buckets/analytics?cleanup=true", true),
|
||||
] {
|
||||
assert_eq!(table_bucket_cleanup_requested_from_query(&uri.parse().unwrap()).unwrap(), expected);
|
||||
}
|
||||
for query in [
|
||||
"cleanup=",
|
||||
"cleanup=1",
|
||||
"cleanup=yes",
|
||||
"cleanup=true&cleanup=false",
|
||||
"cleanup=true&cleanup=true",
|
||||
] {
|
||||
let uri = format!("/iceberg/v1/buckets/analytics?{query}").parse().unwrap();
|
||||
assert!(table_bucket_cleanup_requested_from_query(&uri).is_err(), "invalid query: {query}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catalog_conflicts_use_operation_specific_iceberg_errors() {
|
||||
let already_exists = catalog_store_already_exists_error(crate::table_catalog::TableCatalogStoreError::Conflict(
|
||||
@@ -1689,9 +1720,7 @@ fn table_bucket_toggle_handlers_reload_peer_metadata_after_publication() {
|
||||
(enable, "enable_table_bucket_response", "enable table bucket"),
|
||||
(disable, "disable_table_bucket_response", "disable table bucket"),
|
||||
] {
|
||||
let response_offset = block
|
||||
.find(&format!("let response = {response}"))
|
||||
.expect("table bucket toggle should publish local state");
|
||||
let response_offset = block.find(response).expect("table bucket toggle should publish local state");
|
||||
let reload = block
|
||||
.find(&format!("notify_table_bucket_metadata_reload(warehouse.clone(), \"{operation}\""))
|
||||
.expect("table bucket toggle should reload peer metadata");
|
||||
@@ -16274,3 +16303,799 @@ async fn disable_table_bucket_preserves_objects_and_can_be_reenabled() {
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
}
|
||||
|
||||
async fn create_dropped_empty_tables_for_cleanup<S, B>(
|
||||
catalog: &S,
|
||||
backend: &B,
|
||||
object_store: &ECStore,
|
||||
bucket: &str,
|
||||
table_count: usize,
|
||||
) -> Vec<String>
|
||||
where
|
||||
S: TableCatalogStore + ?Sized,
|
||||
B: TableCatalogObjectBackend,
|
||||
{
|
||||
object_store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
enable_table_bucket_response(catalog, &TableCommitObjectBackend::trusted(backend.clone()), object_store, bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").unwrap();
|
||||
create_namespace_response(
|
||||
catalog,
|
||||
bucket,
|
||||
CreateNamespaceRequest {
|
||||
namespace: vec!["analytics".to_string()],
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut metadata_objects = Vec::new();
|
||||
for number in 0..table_count {
|
||||
let name = format!("events_{number:02}");
|
||||
let request = serde_json::from_value(serde_json::json!({
|
||||
"name": name,
|
||||
"schema": {
|
||||
"type": "struct",
|
||||
"schema-id": 0,
|
||||
"fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
create_table_response(
|
||||
catalog,
|
||||
&TableCommitObjectBackend::trusted(backend.clone()),
|
||||
bucket,
|
||||
&namespace,
|
||||
request,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
metadata_objects.push(
|
||||
catalog
|
||||
.load_table(bucket, &namespace.public_name(), &name)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.metadata_location,
|
||||
);
|
||||
}
|
||||
for number in 0..table_count {
|
||||
catalog
|
||||
.drop_table(bucket, &namespace.public_name(), &format!("events_{number:02}"))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
catalog.drop_namespace(bucket, &namespace.public_name()).await.unwrap();
|
||||
metadata_objects
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn object_catalog_registration_cache_checks_real_records_without_additional_objects() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::{
|
||||
ConfiguredTableCatalogStore, EcStoreTableCatalogObjectBackend, StrongTableCatalogRuntime, TableCatalogBackingMode,
|
||||
TableCatalogObjectPaths, TableCatalogStoreError,
|
||||
};
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("registration-cache-{}", Uuid::new_v4().simple());
|
||||
let backend =
|
||||
EcStoreTableCatalogObjectBackend::new_with_strong_runtime(object_store.clone(), StrongTableCatalogRuntime::default());
|
||||
let catalog = ConfiguredTableCatalogStore::new_for_test(backend.clone(), TableCatalogBackingMode::ObjectBacked);
|
||||
object_store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
enable_table_bucket_response(&catalog, &TableCommitObjectBackend::trusted(backend.clone()), &object_store, &bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let namespace = crate::table_catalog::Namespace::parse("analytics").unwrap();
|
||||
create_namespace_response(
|
||||
&catalog,
|
||||
&bucket,
|
||||
CreateNamespaceRequest {
|
||||
namespace: vec!["analytics".to_string()],
|
||||
properties: BTreeMap::new(),
|
||||
},
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let paths = TableCatalogObjectPaths::default();
|
||||
let bucket_entry = paths.table_bucket_entry_path(&bucket);
|
||||
let catalog_prefix = format!("{}/", bucket_entry.rsplit_once('/').unwrap().0);
|
||||
let mut create_micros = Vec::new();
|
||||
for number in 0..100 {
|
||||
let store = ConfiguredTableCatalogStore::new_for_test(backend.clone(), TableCatalogBackingMode::ObjectBacked);
|
||||
let request = serde_json::from_value(serde_json::json!({
|
||||
"name": format!("events_{number:03}"),
|
||||
"schema": {
|
||||
"type": "struct",
|
||||
"schema-id": 0,
|
||||
"fields": [{"id": 1, "name": "id", "required": true, "type": "long"}]
|
||||
}
|
||||
}))
|
||||
.unwrap();
|
||||
let started = std::time::Instant::now();
|
||||
create_table_response(
|
||||
&store,
|
||||
&TableCommitObjectBackend::trusted(backend.clone()),
|
||||
&bucket,
|
||||
&namespace,
|
||||
request,
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.expect("fresh store instances share the runtime cache and create independent tables");
|
||||
create_micros.push(started.elapsed().as_micros());
|
||||
if matches!(number + 1, 20 | 100) {
|
||||
let before = backend.list_objects(RUSTFS_META_BUCKET, &catalog_prefix).await.unwrap();
|
||||
let cold_backend = EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
StrongTableCatalogRuntime::default(),
|
||||
);
|
||||
let probe = ConfiguredTableCatalogStore::new_for_test(cold_backend, TableCatalogBackingMode::ObjectBacked);
|
||||
let mut duplicate = catalog
|
||||
.load_table(&bucket, "analytics", &format!("events_{number:03}"))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
duplicate.table = "duplicate_probe".to_string();
|
||||
for cache_state in ["cold", "warm"] {
|
||||
let started = std::time::Instant::now();
|
||||
assert!(matches!(
|
||||
probe.create_table(duplicate.clone()).await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
eprintln!(
|
||||
"registration_check tables={} cache={} elapsed_us={}",
|
||||
number + 1,
|
||||
cache_state,
|
||||
started.elapsed().as_micros()
|
||||
);
|
||||
}
|
||||
assert_eq!(before, backend.list_objects(RUSTFS_META_BUCKET, &catalog_prefix).await.unwrap());
|
||||
let mut recent = create_micros[create_micros.len() - 5..].to_vec();
|
||||
recent.sort_unstable();
|
||||
eprintln!("create_table_response tables={} last_five_median_us={}", number + 1, recent[2]);
|
||||
}
|
||||
}
|
||||
assert_eq!(catalog.list_all_tables(&bucket).await.unwrap().len(), 100);
|
||||
let namespace_entry = paths.namespace_entry_path(&bucket, &namespace);
|
||||
let index_prefix = paths.warehouse_index_entries_prefix(&bucket);
|
||||
let objects = backend.list_objects(RUSTFS_META_BUCKET, &catalog_prefix).await.unwrap();
|
||||
assert!(objects.iter().all(|key| {
|
||||
key == &bucket_entry || key == &namespace_entry || key.starts_with(&index_prefix) || key.ends_with("/table-entry.json")
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_reclaims_twenty_dropped_tables_and_allows_bucket_deletion() {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-{}", Uuid::new_v4().simple());
|
||||
let backend = crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
);
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 20).await;
|
||||
assert!(
|
||||
disable_table_bucket_response(
|
||||
&catalog,
|
||||
&TableCommitObjectBackend::trusted(backend.clone()),
|
||||
object_store.as_ref(),
|
||||
&bucket,
|
||||
)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
|
||||
let authorizations = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let publication = TableCommitObjectBackend::test(backend.clone(), authorizations.clone(), None);
|
||||
let result = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket).await;
|
||||
assert!(!publication.finish(result).await.unwrap().enabled);
|
||||
for object in &metadata_objects {
|
||||
assert!(!backend.object_exists(&bucket, object).await.unwrap());
|
||||
let authorized = authorizations.lock().await;
|
||||
assert!(authorized.contains(&(object.clone(), S3Action::GetObjectAction)));
|
||||
assert!(authorized.contains(&(object.clone(), S3Action::DeleteObjectAction)));
|
||||
}
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
let publication = TableCommitObjectBackend::trusted(backend);
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_validates_authorization_and_metadata_before_deleting() {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-check-{}", Uuid::new_v4().simple());
|
||||
let backend = crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
);
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let publication = TableCommitObjectBackend::rolling_upgrade(backend.clone());
|
||||
assert!(
|
||||
cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
|
||||
let authorizations = Arc::new(tokio::sync::Mutex::new(Vec::new()));
|
||||
let denied_object = metadata_objects.last().unwrap().clone();
|
||||
let publication = TableCommitObjectBackend::test(backend.clone(), authorizations, Some(denied_object.clone()));
|
||||
let result = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket).await;
|
||||
assert_eq!(publication.finish(result).await.unwrap_err().code(), &S3ErrorCode::AccessDenied);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
|
||||
let original = backend.read_object(&bucket, &denied_object).await.unwrap().unwrap().data;
|
||||
backend
|
||||
.put_object(
|
||||
&bucket,
|
||||
&denied_object,
|
||||
b"{".to_vec(),
|
||||
crate::table_catalog::TableCatalogPutPrecondition::Any,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
backend
|
||||
.put_object(&bucket, &denied_object, original, crate::table_catalog::TableCatalogPutPrecondition::Any)
|
||||
.await
|
||||
.unwrap();
|
||||
let publication = TableCommitObjectBackend::trusted(backend);
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_handler_enforces_request_object_permissions() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(
|
||||
crate::table_catalog::ENV_TABLE_CATALOG_BACKING,
|
||||
Some(crate::table_catalog::TABLE_CATALOG_BACKING_OBJECT),
|
||||
),
|
||||
(crate::table_catalog::ENV_TABLE_CATALOG_PUBLICATION_FENCE_FLEET_CONFIRMED, Some("true")),
|
||||
],
|
||||
async {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-request-{}", Uuid::new_v4().simple());
|
||||
rustfs_iam::store::object::ObjectStore::new(object_store.clone())
|
||||
.save_iam_config(
|
||||
serde_json::json!({"version": 1}),
|
||||
format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let iam = rustfs_iam::build_iam_sys(object_store.clone()).await.unwrap();
|
||||
let access_key = "cleanup-request-user";
|
||||
let secret_key = "cleanup-request-user-secret";
|
||||
iam.create_user(
|
||||
access_key,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: secret_key.to_string(),
|
||||
policy: None,
|
||||
status: AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let context = Arc::new(AppContext::new(
|
||||
object_store.clone(),
|
||||
Arc::new(RequestIam { handle: iam.clone() }),
|
||||
Arc::new(RequestKms),
|
||||
));
|
||||
assert!(context.publish_action_credentials(rustfs_credentials::Credentials {
|
||||
access_key: "cleanup-request-root".to_string(),
|
||||
secret_key: "cleanup-request-root-secret".to_string(),
|
||||
status: "on".to_string(),
|
||||
..Default::default()
|
||||
}));
|
||||
let backend = crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime(
|
||||
object_store.clone(),
|
||||
context.table_catalog_strong_runtime(),
|
||||
);
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects =
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let request = |prefix: &str| {
|
||||
let mut request = table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
request.method = Method::DELETE;
|
||||
request.uri = format!("{prefix}/v1/buckets/{bucket}?cleanup=true").parse().unwrap();
|
||||
request
|
||||
};
|
||||
let object_resource = format!("arn:aws:s3:::{bucket}/*");
|
||||
let first_metadata_resource = format!("arn:aws:s3:::{bucket}/{}", metadata_objects[0]);
|
||||
for (policy_name, object_statements, allowed) in [
|
||||
("cleanup-request-no-read", Vec::new(), false),
|
||||
(
|
||||
"cleanup-request-partial-delete",
|
||||
vec![
|
||||
serde_json::json!({"Effect":"Allow","Action":["s3:GetObject"],"Resource":[object_resource]}),
|
||||
serde_json::json!({"Effect":"Allow","Action":["s3:DeleteObject"],"Resource":[first_metadata_resource]}),
|
||||
],
|
||||
false,
|
||||
),
|
||||
(
|
||||
"cleanup-request-full",
|
||||
vec![serde_json::json!({
|
||||
"Effect":"Allow",
|
||||
"Action":["s3:GetObject","s3:DeleteObject"],
|
||||
"Resource":[object_resource],
|
||||
})],
|
||||
true,
|
||||
),
|
||||
] {
|
||||
let mut statements = vec![serde_json::json!({"Effect":"Allow","Action":["admin:SetTableBucket"]})];
|
||||
statements.extend(object_statements);
|
||||
let policy = serde_json::to_vec(&serde_json::json!({"Version":"2012-10-17","Statement":statements})).unwrap();
|
||||
iam.set_policy(policy_name, Policy::parse_config(&policy).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
iam.policy_db_set(access_key, UserType::Reg, false, policy_name)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let result = call_disable_table_bucket_handler(request(if allowed { "/_iceberg" } else { "/iceberg" })).await;
|
||||
if allowed {
|
||||
let response = result.expect("request-scoped object permissions should authorize cleanup");
|
||||
assert_eq!(response.output.0, StatusCode::OK);
|
||||
let body: serde_json::Value = serde_json::from_slice(&response.output.1.bytes().unwrap()).unwrap();
|
||||
assert_eq!(body["enabled"], false);
|
||||
for object in &metadata_objects {
|
||||
assert!(!backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
} else {
|
||||
assert_eq!(result.unwrap_err().code(), &S3ErrorCode::AccessDenied);
|
||||
for object in &metadata_objects {
|
||||
assert!(backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
crate::table_catalog::TableCatalogEntryState::Active,
|
||||
);
|
||||
}
|
||||
}
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_retries_after_marker_write_rejection() {
|
||||
use crate::table_catalog::TableCommitPublication;
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-marker-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::default();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
TableCommitPublication::begin_table_bucket(&publication, &bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
let mut cleanup = catalog
|
||||
.prepare_empty_table_bucket_cleanup(table_bucket_entry_from_metadata_marker(&bucket), &publication)
|
||||
.await
|
||||
.unwrap();
|
||||
catalog
|
||||
.execute_empty_table_bucket_cleanup(&mut cleanup, incarnation)
|
||||
.await
|
||||
.unwrap();
|
||||
let marker_result = object_store
|
||||
.update_bucket_metadata_config_for_incarnation_validated(
|
||||
&bucket,
|
||||
crate::table_catalog::TABLE_BUCKET_MARKER_CONFIG,
|
||||
Vec::new(),
|
||||
incarnation,
|
||||
|| {
|
||||
Err(crate::admin::storage_api::error::StorageError::other(
|
||||
"injected marker publication rejection",
|
||||
))
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(marker_result.is_err());
|
||||
drop(cleanup);
|
||||
TableCommitPublication::complete(&publication);
|
||||
for object in &metadata_objects {
|
||||
assert!(!backend.object_exists(&bucket, object).await.unwrap());
|
||||
}
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
crate::table_catalog::TableCatalogEntryState::Deleted,
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend);
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_reads_incarnation_after_publication_wait() {
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-recreated-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::default();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 0).await;
|
||||
let original_incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
let publication_lock = crate::table_catalog::default_table_bucket_publication_lock_path();
|
||||
let guard = backend.acquire_write_lock(&bucket, &publication_lock).await.unwrap();
|
||||
let attempts = backend.write_lock_acquisition_count(&bucket, &publication_lock).await;
|
||||
let cleanup_bucket = bucket.clone();
|
||||
let cleanup_store = object_store.clone();
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
let cleanup = tokio::spawn(async move {
|
||||
cleanup_empty_table_bucket_response(&catalog, &publication, cleanup_store.as_ref(), &cleanup_bucket).await
|
||||
});
|
||||
tokio::time::timeout(StdDuration::from_secs(5), async {
|
||||
while backend.write_lock_acquisition_count(&bucket, &publication_lock).await == attempts {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("cleanup should wait for its publication lock");
|
||||
|
||||
// Replace the real bucket while the in-memory publication backend holds cleanup at its lock boundary.
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
object_store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
enable_table_bucket_marker(object_store.as_ref(), &bucket).await.unwrap();
|
||||
let replacement_incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
assert_ne!(replacement_incarnation, original_incarnation);
|
||||
drop(guard);
|
||||
let response = tokio::time::timeout(StdDuration::from_secs(10), cleanup)
|
||||
.await
|
||||
.expect("cleanup should resume after publication becomes available")
|
||||
.unwrap()
|
||||
.expect("cleanup must use the incarnation observed while holding its publication lock");
|
||||
assert!(!response.enabled);
|
||||
assert_eq!(
|
||||
object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap(),
|
||||
replacement_incarnation
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_retries_metadata_delete_failures_without_removing_protection() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::{TableCatalogEntryState, TableCatalogObjectPaths};
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
for fail_after_delete in [false, true] {
|
||||
let bucket = format!("cleanup-retry-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let mut metadata_objects =
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 3).await;
|
||||
metadata_objects.sort();
|
||||
let paths = TableCatalogObjectPaths::default();
|
||||
let index_prefix = paths.warehouse_index_entries_prefix(&bucket);
|
||||
let indexes = backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap();
|
||||
assert!(indexes.len() >= metadata_objects.len());
|
||||
if fail_after_delete {
|
||||
backend.fail_after_next_delete(&bucket, &metadata_objects[1]).await;
|
||||
} else {
|
||||
backend.fail_delete_attempt(&bucket, &metadata_objects[1], 1).await;
|
||||
}
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
let error = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("injected"));
|
||||
assert!(!backend.object_exists(&bucket, &metadata_objects[0]).await.unwrap());
|
||||
assert_eq!(backend.object_exists(&bucket, &metadata_objects[1]).await.unwrap(), !fail_after_delete);
|
||||
assert!(backend.object_exists(&bucket, &metadata_objects[2]).await.unwrap());
|
||||
assert_eq!(backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap(), indexes);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Active
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let restarted = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&restarted, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(backend.list_objects(&bucket, "").await.unwrap().is_empty());
|
||||
assert!(
|
||||
backend
|
||||
.list_objects(RUSTFS_META_BUCKET, &index_prefix)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(
|
||||
restarted.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Deleted
|
||||
);
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_retries_partial_warehouse_index_deletion() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::{TableCatalogEntryState, TableCatalogObjectPaths};
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-index-retry-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let index_prefix = TableCatalogObjectPaths::default().warehouse_index_entries_prefix(&bucket);
|
||||
let indexes = backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap();
|
||||
assert!(indexes.len() >= 2);
|
||||
let last_index = indexes.last().unwrap();
|
||||
backend.fail_delete_attempt(RUSTFS_META_BUCKET, last_index, 1).await;
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
let error = cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(error.to_string().contains("injected"));
|
||||
assert!(backend.list_objects(&bucket, "").await.unwrap().is_empty());
|
||||
assert_eq!(
|
||||
backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap(),
|
||||
vec![last_index.clone()]
|
||||
);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Active
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(
|
||||
backend
|
||||
.list_objects(RUSTFS_META_BUCKET, &index_prefix)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_empty()
|
||||
);
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial_test::serial]
|
||||
async fn cleanup_empty_table_bucket_stops_before_delete_when_publication_lease_is_lost() {
|
||||
use crate::admin::storage_api::RUSTFS_META_BUCKET;
|
||||
use crate::table_catalog::test_support::RecordingLeaseClient;
|
||||
use crate::table_catalog::{TableCatalogEntryState, TableCatalogLockGuard, TableCatalogObjectPaths, TableCommitPublication};
|
||||
use rustfs_lock::LockClient;
|
||||
|
||||
let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await;
|
||||
let bucket = format!("cleanup-lease-{}", Uuid::new_v4().simple());
|
||||
let backend = TestTableCatalogObjectBackend::content_addressed();
|
||||
let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test(
|
||||
backend.clone(),
|
||||
crate::table_catalog::TableCatalogBackingMode::ObjectBacked,
|
||||
);
|
||||
let metadata_objects = create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 1).await;
|
||||
let index_prefix = TableCatalogObjectPaths::default().warehouse_index_entries_prefix(&bucket);
|
||||
let indexes = backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap();
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
TableCommitPublication::begin_table_bucket(&publication, &bucket)
|
||||
.await
|
||||
.unwrap();
|
||||
let client = Arc::new(RecordingLeaseClient {
|
||||
inner: rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
)))),
|
||||
acquired: std::sync::Mutex::new(None),
|
||||
});
|
||||
let lock = rustfs_lock::NamespaceLockWrapper::new(
|
||||
rustfs_lock::NamespaceLock::with_clients_and_quorum(
|
||||
"cleanup-publication-fence".to_string(),
|
||||
vec![client.clone() as Arc<dyn LockClient>],
|
||||
1,
|
||||
),
|
||||
rustfs_lock::ObjectKey::new(bucket.as_str(), "cleanup-publication-fence"),
|
||||
"cleanup-publication-owner".to_string(),
|
||||
);
|
||||
let guard = lock.get_read_lock(StdDuration::from_secs(5)).await.unwrap();
|
||||
let signal = guard.lock_lost_signal().unwrap();
|
||||
let lock_id = client.acquired.lock().unwrap().clone().unwrap();
|
||||
publication
|
||||
.publication
|
||||
.lock()
|
||||
.guards
|
||||
.push(TableCatalogLockGuard::test_namespace(guard));
|
||||
let mut cleanup = catalog
|
||||
.prepare_empty_table_bucket_cleanup(table_bucket_entry_from_metadata_marker(&bucket), &publication)
|
||||
.await
|
||||
.unwrap();
|
||||
let incarnation = object_store.bucket_incarnation_id_from_disk(&bucket).await.unwrap();
|
||||
let pause = backend.pause_next_delete(&bucket, &metadata_objects[0]).await;
|
||||
let operation = catalog.execute_empty_table_bucket_cleanup(&mut cleanup, incarnation);
|
||||
let interfere = async {
|
||||
tokio::time::timeout(StdDuration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(client.force_release(&lock_id).await.unwrap());
|
||||
tokio::time::timeout(StdDuration::from_secs(15), signal.notified())
|
||||
.await
|
||||
.expect("cleanup publication must observe lease loss");
|
||||
pause.release();
|
||||
};
|
||||
let (result, ()) = tokio::join!(operation, interfere);
|
||||
assert!(matches!(result, Err(crate::table_catalog::TableCatalogStoreError::Unavailable(_))));
|
||||
drop(cleanup);
|
||||
TableCommitPublication::complete(&publication);
|
||||
assert!(backend.object_exists(&bucket, &metadata_objects[0]).await.unwrap());
|
||||
assert_eq!(backend.list_objects(RUSTFS_META_BUCKET, &index_prefix).await.unwrap(), indexes);
|
||||
assert_eq!(
|
||||
catalog.get_table_bucket(&bucket).await.unwrap().unwrap().state,
|
||||
TableCatalogEntryState::Active
|
||||
);
|
||||
assert!(
|
||||
object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
|
||||
let publication = TableCommitObjectBackend::trusted(backend.clone());
|
||||
assert!(
|
||||
!cleanup_empty_table_bucket_response(&catalog, &publication, object_store.as_ref(), &bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.enabled
|
||||
);
|
||||
assert!(!backend.object_exists(&bucket, &metadata_objects[0]).await.unwrap());
|
||||
assert!(
|
||||
!object_store
|
||||
.get_bucket_metadata(&bucket)
|
||||
.await
|
||||
.unwrap()
|
||||
.table_bucket_enabled()
|
||||
);
|
||||
object_store.delete_bucket(&bucket, &Default::default()).await.unwrap();
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ pub struct AppContext {
|
||||
object_data_cache: Arc<ObjectDataCacheAdapter>,
|
||||
object_traffic_health: Arc<ObjectTrafficHealth>,
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime,
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache,
|
||||
}
|
||||
|
||||
impl AppContext {
|
||||
@@ -127,6 +128,7 @@ impl AppContext {
|
||||
object_data_cache,
|
||||
object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,6 +152,10 @@ impl AppContext {
|
||||
self.table_catalog_strong_runtime.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn table_catalog_registration_cache(&self) -> crate::table_catalog::TableCatalogRegistrationCache {
|
||||
self.table_catalog_registration_cache.clone()
|
||||
}
|
||||
|
||||
pub fn iam(&self) -> Arc<dyn IamInterface> {
|
||||
self.iam.clone()
|
||||
}
|
||||
@@ -360,6 +366,7 @@ impl AppContext {
|
||||
object_data_cache: ObjectDataCacheAdapter::disabled_arc(),
|
||||
object_traffic_health: Arc::new(ObjectTrafficHealth::from_env()),
|
||||
table_catalog_strong_runtime: crate::table_catalog::StrongTableCatalogRuntime::default(),
|
||||
table_catalog_registration_cache: crate::table_catalog::TableCatalogRegistrationCache::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -490,7 +490,8 @@ pub(crate) mod table {
|
||||
|
||||
pub(crate) mod list {
|
||||
pub(crate) use super::super::super::storage_contracts::{
|
||||
ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr, WalkOptions,
|
||||
AuthoritativeListOperations, ListObjectVersionsInfo, ListObjectsV2Info, ListOperations, ObjectInfoOrErr,
|
||||
WalkOptions,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -498,6 +499,10 @@ pub(crate) mod table {
|
||||
pub(crate) use super::super::super::storage_contracts::NamespaceLocking;
|
||||
}
|
||||
|
||||
pub(crate) mod multipart {
|
||||
pub(crate) use super::super::super::storage_contracts::MultipartUploadInspection;
|
||||
}
|
||||
|
||||
pub(crate) mod object {
|
||||
pub(crate) use super::super::super::storage_contracts::{ObjectIO, ObjectOperations};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,342 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// Validates one metadata object; callers must also validate the complete object inventory.
|
||||
pub(crate) fn validate_empty_table_cleanup_metadata(
|
||||
table_bucket: &str,
|
||||
object_key: &str,
|
||||
metadata: &[u8],
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let (namespace, table) = table_identity_from_metadata_object_key(object_key).ok_or_else(|| {
|
||||
TableCatalogStoreError::Invalid("cleanup metadata must use the default table metadata directory".to_string())
|
||||
})?;
|
||||
let metadata = decode_table_metadata_json(object_key, metadata)?;
|
||||
validate_supported_table_metadata(&metadata)?;
|
||||
|
||||
// Future metadata fields may introduce additional object references.
|
||||
for field in metadata.as_object().expect("validated metadata is a JSON object").keys() {
|
||||
if !matches!(
|
||||
field.as_str(),
|
||||
"format-version"
|
||||
| "table-uuid"
|
||||
| "location"
|
||||
| "last-sequence-number"
|
||||
| "last-updated-ms"
|
||||
| "last-column-id"
|
||||
| "schema"
|
||||
| "schemas"
|
||||
| "current-schema-id"
|
||||
| "partition-spec"
|
||||
| "partition-specs"
|
||||
| "default-spec-id"
|
||||
| "last-partition-id"
|
||||
| "properties"
|
||||
| "current-snapshot-id"
|
||||
| "snapshots"
|
||||
| "snapshot-log"
|
||||
| "metadata-log"
|
||||
| "sort-orders"
|
||||
| "default-sort-order-id"
|
||||
| "refs"
|
||||
| "statistics"
|
||||
| "partition-statistics"
|
||||
) {
|
||||
return Err(TableCatalogStoreError::Unsupported(format!(
|
||||
"empty table cleanup does not support metadata field {field}"
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
for field in ["snapshots", "snapshot-log", "statistics", "partition-statistics"] {
|
||||
let Some(value) = metadata.get(field) else {
|
||||
continue;
|
||||
};
|
||||
let entries = value
|
||||
.as_array()
|
||||
.ok_or_else(|| TableCatalogStoreError::Invalid(format!("cleanup metadata field {field} must be an array")))?;
|
||||
if !entries.is_empty() {
|
||||
return Err(TableCatalogStoreError::Conflict(format!(
|
||||
"empty table cleanup requires metadata field {field} to be empty"
|
||||
)));
|
||||
}
|
||||
}
|
||||
if let Some(value) = metadata.get("refs") {
|
||||
let refs = value
|
||||
.as_object()
|
||||
.ok_or_else(|| TableCatalogStoreError::Invalid("cleanup metadata refs must be an object".to_string()))?;
|
||||
if !refs.is_empty() {
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"empty table cleanup requires metadata refs to be empty".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let table_uuid = table_metadata_uuid(&metadata)?;
|
||||
if Uuid::parse_str(table_uuid).ok().filter(|uuid| !uuid.is_nil()).is_none() {
|
||||
return Err(TableCatalogStoreError::Invalid(
|
||||
"cleanup metadata table-uuid must be a non-nil UUID".to_string(),
|
||||
));
|
||||
}
|
||||
let warehouse_prefix = table_warehouse_object_prefix_from_location(table_bucket, table_metadata_location(&metadata)?)?;
|
||||
if warehouse_prefix
|
||||
.strip_prefix("tables/")
|
||||
.and_then(|prefix| prefix.strip_suffix('/'))
|
||||
.and_then(|table_id| Uuid::parse_str(table_id).ok())
|
||||
.filter(|table_id| !table_id.is_nil())
|
||||
.is_none()
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"empty table cleanup requires the default tables/<table-id> warehouse location".to_string(),
|
||||
));
|
||||
}
|
||||
if let Some(entries) = metadata.get("metadata-log").and_then(serde_json::Value::as_array) {
|
||||
for entry in entries {
|
||||
let entry = entry.as_object().expect("validated metadata log entries are JSON objects");
|
||||
if entry
|
||||
.keys()
|
||||
.any(|field| !matches!(field.as_str(), "timestamp-ms" | "metadata-file"))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Unsupported(
|
||||
"empty table cleanup does not support unknown metadata-log fields".to_string(),
|
||||
));
|
||||
}
|
||||
let location = entry["metadata-file"]
|
||||
.as_str()
|
||||
.expect("validated metadata log entries contain metadata-file");
|
||||
table_catalog_object_key_from_location(table_bucket, location)
|
||||
.filter(|location| is_valid_table_metadata_location(&namespace, &table, location))
|
||||
.filter(|location| location != object_key)
|
||||
.ok_or_else(|| {
|
||||
TableCatalogStoreError::Invalid(
|
||||
"cleanup metadata-log must reference another file in the same table metadata directory".to_string(),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::table_catalog::test_support::table_metadata_json;
|
||||
use serde_json::{Value, json};
|
||||
use std::io::Write;
|
||||
|
||||
const BUCKET: &str = "warehouse";
|
||||
const TABLE_UUID: &str = "f6dd46c2-4971-4d41-b188-452d5f5c2c01";
|
||||
const WAREHOUSE_LOCATION: &str = "s3://warehouse/tables/a9d8f3b1-abde-4a0b-b6fb-e36a30abff16";
|
||||
const OBJECT: &str =
|
||||
".rustfs-table/warehouses/default/namespaces/analytics/events/tables/orders/metadata/00002.metadata.json";
|
||||
const PREVIOUS: &str =
|
||||
".rustfs-table/warehouses/default/namespaces/analytics/events/tables/orders/metadata/00001.metadata.json";
|
||||
|
||||
fn empty_metadata() -> Value {
|
||||
table_metadata_json(TABLE_UUID, WAREHOUSE_LOCATION)
|
||||
}
|
||||
|
||||
fn validate(metadata: &Value) -> TableCatalogStoreResult<()> {
|
||||
validate_empty_table_cleanup_metadata(BUCKET, OBJECT, &serde_json::to_vec(metadata).unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_accepts_v2_and_same_directory_history() {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["metadata-log"] = json!([
|
||||
{"timestamp-ms": 1, "metadata-file": PREVIOUS},
|
||||
{"timestamp-ms": 1, "metadata-file": format!("s3://{BUCKET}/{PREVIOUS}")}
|
||||
]);
|
||||
assert!(validate(&metadata).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_accepts_v1_without_snapshots() {
|
||||
let mut metadata = json!({
|
||||
"format-version": 1,
|
||||
"table-uuid": TABLE_UUID,
|
||||
"location": WAREHOUSE_LOCATION,
|
||||
"last-updated-ms": 1,
|
||||
"last-column-id": 0,
|
||||
"schema": {"type": "struct", "fields": []},
|
||||
"partition-spec": [],
|
||||
"current-snapshot-id": -1
|
||||
});
|
||||
assert!(validate(&metadata).is_ok());
|
||||
metadata["snapshots"] = json!([{
|
||||
"snapshot-id": 1,
|
||||
"timestamp-ms": 1,
|
||||
"manifests": ["s3://warehouse/tables/table-id/metadata/manifest.avro"]
|
||||
}]);
|
||||
validate_supported_table_metadata(&metadata).unwrap();
|
||||
assert!(matches!(validate(&metadata), Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_unsupported_versions_and_unknown_references() {
|
||||
for version in [0, 3, 4] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["format-version"] = json!(version);
|
||||
assert!(validate(&metadata).is_err(), "version {version}");
|
||||
}
|
||||
for field in [
|
||||
"manifest-list",
|
||||
"manifests",
|
||||
"data-files",
|
||||
"delete-files",
|
||||
"next-row-id",
|
||||
"future-field",
|
||||
] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata[field] = json!([]);
|
||||
assert!(validate(&metadata).is_err(), "field {field}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_snapshot_and_history_references() {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["snapshots"] = json!([{
|
||||
"snapshot-id": 1,
|
||||
"sequence-number": 0,
|
||||
"timestamp-ms": 1,
|
||||
"manifest-list": "s3://warehouse/tables/table-id/metadata/list.avro",
|
||||
"summary": {"operation": "append"}
|
||||
}]);
|
||||
validate_supported_table_metadata(&metadata).unwrap();
|
||||
assert!(matches!(validate(&metadata), Err(TableCatalogStoreError::Conflict(_))));
|
||||
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["snapshot-log"] = json!([{"timestamp-ms": 1, "snapshot-id": 1}]);
|
||||
assert!(validate(&metadata).is_err());
|
||||
metadata = empty_metadata();
|
||||
metadata["refs"] = json!({"main": {"snapshot-id": 1, "type": "branch"}});
|
||||
assert!(validate(&metadata).is_err());
|
||||
metadata = empty_metadata();
|
||||
metadata["current-snapshot-id"] = json!(1);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_accepts_only_absent_null_or_minus_one_current_snapshot() {
|
||||
for version in [1, 2] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["format-version"] = json!(version);
|
||||
if version == 1 {
|
||||
metadata["schema"] = metadata["schemas"][0].clone();
|
||||
metadata["partition-spec"] = json!([]);
|
||||
}
|
||||
assert!(validate(&metadata).is_ok());
|
||||
for value in [Value::Null, json!(-1)] {
|
||||
metadata["current-snapshot-id"] = value;
|
||||
assert!(validate(&metadata).is_ok(), "version {version}");
|
||||
}
|
||||
for value in [json!(1), json!(0), json!(-2), json!("1"), json!(false), json!([])] {
|
||||
metadata["current-snapshot-id"] = value;
|
||||
assert!(validate(&metadata).is_err(), "version {version}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_statistics_references_and_malformed_empty_fields() {
|
||||
for field in ["snapshots", "snapshot-log", "statistics", "partition-statistics"] {
|
||||
for value in [Value::Null, json!({}), json!("empty"), json!([{}])] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata[field] = value;
|
||||
assert!(validate(&metadata).is_err(), "field {field}");
|
||||
}
|
||||
}
|
||||
for value in [Value::Null, json!([]), json!("empty")] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["refs"] = value;
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_unknown_or_cross_table_history() {
|
||||
for location in [
|
||||
format!("s3://other/{PREVIOUS}"),
|
||||
PREVIOUS.replace("/orders/", "/other/"),
|
||||
PREVIOUS.replace("/analytics/events/", "/another/"),
|
||||
PREVIOUS.replace("00001.metadata.json", "../00001.metadata.json"),
|
||||
PREVIOUS.replace("00001.metadata.json", "%2e%2e.json"),
|
||||
format!("{PREVIOUS}?versionId=previous"),
|
||||
format!("/{PREVIOUS}"),
|
||||
OBJECT.to_string(),
|
||||
] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["metadata-log"] = json!([{"timestamp-ms": 1, "metadata-file": location}]);
|
||||
assert!(validate(&metadata).is_err(), "location {location}");
|
||||
}
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["metadata-log"] = json!([{"timestamp-ms": 1, "metadata-file": PREVIOUS, "future-file": "data.parquet"}]);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_requires_valid_schema_uuid_and_warehouse() {
|
||||
for uuid in ["table-uuid", "00000000-0000-0000-0000-000000000000", ""] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["table-uuid"] = json!(uuid);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
for location in [
|
||||
"s3://other/tables/table-id",
|
||||
"s3://warehouse/",
|
||||
"s3://warehouse/.rustfs-table/metadata",
|
||||
"file:///table",
|
||||
"s3://warehouse/custom/table-id",
|
||||
"s3://warehouse/tables/table-id",
|
||||
"s3://warehouse/tables/00000000-0000-0000-0000-000000000000",
|
||||
"s3://warehouse/tables/a9d8f3b1-abde-4a0b-b6fb-e36a30abff16/nested",
|
||||
] {
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["location"] = json!(location);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
let mut metadata = empty_metadata();
|
||||
metadata["schemas"] = json!([]);
|
||||
assert!(validate(&metadata).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_rejects_non_default_metadata_paths() {
|
||||
let bytes = serde_json::to_vec(&empty_metadata()).unwrap();
|
||||
for object in [
|
||||
OBJECT.replace("/default/", "/custom/"),
|
||||
OBJECT.replace("/tables/", "/views/"),
|
||||
OBJECT.replace("/metadata/", "/data/"),
|
||||
OBJECT.replace("00002.metadata.json", "part.parquet"),
|
||||
OBJECT.replace("00002.metadata.json", "nested/00002.metadata.json"),
|
||||
format!("s3://{BUCKET}/{OBJECT}"),
|
||||
] {
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, &object, &bytes).is_err(), "object {object}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_table_cleanup_decodes_gzip_and_rejects_invalid_json() {
|
||||
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
encoder.write_all(&serde_json::to_vec(&empty_metadata()).unwrap()).unwrap();
|
||||
let bytes = encoder.finish().unwrap();
|
||||
let object = OBJECT.replace(".metadata.json", ".gz.metadata.json");
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, &object, &bytes).is_ok());
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, OBJECT, b"{}").is_err());
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, OBJECT, b"[").is_err());
|
||||
assert!(validate_empty_table_cleanup_metadata(BUCKET, &object, b"invalid gzip").is_err());
|
||||
}
|
||||
}
|
||||
@@ -28,9 +28,11 @@ use std::{
|
||||
|
||||
use crate::storage_api::table::contract::http::HTTPPreconditions;
|
||||
use crate::storage_api::table::contract::list::{
|
||||
ListObjectVersionsInfo as StorageListObjectVersionsInfo, ListObjectsV2Info as StorageListObjectsV2Info,
|
||||
ListOperations as StorageListOperations, ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
|
||||
AuthoritativeListOperations as StorageAuthoritativeListOperations, ListObjectVersionsInfo as StorageListObjectVersionsInfo,
|
||||
ListObjectsV2Info as StorageListObjectsV2Info, ListOperations as StorageListOperations,
|
||||
ObjectInfoOrErr as StorageObjectInfoOrErr, WalkOptions as StorageWalkOptions,
|
||||
};
|
||||
use crate::storage_api::table::contract::multipart::MultipartUploadInspection as StorageMultipartUploadInspection;
|
||||
use crate::storage_api::table::contract::namespace::NamespaceLocking as StorageNamespaceLocking;
|
||||
use crate::storage_api::table::contract::object::{ObjectIO as StorageObjectIO, ObjectOperations as StorageObjectOperations};
|
||||
use crate::storage_api::table::contract::range::HTTPRangeSpec;
|
||||
@@ -58,6 +60,7 @@ use crate::storage_api::table::{
|
||||
StorageObjectOptions as ObjectOptions, StorageObjectToDelete as ObjectToDelete, StoragePutObjReader as PutObjReader,
|
||||
};
|
||||
|
||||
mod cleanup_metadata;
|
||||
mod error;
|
||||
mod iceberg;
|
||||
mod identifier;
|
||||
@@ -246,7 +249,9 @@ pub(crate) trait TableCatalogStorage:
|
||||
WalkOptions = CatalogWalkOptions,
|
||||
WalkCancellation = tokio_util::sync::CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<CatalogObjectInfoOrErr>,
|
||||
> + StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
> + StorageAuthoritativeListOperations
|
||||
+ StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
+ StorageMultipartUploadInspection<Error = EcstoreError>
|
||||
{
|
||||
}
|
||||
|
||||
@@ -274,7 +279,9 @@ impl<T> TableCatalogStorage for T where
|
||||
WalkOptions = CatalogWalkOptions,
|
||||
WalkCancellation = tokio_util::sync::CancellationToken,
|
||||
WalkResultSender = tokio::sync::mpsc::Sender<CatalogObjectInfoOrErr>,
|
||||
> + StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
> + StorageAuthoritativeListOperations
|
||||
+ StorageNamespaceLocking<Error = EcstoreError, NamespaceLock = rustfs_lock::NamespaceLockWrapper>
|
||||
+ StorageMultipartUploadInspection<Error = EcstoreError>
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::object::{
|
||||
next_table_catalog_update_time, table_warehouse_index_state_ready, validate_table_warehouse_index_entry_object,
|
||||
};
|
||||
use super::*;
|
||||
|
||||
pub(super) struct TableBucketDisablement {
|
||||
pub(super) entry: TableBucketEntry,
|
||||
guards: Vec<TableCatalogLockGuard>,
|
||||
}
|
||||
|
||||
/// An in-memory, fenced cleanup plan. No mutation occurs until every object has
|
||||
/// been validated and the caller has authorized all metadata deletions.
|
||||
pub(crate) struct EmptyTableBucketCleanup {
|
||||
disablement: TableBucketDisablement,
|
||||
metadata: Vec<(String, String)>,
|
||||
indexes: Vec<(String, String)>,
|
||||
fence: TableCatalogObjectMutationFence,
|
||||
}
|
||||
|
||||
impl EmptyTableBucketCleanup {
|
||||
pub(crate) fn metadata_objects(&self) -> impl Iterator<Item = &str> {
|
||||
self.metadata.iter().map(|(key, _)| key.as_str())
|
||||
}
|
||||
|
||||
pub(crate) fn ensure_held(&self) -> TableCatalogStoreResult<()> {
|
||||
self.fence.ensure_held()
|
||||
}
|
||||
|
||||
pub(crate) fn mutation_fence(&self) -> TableCatalogObjectMutationFence {
|
||||
self.fence.clone()
|
||||
}
|
||||
}
|
||||
|
||||
fn cleanup_conflict(message: impl Into<String>) -> TableCatalogStoreError {
|
||||
TableCatalogStoreError::Conflict(message.into())
|
||||
}
|
||||
|
||||
impl<B: TableCatalogObjectBackend> ObjectTableCatalogStore<B> {
|
||||
pub(super) async fn begin_table_bucket_disablement(
|
||||
&self,
|
||||
mut entry: TableBucketEntry,
|
||||
publication: Option<&dyn TableCommitPublication>,
|
||||
) -> TableCatalogStoreResult<TableBucketDisablement> {
|
||||
validate_table_bucket_entry(&entry)?;
|
||||
let mut guards = Vec::new();
|
||||
if publication.is_some_and(|publication| publication.catalog_migration_read_permit_status() == Some(true)) {
|
||||
if self
|
||||
.backend
|
||||
.read_object(self.catalog_bucket(), &self.paths.backing_migration_global_fence_path())
|
||||
.await?
|
||||
.is_some()
|
||||
{
|
||||
return Err(cleanup_conflict("table bucket registry migration is in progress"));
|
||||
}
|
||||
self.ensure_object_backed_catalog_write_permit_after_lock(&entry.table_bucket)
|
||||
.await?;
|
||||
} else {
|
||||
guards.push(self.acquire_table_bucket_registry_write_permit().await?);
|
||||
guards.push(self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?);
|
||||
}
|
||||
let object = self.paths.table_bucket_entry_path(&entry.table_bucket);
|
||||
guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?);
|
||||
if let Some((current, _)) = self.read_table_bucket_with_etag_unlocked(&entry.table_bucket).await? {
|
||||
if current.active_rename_id.is_some()
|
||||
|| !matches!(current.state, TableCatalogEntryState::Active | TableCatalogEntryState::Deleted)
|
||||
{
|
||||
return Err(cleanup_conflict("table bucket has an operation in progress"));
|
||||
}
|
||||
entry = current;
|
||||
}
|
||||
Ok(TableBucketDisablement { entry, guards })
|
||||
}
|
||||
|
||||
pub(super) async fn finish_table_bucket_disablement(
|
||||
&self,
|
||||
disablement: &mut TableBucketDisablement,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let mut entry = disablement.entry.clone();
|
||||
entry.state = TableCatalogEntryState::Deleted;
|
||||
entry.updated_at = Some(next_table_catalog_update_time(entry.updated_at.as_deref()));
|
||||
self.write_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&self.paths.table_bucket_entry_path(&entry.table_bucket),
|
||||
&entry,
|
||||
TableCatalogPutPrecondition::Any,
|
||||
disablement
|
||||
.guards
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn prepare_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
entry: TableBucketEntry,
|
||||
publication: &impl TableCatalogObjectBackend,
|
||||
) -> TableCatalogStoreResult<EmptyTableBucketCleanup> {
|
||||
if !publication.holds_table_bucket(&entry.table_bucket) {
|
||||
return Err(cleanup_conflict("empty table bucket cleanup requires a publication fence"));
|
||||
}
|
||||
let disablement = self.begin_table_bucket_disablement(entry, Some(publication)).await?;
|
||||
let bucket = &disablement.entry.table_bucket;
|
||||
let mut fence = publication
|
||||
.object_mutation_fence()
|
||||
.ok_or_else(|| cleanup_conflict("cleanup requires storage fencing"))?;
|
||||
fence
|
||||
.signals
|
||||
.extend(disablement.guards.iter().filter_map(TableCatalogLockGuard::lock_lost_signal));
|
||||
fence.ensure_held()?;
|
||||
|
||||
let catalog_objects = self
|
||||
.backend
|
||||
.list_objects_for_cleanup(self.catalog_bucket(), &self.paths.table_bucket_root_prefix(bucket))
|
||||
.await?;
|
||||
let bucket_path = self.paths.table_bucket_entry_path(bucket);
|
||||
let index_state_path = self.paths.warehouse_index_state_path(bucket);
|
||||
let index_prefix = self.paths.warehouse_index_entries_prefix(bucket);
|
||||
let mut indexes = Vec::new();
|
||||
for object in catalog_objects {
|
||||
if object.delete_marker
|
||||
|| object
|
||||
.version_id
|
||||
.as_deref()
|
||||
.is_some_and(|version| !version.is_empty() && version != "null")
|
||||
{
|
||||
return Err(cleanup_conflict("catalog contains versioned or deleted records"));
|
||||
}
|
||||
if object.key == bucket_path {
|
||||
continue;
|
||||
}
|
||||
let etag = if object.key == index_state_path {
|
||||
let (state, etag) = self
|
||||
.read_entry::<TableWarehouseIndexStateEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
.ok_or_else(|| cleanup_conflict("warehouse index state changed during cleanup"))?;
|
||||
table_warehouse_index_state_ready(&state, bucket)?;
|
||||
if !matches!(state.state, TableCatalogEntryState::Active | TableCatalogEntryState::Deleted) {
|
||||
return Err(cleanup_conflict("warehouse index has an operation in progress"));
|
||||
}
|
||||
etag
|
||||
} else if object.key.starts_with(&index_prefix) {
|
||||
let (index, etag) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
.ok_or_else(|| cleanup_conflict("warehouse index changed during cleanup"))?;
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object.key, &index)?;
|
||||
if index.table_bucket != *bucket || index.state != TableCatalogEntryState::Deleted {
|
||||
return Err(cleanup_conflict("table bucket contains an active warehouse index"));
|
||||
}
|
||||
Namespace::parse(&index.namespace).map_err(|err| cleanup_conflict(err.to_string()))?;
|
||||
IdentifierSegment::parse(index.table.clone()).map_err(|err| cleanup_conflict(err.to_string()))?;
|
||||
if Uuid::parse_str(&index.table_id).ok().filter(|id| !id.is_nil()).is_none() {
|
||||
return Err(cleanup_conflict("warehouse index table identity is invalid"));
|
||||
}
|
||||
let expected = format!("tables/{}/", index.table_id);
|
||||
if index.warehouse_object_prefix != expected {
|
||||
return Err(cleanup_conflict("cleanup requires the default table warehouse location"));
|
||||
}
|
||||
etag
|
||||
} else {
|
||||
return Err(cleanup_conflict("table bucket contains namespaces or retained catalog resources"));
|
||||
};
|
||||
let etag = etag.ok_or_else(|| cleanup_conflict("catalog cleanup object has no etag"))?;
|
||||
indexes.push((object.key, etag));
|
||||
}
|
||||
self.backend.ensure_no_multipart_uploads(bucket).await?;
|
||||
let mut metadata = Vec::new();
|
||||
for object in self.backend.list_objects_for_cleanup(bucket, "").await? {
|
||||
if object.delete_marker
|
||||
|| object
|
||||
.version_id
|
||||
.as_deref()
|
||||
.is_some_and(|version| !version.is_empty() && version != "null")
|
||||
{
|
||||
return Err(cleanup_conflict("table bucket contains object versions or delete markers"));
|
||||
}
|
||||
let value = publication
|
||||
.read_object_limited(bucket, &object.key, TABLE_METADATA_JSON_MAX_SIZE)
|
||||
.await?
|
||||
.ok_or_else(|| cleanup_conflict("metadata object changed during cleanup"))?;
|
||||
crate::table_catalog::cleanup_metadata::validate_empty_table_cleanup_metadata(bucket, &object.key, &value.data)?;
|
||||
let etag = value
|
||||
.etag
|
||||
.ok_or_else(|| cleanup_conflict("metadata cleanup object has no etag"))?;
|
||||
metadata.push((object.key, etag));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
Ok(EmptyTableBucketCleanup {
|
||||
disablement,
|
||||
metadata,
|
||||
indexes,
|
||||
fence,
|
||||
})
|
||||
}
|
||||
|
||||
async fn execute_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
cleanup: &mut EmptyTableBucketCleanup,
|
||||
incarnation: Uuid,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let bucket = &cleanup.disablement.entry.table_bucket;
|
||||
for (key, etag) in &cleanup.metadata {
|
||||
cleanup.ensure_held()?;
|
||||
self.backend
|
||||
.delete_object_if_match_fenced_for_incarnation(bucket, key, etag, incarnation, &cleanup.fence)
|
||||
.await?;
|
||||
}
|
||||
// A retry inventories the remaining metadata. Keep warehouse protection
|
||||
// until all user objects and uploads have been checked again.
|
||||
if !self.backend.list_objects_for_cleanup(bucket, "").await?.is_empty() {
|
||||
return Err(cleanup_conflict("table bucket changed during cleanup"));
|
||||
}
|
||||
self.backend.ensure_no_multipart_uploads(bucket).await?;
|
||||
for (key, etag) in &cleanup.indexes {
|
||||
cleanup.ensure_held()?;
|
||||
self.backend
|
||||
.delete_object_if_match_fenced(self.catalog_bucket(), key, etag, &cleanup.fence)
|
||||
.await?;
|
||||
}
|
||||
cleanup.ensure_held()?;
|
||||
self.finish_table_bucket_disablement(&mut cleanup.disablement).await
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: TableCatalogObjectBackend> ConfiguredTableCatalogStore<B> {
|
||||
pub(crate) async fn prepare_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
entry: TableBucketEntry,
|
||||
publication: &impl TableCatalogObjectBackend,
|
||||
) -> TableCatalogStoreResult<EmptyTableBucketCleanup> {
|
||||
match self {
|
||||
Self::ObjectBacked(store) => store.prepare_empty_table_bucket_cleanup(entry, publication).await,
|
||||
Self::DurableStrong(_) => Err(TableCatalogStoreError::Unsupported(
|
||||
"empty table bucket cleanup requires object-backed catalog storage".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn execute_empty_table_bucket_cleanup(
|
||||
&self,
|
||||
cleanup: &mut EmptyTableBucketCleanup,
|
||||
incarnation: Uuid,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
match self {
|
||||
Self::ObjectBacked(store) => store.execute_empty_table_bucket_cleanup(cleanup, incarnation).await,
|
||||
Self::DurableStrong(_) => Err(TableCatalogStoreError::Unsupported(
|
||||
"empty table bucket cleanup requires object-backed catalog storage".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,8 +17,13 @@ use super::*;
|
||||
use crate::storage_api::table::{WriteCommitGuard, WriteCompletion};
|
||||
|
||||
mod backup;
|
||||
mod cleanup;
|
||||
mod migration;
|
||||
mod object;
|
||||
mod registration_cache;
|
||||
#[cfg(test)]
|
||||
mod registration_cache_tests;
|
||||
pub(crate) use registration_cache::TableCatalogRegistrationCache;
|
||||
mod strong;
|
||||
|
||||
pub(crate) use backup::{TableCatalogBackupReport, TableCatalogRestoreReport};
|
||||
@@ -775,6 +780,14 @@ pub(crate) struct TableCatalogObjectListPage {
|
||||
pub is_truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) struct TableCatalogCleanupObject {
|
||||
pub key: String,
|
||||
pub version_id: Option<String>,
|
||||
pub etag: Option<String>,
|
||||
pub delete_marker: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub(crate) enum TableCatalogPutPrecondition {
|
||||
Any,
|
||||
@@ -803,6 +816,10 @@ pub(in crate::table_catalog) fn catalog_list_next_continuation(
|
||||
|
||||
#[async_trait::async_trait]
|
||||
pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static {
|
||||
fn registration_cache(&self) -> Option<TableCatalogRegistrationCache> {
|
||||
None
|
||||
}
|
||||
|
||||
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
|
||||
None
|
||||
}
|
||||
@@ -918,6 +935,30 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static {
|
||||
fence.ensure_held()
|
||||
}
|
||||
|
||||
async fn put_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: Vec<u8>,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
_write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
self.put_object_fenced(bucket, object, data, precondition, fence).await
|
||||
}
|
||||
|
||||
async fn delete_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
_write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
self.delete_object_unlocked(bucket, object).await?;
|
||||
fence.ensure_held()
|
||||
}
|
||||
|
||||
async fn delete_object(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()>;
|
||||
|
||||
async fn delete_object_if_match(&self, bucket: &str, object: &str, expected_etag: &str) -> TableCatalogStoreResult<()>;
|
||||
@@ -934,12 +975,41 @@ pub(crate) trait TableCatalogObjectBackend: Clone + Send + Sync + 'static {
|
||||
fence.ensure_held()
|
||||
}
|
||||
|
||||
async fn delete_object_if_match_fenced_for_incarnation(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_expected_etag: &str,
|
||||
_expected_incarnation_id: Uuid,
|
||||
_fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
Err(TableCatalogStoreError::Unsupported(
|
||||
"incarnation-checked cleanup is not supported by this backend".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn delete_object_unlocked(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<()> {
|
||||
self.delete_object(bucket, object).await
|
||||
}
|
||||
|
||||
async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult<Vec<String>>;
|
||||
|
||||
async fn list_objects_for_cleanup(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_prefix: &str,
|
||||
) -> TableCatalogStoreResult<Vec<TableCatalogCleanupObject>> {
|
||||
Err(TableCatalogStoreError::Unsupported(
|
||||
"authoritative cleanup listing is not supported by this backend".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, _bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
Err(TableCatalogStoreError::Unsupported(
|
||||
"multipart cleanup validation is not supported by this backend".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn list_objects_page(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2204,6 +2274,7 @@ where
|
||||
pub(crate) struct EcStoreTableCatalogObjectBackend<S> {
|
||||
store: Arc<S>,
|
||||
strong_runtime: StrongTableCatalogRuntime,
|
||||
registration_cache: TableCatalogRegistrationCache,
|
||||
}
|
||||
|
||||
impl<S> Clone for EcStoreTableCatalogObjectBackend<S> {
|
||||
@@ -2211,6 +2282,7 @@ impl<S> Clone for EcStoreTableCatalogObjectBackend<S> {
|
||||
Self {
|
||||
store: self.store.clone(),
|
||||
strong_runtime: self.strong_runtime.clone(),
|
||||
registration_cache: self.registration_cache.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2220,7 +2292,19 @@ where
|
||||
S: TableCatalogStorage,
|
||||
{
|
||||
pub fn new_with_strong_runtime(store: Arc<S>, strong_runtime: StrongTableCatalogRuntime) -> Self {
|
||||
Self { store, strong_runtime }
|
||||
Self::new_with_runtimes(store, strong_runtime, TableCatalogRegistrationCache::default())
|
||||
}
|
||||
|
||||
pub fn new_with_runtimes(
|
||||
store: Arc<S>,
|
||||
strong_runtime: StrongTableCatalogRuntime,
|
||||
registration_cache: TableCatalogRegistrationCache,
|
||||
) -> Self {
|
||||
Self {
|
||||
store,
|
||||
strong_runtime,
|
||||
registration_cache,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2231,6 +2315,10 @@ impl<S> TableCatalogObjectBackend for EcStoreTableCatalogObjectBackend<S>
|
||||
where
|
||||
S: TableCatalogStorage,
|
||||
{
|
||||
fn registration_cache(&self) -> Option<TableCatalogRegistrationCache> {
|
||||
Some(self.registration_cache.clone())
|
||||
}
|
||||
|
||||
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
|
||||
Some(self.strong_runtime.clone())
|
||||
}
|
||||
@@ -2390,6 +2478,46 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
data: Vec<u8>,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
self.put_object_with_options(bucket, object, data, precondition, Some(write_guards), Some(fence))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_object_unlocked_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
if write_guards.is_empty() {
|
||||
return Err(TableCatalogStoreError::Internal("catalog delete has no namespace owner".to_string()));
|
||||
}
|
||||
let mut options = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
fence.add_to_object_options(&mut options);
|
||||
for guard in &write_guards {
|
||||
options.add_write_commit_guard(guard);
|
||||
}
|
||||
match self.store.delete_object(bucket, object, options).await {
|
||||
Ok(_) => fence.ensure_held(),
|
||||
Err(err) if is_missing_storage_error(&err) => fence.ensure_held(),
|
||||
Err(err) => Err(storage_error_to_catalog("delete fenced catalog object", err)),
|
||||
}
|
||||
}
|
||||
|
||||
async fn put_object_fenced(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2412,7 +2540,7 @@ where
|
||||
}
|
||||
|
||||
async fn delete_object_if_match(&self, bucket: &str, object: &str, expected_etag: &str) -> TableCatalogStoreResult<()> {
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, None)
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, None, None)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -2424,7 +2552,25 @@ where
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
fence.ensure_held()?;
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, Some(fence))
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, Some(fence), None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_object_if_match_fenced_for_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_etag: &str,
|
||||
expected_incarnation_id: Uuid,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
if expected_incarnation_id.is_nil() {
|
||||
return Err(TableCatalogStoreError::Invalid(
|
||||
"cleanup requires a non-nil bucket incarnation".to_string(),
|
||||
));
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
self.delete_object_if_match_with_fence(bucket, object, expected_etag, Some(fence), Some(expected_incarnation_id))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -2498,6 +2644,57 @@ where
|
||||
})
|
||||
}
|
||||
|
||||
async fn list_objects_for_cleanup(
|
||||
&self,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
) -> TableCatalogStoreResult<Vec<TableCatalogCleanupObject>> {
|
||||
let cancel = tokio_util::sync::CancellationToken::new();
|
||||
let _cancel_on_drop = cancel.clone().drop_guard();
|
||||
let (sender, mut receiver) = tokio::sync::mpsc::channel::<CatalogObjectInfoOrErr>(100);
|
||||
let walk = self.store.clone().walk_authoritative(
|
||||
cancel,
|
||||
bucket,
|
||||
prefix,
|
||||
sender,
|
||||
CatalogWalkOptions {
|
||||
include_free_versions: true,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let collect = async move {
|
||||
let mut objects = Vec::new();
|
||||
while let Some(result) = receiver.recv().await {
|
||||
if let Some(err) = result.err {
|
||||
return Err(storage_error_to_catalog("inspect cleanup object versions", err));
|
||||
}
|
||||
let object = result.item.ok_or_else(|| {
|
||||
TableCatalogStoreError::Internal("cleanup object walk returned an empty result".to_string())
|
||||
})?;
|
||||
objects.push(TableCatalogCleanupObject {
|
||||
key: object.name,
|
||||
version_id: object.version_id.filter(|id| !id.is_nil()).map(|id| id.to_string()),
|
||||
etag: object.etag,
|
||||
delete_marker: object.delete_marker,
|
||||
});
|
||||
}
|
||||
Ok(objects)
|
||||
};
|
||||
let (walk_result, objects) = tokio::join!(walk, collect);
|
||||
let objects = objects?;
|
||||
walk_result.map_err(|err| storage_error_to_catalog("list cleanup object versions", err))?;
|
||||
Ok(objects)
|
||||
}
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
self.store.ensure_no_multipart_uploads(bucket).await.map_err(|err| match err {
|
||||
StorageError::BucketNotEmpty(_) => {
|
||||
TableCatalogStoreError::Conflict(format!("table bucket {bucket} contains unfinished multipart uploads"))
|
||||
}
|
||||
other => storage_error_to_catalog("inspect unfinished multipart uploads", other),
|
||||
})
|
||||
}
|
||||
|
||||
async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<TableCatalogLockGuard> {
|
||||
let lock = self
|
||||
.store
|
||||
@@ -2534,12 +2731,14 @@ where
|
||||
object: &str,
|
||||
expected_etag: &str,
|
||||
fence: Option<&TableCatalogObjectMutationFence>,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
let mut options = ObjectOptions {
|
||||
http_preconditions: Some(HTTPPreconditions {
|
||||
if_match: Some(expected_etag.to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
expected_bucket_incarnation_id: expected_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(fence) = fence {
|
||||
|
||||
@@ -13,7 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use futures::{StreamExt, TryStreamExt, stream};
|
||||
|
||||
pub(super) const TABLE_REGISTRATION_READ_CONCURRENCY: usize = 8;
|
||||
const TABLE_MAINTENANCE_RENAME_MAX_OBJECTS: usize = 4096;
|
||||
const TABLE_MAINTENANCE_SCHEDULER_HISTORY_MAX_OBJECTS: usize = 4096;
|
||||
|
||||
@@ -261,7 +263,7 @@ fn validate_table_rename_intent_object(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn next_table_catalog_update_time(previous: Option<&str>) -> String {
|
||||
pub(super) fn next_table_catalog_update_time(previous: Option<&str>) -> String {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let update_time = previous
|
||||
.and_then(|value| OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339).ok())
|
||||
@@ -360,7 +362,7 @@ pub(super) fn validate_table_maintenance_report_owner(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_table_warehouse_index_entry_object(
|
||||
pub(super) fn validate_table_warehouse_index_entry_object(
|
||||
paths: &TableCatalogObjectPaths,
|
||||
object: &str,
|
||||
index: &TableWarehouseIndexEntry,
|
||||
@@ -374,7 +376,10 @@ fn validate_table_warehouse_index_entry_object(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn table_warehouse_index_state_ready(state: &TableWarehouseIndexStateEntry, table_bucket: &str) -> TableCatalogStoreResult<bool> {
|
||||
pub(super) fn table_warehouse_index_state_ready(
|
||||
state: &TableWarehouseIndexStateEntry,
|
||||
table_bucket: &str,
|
||||
) -> TableCatalogStoreResult<bool> {
|
||||
if state.version == 0 || state.version > TABLE_WAREHOUSE_INDEX_STATE_VERSION {
|
||||
return Err(TableCatalogStoreError::Invalid(format!(
|
||||
"unsupported warehouse index state version: {}",
|
||||
@@ -1217,7 +1222,7 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_entry_unlocked<T>(
|
||||
pub(super) async fn write_entry_unlocked<T>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -1225,14 +1230,39 @@ where
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
) -> TableCatalogStoreResult<()>
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
self.write_entry_unlocked_with_fence(bucket, object, entry, precondition, write_guards, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_entry_unlocked_with_fence<T>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
entry: &T,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: Option<&TableCatalogObjectMutationFence>,
|
||||
) -> TableCatalogStoreResult<()>
|
||||
where
|
||||
T: Serialize,
|
||||
{
|
||||
let data = serde_json::to_vec(entry)
|
||||
.map_err(|err| TableCatalogStoreError::Internal(format!("failed to serialize catalog entry {object}: {err}")))?;
|
||||
self.backend
|
||||
.put_object_unlocked(bucket, object, data, precondition, write_guards)
|
||||
.await
|
||||
match fence {
|
||||
Some(fence) => {
|
||||
self.backend
|
||||
.put_object_unlocked_fenced(bucket, object, data, precondition, write_guards, fence)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
self.backend
|
||||
.put_object_unlocked(bucket, object, data, precondition, write_guards)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn write_exact_entry_unlocked<T>(
|
||||
@@ -1243,11 +1273,27 @@ where
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
) -> TableCatalogStoreResult<String>
|
||||
where
|
||||
T: DeserializeOwned + PartialEq + Serialize,
|
||||
{
|
||||
self.write_exact_entry_unlocked_with_fence(bucket, object, entry, precondition, write_guards, None)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn write_exact_entry_unlocked_with_fence<T>(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
entry: &T,
|
||||
precondition: TableCatalogPutPrecondition,
|
||||
write_guards: Vec<WriteCommitGuard>,
|
||||
fence: Option<&TableCatalogObjectMutationFence>,
|
||||
) -> TableCatalogStoreResult<String>
|
||||
where
|
||||
T: DeserializeOwned + PartialEq + Serialize,
|
||||
{
|
||||
let write_result = self
|
||||
.write_entry_unlocked(bucket, object, entry, precondition, write_guards)
|
||||
.write_entry_unlocked_with_fence(bucket, object, entry, precondition, write_guards, fence)
|
||||
.await;
|
||||
let current = self.read_entry_unlocked::<T>(bucket, object).await?;
|
||||
match current {
|
||||
@@ -1264,7 +1310,7 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_table_bucket_with_etag_unlocked(
|
||||
pub(super) async fn read_table_bucket_with_etag_unlocked(
|
||||
&self,
|
||||
table_bucket: &str,
|
||||
) -> TableCatalogStoreResult<Option<(TableBucketEntry, String)>> {
|
||||
@@ -1387,6 +1433,7 @@ where
|
||||
"table rename recovery requires a table-bucket publication fence".to_string(),
|
||||
));
|
||||
}
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
let bucket_object = self.paths.table_bucket_entry_path(table_bucket);
|
||||
let Some((observed_bucket, _)) = self
|
||||
.read_entry::<TableBucketEntry>(self.catalog_bucket(), &bucket_object)
|
||||
@@ -1511,7 +1558,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == source_fence || current == source_tombstone => {}
|
||||
Some((current, current_etag)) if current == intent.source && current_etag == intent.source_etag => {
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&source_object,
|
||||
&source_fence,
|
||||
@@ -1520,6 +1567,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1549,7 +1597,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == destination_fence || current == intent.destination => {}
|
||||
None if intent.destination_etag.is_none() => {
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&destination_object,
|
||||
&destination_fence,
|
||||
@@ -1558,6 +1606,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1565,7 +1614,7 @@ where
|
||||
if current.state == TableCatalogEntryState::Deleted
|
||||
&& Some(current_etag.as_str()) == intent.destination_etag.as_deref() =>
|
||||
{
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&destination_object,
|
||||
&destination_fence,
|
||||
@@ -1574,6 +1623,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1601,7 +1651,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == source_tombstone => {}
|
||||
Some((current, current_etag)) if current == source_fence => {
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&source_object,
|
||||
&source_tombstone,
|
||||
@@ -1610,6 +1660,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1647,7 +1698,7 @@ where
|
||||
"table rename warehouse index changed during recovery".to_string(),
|
||||
));
|
||||
}
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&index_object,
|
||||
&destination_index,
|
||||
@@ -1656,6 +1707,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1674,7 +1726,7 @@ where
|
||||
{
|
||||
Some((current, _)) if current == intent.destination => {}
|
||||
Some((current, current_etag)) if current == destination_fence => {
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&destination_object,
|
||||
&intent.destination,
|
||||
@@ -1683,6 +1735,7 @@ where
|
||||
.iter()
|
||||
.flat_map(TableCatalogLockGuard::write_commit_guards)
|
||||
.collect(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -1720,12 +1773,13 @@ where
|
||||
}
|
||||
bucket_entry.active_rename_id = None;
|
||||
bucket_entry.updated_at = Some(next_table_catalog_update_time(bucket_entry.updated_at.as_deref()));
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&bucket_object,
|
||||
&bucket_entry,
|
||||
TableCatalogPutPrecondition::IfMatch(bucket_etag),
|
||||
_bucket_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
@@ -1873,7 +1927,12 @@ where
|
||||
let candidate = table_warehouse_index_entry(entry)?;
|
||||
validate_table_entry_version_and_id(entry)?;
|
||||
for existing in self
|
||||
.list_all_table_entries_with_limit(&candidate.table_bucket, None, locked_table_entry)
|
||||
.list_all_table_entries_with_limit(
|
||||
&candidate.table_bucket,
|
||||
None,
|
||||
locked_table_entry,
|
||||
TABLE_REGISTRATION_READ_CONCURRENCY,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
if existing.state != TableCatalogEntryState::Active {
|
||||
@@ -1896,34 +1955,38 @@ where
|
||||
}
|
||||
}
|
||||
let state_object = self.paths.warehouse_index_state_path(&candidate.table_bucket);
|
||||
for object in self
|
||||
let index_objects = self
|
||||
.backend
|
||||
.list_objects(self.catalog_bucket(), &self.paths.warehouse_index_entries_prefix(&candidate.table_bucket))
|
||||
.await?
|
||||
{
|
||||
if object == state_object {
|
||||
continue;
|
||||
}
|
||||
let Some((existing, _)) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object)
|
||||
.await?
|
||||
else {
|
||||
continue;
|
||||
};
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object, &existing)?;
|
||||
if existing.table_id == candidate.table_id || existing.state != TableCatalogEntryState::Active {
|
||||
continue;
|
||||
}
|
||||
if warehouse_object_prefixes_overlap(&existing.warehouse_object_prefix, &candidate.warehouse_object_prefix)
|
||||
&& self.warehouse_index_entry_has_active_owner(&existing).await?
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(format!(
|
||||
"table warehouse location overlaps an active table: {}",
|
||||
candidate.warehouse_object_prefix
|
||||
)));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
.await?;
|
||||
stream::iter(index_objects.into_iter().filter(|object| object != &state_object))
|
||||
.map(|object| {
|
||||
let candidate = &candidate;
|
||||
async move {
|
||||
let Some((existing, _)) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object)
|
||||
.await?
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object, &existing)?;
|
||||
if existing.table_id == candidate.table_id || existing.state != TableCatalogEntryState::Active {
|
||||
return Ok(());
|
||||
}
|
||||
if warehouse_object_prefixes_overlap(&existing.warehouse_object_prefix, &candidate.warehouse_object_prefix)
|
||||
&& self.warehouse_index_entry_has_active_owner(&existing).await?
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(format!(
|
||||
"table warehouse location overlaps an active table: {}",
|
||||
candidate.warehouse_object_prefix
|
||||
)));
|
||||
}
|
||||
Ok::<_, TableCatalogStoreError>(())
|
||||
}
|
||||
})
|
||||
.buffered(TABLE_REGISTRATION_READ_CONCURRENCY)
|
||||
.try_for_each(|()| async { Ok(()) })
|
||||
.await
|
||||
}
|
||||
|
||||
async fn reserve_table_warehouse_index(
|
||||
@@ -2364,7 +2427,7 @@ where
|
||||
) -> TableCatalogStoreResult<Option<TableDataPlaneResource>> {
|
||||
let mut matched: Option<TableDataPlaneResource> = None;
|
||||
for table in self
|
||||
.list_all_table_entries_with_limit(table_bucket, Some(TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_CATALOG_OBJECTS), None)
|
||||
.list_all_table_entries_with_limit(table_bucket, Some(TABLE_DATA_PLANE_INDEX_MISS_SCAN_MAX_CATALOG_OBJECTS), None, 1)
|
||||
.await?
|
||||
{
|
||||
if table.state != TableCatalogEntryState::Active {
|
||||
@@ -2456,7 +2519,7 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
let tables = self
|
||||
.list_all_table_entries_with_limit(table_bucket, max_catalog_objects, None)
|
||||
.list_all_table_entries_with_limit(table_bucket, max_catalog_objects, None, 1)
|
||||
.await?
|
||||
.into_iter()
|
||||
.filter(|table| table.state == TableCatalogEntryState::Active)
|
||||
@@ -2537,7 +2600,7 @@ where
|
||||
}
|
||||
|
||||
async fn list_all_table_entries(&self, table_bucket: &str) -> TableCatalogStoreResult<Vec<TableEntry>> {
|
||||
self.list_all_table_entries_with_limit(table_bucket, None, None).await
|
||||
self.list_all_table_entries_with_limit(table_bucket, None, None, 1).await
|
||||
}
|
||||
|
||||
async fn list_table_entry_objects_for_data_plane_scan(
|
||||
@@ -2586,8 +2649,8 @@ where
|
||||
table_bucket: &str,
|
||||
max_catalog_objects: Option<usize>,
|
||||
locked_table_entry: Option<&str>,
|
||||
read_concurrency: usize,
|
||||
) -> TableCatalogStoreResult<Vec<TableEntry>> {
|
||||
let mut entries = Vec::new();
|
||||
let table_objects = match max_catalog_objects {
|
||||
Some(max_catalog_objects) => {
|
||||
self.list_table_entry_objects_for_data_plane_scan(table_bucket, max_catalog_objects)
|
||||
@@ -2601,19 +2664,24 @@ where
|
||||
.filter(|object| object.ends_with(TABLE_ENTRY_FILE))
|
||||
.collect(),
|
||||
};
|
||||
for object in table_objects {
|
||||
// Only the exact entry protected by the caller's write lock can bypass a read lock.
|
||||
let entry = if locked_table_entry == Some(object.as_str()) {
|
||||
self.read_entry_unlocked::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
} else {
|
||||
self.read_entry::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
};
|
||||
let Some((entry, _)) = entry else {
|
||||
continue;
|
||||
};
|
||||
validate_table_entry_object(&self.paths, &object, &entry)?;
|
||||
entries.push(entry);
|
||||
}
|
||||
let mut entries = stream::iter(table_objects)
|
||||
.map(|object| async move {
|
||||
// Only the exact entry protected by the caller's write lock can bypass a read lock.
|
||||
let entry = if locked_table_entry == Some(object.as_str()) {
|
||||
self.read_entry_unlocked::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
} else {
|
||||
self.read_entry::<TableEntry>(self.catalog_bucket(), &object).await?
|
||||
};
|
||||
let Some((entry, _)) = entry else {
|
||||
return Ok(None);
|
||||
};
|
||||
validate_table_entry_object(&self.paths, &object, &entry)?;
|
||||
Ok::<_, TableCatalogStoreError>(Some(entry))
|
||||
})
|
||||
.buffered(read_concurrency)
|
||||
.try_filter_map(|entry| async move { Ok(entry) })
|
||||
.try_collect::<Vec<_>>()
|
||||
.await?;
|
||||
entries.sort_by(|left, right| (&left.namespace, &left.table).cmp(&(&right.namespace, &right.table)));
|
||||
Ok(entries)
|
||||
}
|
||||
@@ -2772,8 +2840,7 @@ where
|
||||
"table registration requires a table publication fence".to_string(),
|
||||
));
|
||||
}
|
||||
self.ensure_table_warehouse_prefix_available(&entry, Some(&table_path))
|
||||
.await?;
|
||||
self.check_registration_ownership(&entry, &table_path, publication).await?;
|
||||
let reservation = self.reserve_table_warehouse_index(&entry, true, &table_path).await?;
|
||||
if !publication.holds_table_bucket(&entry.table_bucket)
|
||||
|| !publication.holds_table(&entry.table_bucket, &entry.namespace, &entry.table)
|
||||
@@ -2784,20 +2851,23 @@ where
|
||||
"table registration publication fence was lost before catalog update".to_string(),
|
||||
));
|
||||
}
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
let result = self
|
||||
.write_entry_unlocked(
|
||||
.write_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&table_path,
|
||||
&entry,
|
||||
precondition,
|
||||
_table_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await;
|
||||
if result.is_err() {
|
||||
self.rollback_table_warehouse_index_reservation(&entry, reservation, "table entry write failed")
|
||||
.await;
|
||||
}
|
||||
result
|
||||
result?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_view_entry(&self, entry: ViewEntry, precondition: TableCatalogPutPrecondition) -> TableCatalogStoreResult<()> {
|
||||
@@ -6258,27 +6328,15 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
async fn disable_empty_table_bucket(&self, mut entry: TableBucketEntry) -> TableCatalogStoreResult<()> {
|
||||
validate_table_bucket_entry(&entry)?;
|
||||
let _registry_guard = self.acquire_table_bucket_registry_write_permit().await?;
|
||||
let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?;
|
||||
let object = self.paths.table_bucket_entry_path(&entry.table_bucket);
|
||||
let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?;
|
||||
if let Some((current, _)) = self.read_table_bucket_with_etag_unlocked(&entry.table_bucket).await? {
|
||||
if current.active_rename_id.is_some()
|
||||
|| !matches!(current.state, TableCatalogEntryState::Active | TableCatalogEntryState::Deleted)
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict("table bucket has an operation in progress".to_string()));
|
||||
}
|
||||
entry = current;
|
||||
}
|
||||
async fn disable_empty_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> {
|
||||
let mut disablement = self.begin_table_bucket_disablement(entry, None).await?;
|
||||
// Namespace creation holds this same bucket lock. Retained table, view, and
|
||||
// maintenance records also prevent removing their data-plane protections.
|
||||
let page = self
|
||||
.backend
|
||||
.list_objects_page(
|
||||
self.catalog_bucket(),
|
||||
&self.paths.namespace_entries_prefix(&entry.table_bucket),
|
||||
&self.paths.namespace_entries_prefix(&disablement.entry.table_bucket),
|
||||
None,
|
||||
NonZeroUsize::MIN,
|
||||
)
|
||||
@@ -6288,16 +6346,7 @@ where
|
||||
"table bucket contains namespaces or retained catalog resources".to_string(),
|
||||
));
|
||||
}
|
||||
entry.state = TableCatalogEntryState::Deleted;
|
||||
entry.updated_at = Some(next_table_catalog_update_time(entry.updated_at.as_deref()));
|
||||
self.write_entry_unlocked(
|
||||
self.catalog_bucket(),
|
||||
&object,
|
||||
&entry,
|
||||
TableCatalogPutPrecondition::Any,
|
||||
_guard.write_commit_guards(),
|
||||
)
|
||||
.await
|
||||
self.finish_table_bucket_disablement(&mut disablement).await
|
||||
}
|
||||
|
||||
async fn create_namespace(&self, entry: NamespaceEntry) -> TableCatalogStoreResult<()> {
|
||||
@@ -6681,6 +6730,7 @@ where
|
||||
));
|
||||
}
|
||||
let _publication_completion = TableCommitPublicationCompletion::new(&publication);
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
self.recover_active_table_rename(table_bucket, &publication).await?;
|
||||
|
||||
{
|
||||
@@ -6820,12 +6870,13 @@ where
|
||||
));
|
||||
}
|
||||
None => {
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&index_object,
|
||||
&source_index,
|
||||
TableCatalogPutPrecondition::IfAbsent,
|
||||
_index_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
@@ -6846,22 +6897,24 @@ where
|
||||
validate_table_rename_intent_object(&self.paths, &intent_object, &intent)?;
|
||||
|
||||
// An orphan intent has no catalog effect; a published bucket fence without its intent cannot be recovered safely.
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&intent_object,
|
||||
&intent,
|
||||
TableCatalogPutPrecondition::IfAbsent,
|
||||
_intent_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
bucket_entry.active_rename_id = Some(rename_id);
|
||||
bucket_entry.updated_at = Some(now);
|
||||
self.write_exact_entry_unlocked(
|
||||
self.write_exact_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&bucket_object,
|
||||
&bucket_entry,
|
||||
TableCatalogPutPrecondition::IfMatch(bucket_etag),
|
||||
_bucket_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
@@ -7311,13 +7364,15 @@ where
|
||||
}
|
||||
|
||||
let cas_started = Instant::now();
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
let cas_result = self
|
||||
.write_entry_unlocked(
|
||||
.write_entry_unlocked_with_fence(
|
||||
self.catalog_bucket(),
|
||||
&table_path,
|
||||
&next,
|
||||
TableCatalogPutPrecondition::IfMatch(current_etag),
|
||||
_guard.write_commit_guards(),
|
||||
Some(&mutation_fence),
|
||||
)
|
||||
.await;
|
||||
record_table_commit_cas_result(&request.operation, cas_started, &cas_result);
|
||||
@@ -7388,6 +7443,7 @@ where
|
||||
table.as_str()
|
||||
)));
|
||||
};
|
||||
let mutation_fence = publication.object_mutation_fence().unwrap_or_default();
|
||||
self.tombstone_table_warehouse_index_for_drop(&entry, false).await?;
|
||||
if !publication.holds_table_bucket(table_bucket)
|
||||
|| !publication.holds_table(table_bucket, &namespace.public_name(), table.as_str())
|
||||
@@ -7398,7 +7454,11 @@ where
|
||||
"table drop publication fence was lost before catalog update".to_string(),
|
||||
));
|
||||
}
|
||||
if let Err(err) = self.backend.delete_object_unlocked(self.catalog_bucket(), &object).await {
|
||||
if let Err(err) = self
|
||||
.backend
|
||||
.delete_object_unlocked_fenced(self.catalog_bucket(), &object, _guard.write_commit_guards(), &mutation_fence)
|
||||
.await
|
||||
{
|
||||
match self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await {
|
||||
Ok(None) => return Ok(()),
|
||||
Ok(Some((current, _))) if current == entry => {
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::object::{
|
||||
TABLE_REGISTRATION_READ_CONCURRENCY, table_warehouse_index_state_ready, validate_table_entry_object,
|
||||
validate_table_warehouse_index_entry_object,
|
||||
};
|
||||
use super::*;
|
||||
use futures::{StreamExt, TryStreamExt, stream};
|
||||
use std::sync::OnceLock;
|
||||
|
||||
const REGISTRATION_CACHE_BYTES: u64 = 16 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CachedOwner {
|
||||
etag: String,
|
||||
owner: Arc<TableWarehouseIndexEntry>,
|
||||
}
|
||||
|
||||
/// Runtime-local object projections. Every lookup requires a fresh committed inventory.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct TableCatalogRegistrationCache {
|
||||
entries: Arc<OnceLock<moka::sync::Cache<String, CachedOwner>>>,
|
||||
capacity_bytes: u64,
|
||||
}
|
||||
|
||||
impl Default for TableCatalogRegistrationCache {
|
||||
fn default() -> Self {
|
||||
Self::with_capacity(REGISTRATION_CACHE_BYTES)
|
||||
}
|
||||
}
|
||||
|
||||
impl TableCatalogRegistrationCache {
|
||||
fn with_capacity(bytes: u64) -> Self {
|
||||
Self {
|
||||
entries: Arc::new(OnceLock::new()),
|
||||
capacity_bytes: bytes,
|
||||
}
|
||||
}
|
||||
|
||||
fn entries(&self) -> &moka::sync::Cache<String, CachedOwner> {
|
||||
self.entries.get_or_init(|| {
|
||||
moka::sync::Cache::builder()
|
||||
.max_capacity(self.capacity_bytes)
|
||||
.weigher(|key: &String, value: &CachedOwner| {
|
||||
let owner = &value.owner;
|
||||
let bytes = [
|
||||
key.len(),
|
||||
value.etag.len(),
|
||||
owner.table_bucket.len(),
|
||||
owner.namespace.len(),
|
||||
owner.table.len(),
|
||||
owner.table_id.len(),
|
||||
owner.warehouse_object_prefix.len(),
|
||||
]
|
||||
.into_iter()
|
||||
.fold(256_usize, usize::saturating_add);
|
||||
u32::try_from(bytes).unwrap_or(u32::MAX)
|
||||
})
|
||||
.build()
|
||||
})
|
||||
}
|
||||
|
||||
fn get(&self, object: &TableCatalogCleanupObject) -> Option<Arc<TableWarehouseIndexEntry>> {
|
||||
let etag = object.etag.as_deref().filter(|etag| !etag.is_empty())?;
|
||||
self.entries
|
||||
.get()?
|
||||
.get(&object.key)
|
||||
.filter(|cached| cached.etag == etag)
|
||||
.map(|cached| cached.owner)
|
||||
}
|
||||
|
||||
fn insert(&self, object: String, etag: Option<String>, owner: Arc<TableWarehouseIndexEntry>) {
|
||||
if let Some(etag) = etag.filter(|etag| !etag.is_empty()) {
|
||||
self.entries().insert(object, CachedOwner { etag, owner });
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(in crate::table_catalog) fn clear(&self) {
|
||||
if let Some(entries) = self.entries.get() {
|
||||
entries.invalidate_all();
|
||||
entries.run_pending_tasks();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_inventory(objects: &mut [TableCatalogCleanupObject], prefix: &str) -> TableCatalogStoreResult<()> {
|
||||
objects.sort_unstable_by(|left, right| left.key.cmp(&right.key));
|
||||
if objects.iter().any(|object| {
|
||||
!object.key.starts_with(prefix)
|
||||
|| object.delete_marker
|
||||
|| object
|
||||
.version_id
|
||||
.as_deref()
|
||||
.is_some_and(|version| !version.is_empty() && version != "null")
|
||||
}) || objects.windows(2).any(|pair| pair[0].key == pair[1].key)
|
||||
{
|
||||
return Err(TableCatalogStoreError::Invalid(
|
||||
"registration requires an unversioned, complete catalog inventory".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl<B: TableCatalogObjectBackend> ObjectTableCatalogStore<B> {
|
||||
async fn registration_owner(
|
||||
&self,
|
||||
object: TableCatalogCleanupObject,
|
||||
table_record: bool,
|
||||
locked_table_entry: &str,
|
||||
cache: Option<&TableCatalogRegistrationCache>,
|
||||
) -> TableCatalogStoreResult<Arc<TableWarehouseIndexEntry>> {
|
||||
if let Some(owner) = cache.and_then(|cache| cache.get(&object)) {
|
||||
return Ok(owner);
|
||||
}
|
||||
let (owner, etag) = if table_record {
|
||||
let value = if object.key == locked_table_entry {
|
||||
self.read_entry_unlocked::<TableEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
} else {
|
||||
self.read_entry::<TableEntry>(self.catalog_bucket(), &object.key).await?
|
||||
};
|
||||
let (entry, etag) = value.ok_or_else(|| {
|
||||
TableCatalogStoreError::Unavailable("table record changed during registration inventory".to_string())
|
||||
})?;
|
||||
validate_table_entry_object(&self.paths, &object.key, &entry)?;
|
||||
(table_warehouse_index_entry(&entry)?, etag)
|
||||
} else {
|
||||
let (entry, etag) = self
|
||||
.read_entry::<TableWarehouseIndexEntry>(self.catalog_bucket(), &object.key)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
TableCatalogStoreError::Unavailable("warehouse index changed during registration inventory".to_string())
|
||||
})?;
|
||||
validate_table_warehouse_index_entry_object(&self.paths, &object.key, &entry)?;
|
||||
(entry, etag)
|
||||
};
|
||||
let owner = Arc::new(owner);
|
||||
if let Some(cache) = cache {
|
||||
// A concurrent index repair may change its ETag after enumeration.
|
||||
// Associate the projection only with the object that GET validated.
|
||||
cache.insert(object.key, etag, Arc::clone(&owner));
|
||||
}
|
||||
Ok(owner)
|
||||
}
|
||||
|
||||
pub(super) async fn check_registration_ownership(
|
||||
&self,
|
||||
entry: &TableEntry,
|
||||
locked_table_entry: &str,
|
||||
publication: &(dyn TableCommitPublication + Sync),
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
if !publication.holds_table_bucket(&entry.table_bucket) {
|
||||
return Err(TableCatalogStoreError::Unavailable("registration publication fence was lost".to_string()));
|
||||
}
|
||||
validate_table_entry_version_and_id(entry)?;
|
||||
let candidate = table_warehouse_index_entry(entry)?;
|
||||
let namespace_prefix = self.paths.namespace_entries_prefix(&entry.table_bucket);
|
||||
let index_prefix = self.paths.warehouse_index_entries_prefix(&entry.table_bucket);
|
||||
// The backend must preserve system-bucket committed-read semantics as
|
||||
// well as completeness; ordinary S3 LIST is insufficient for cache hits.
|
||||
let mut tables = self
|
||||
.backend
|
||||
.list_objects_for_cleanup(self.catalog_bucket(), &namespace_prefix)
|
||||
.await?;
|
||||
let mut indexes = self
|
||||
.backend
|
||||
.list_objects_for_cleanup(self.catalog_bucket(), &index_prefix)
|
||||
.await?;
|
||||
validate_inventory(&mut tables, &namespace_prefix)?;
|
||||
validate_inventory(&mut indexes, &index_prefix)?;
|
||||
let cache = self.backend.registration_cache();
|
||||
let mut owners = BTreeMap::new();
|
||||
let mut table_ids = BTreeSet::new();
|
||||
let mut table_reads = stream::iter(tables.into_iter().filter(|object| object.key.ends_with(TABLE_ENTRY_FILE)))
|
||||
.map(|object| self.registration_owner(object, true, locked_table_entry, cache.as_ref()))
|
||||
.buffered(TABLE_REGISTRATION_READ_CONCURRENCY);
|
||||
while let Some(owner) = table_reads.try_next().await? {
|
||||
if owner.state != TableCatalogEntryState::Active {
|
||||
continue;
|
||||
}
|
||||
if !table_ids.insert(owner.table_id.clone())
|
||||
|| (owner.table_id == candidate.table_id
|
||||
&& (owner.namespace != candidate.namespace || owner.table != candidate.table))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"table id is already registered in this table bucket".to_string(),
|
||||
));
|
||||
}
|
||||
if owner.table_id != candidate.table_id
|
||||
&& warehouse_object_prefixes_overlap(&owner.warehouse_object_prefix, &candidate.warehouse_object_prefix)
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"table warehouse location overlaps an active table".to_string(),
|
||||
));
|
||||
}
|
||||
owners.insert((owner.namespace.clone(), owner.table.clone()), owner);
|
||||
}
|
||||
let mut prefixes = owners
|
||||
.values()
|
||||
.map(|owner| owner.warehouse_object_prefix.as_str())
|
||||
.collect::<Vec<_>>();
|
||||
prefixes.sort_unstable();
|
||||
if prefixes
|
||||
.windows(2)
|
||||
.any(|pair| warehouse_object_prefixes_overlap(pair[0], pair[1]))
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict("active table warehouse locations overlap".to_string()));
|
||||
}
|
||||
let state_path = self.paths.warehouse_index_state_path(&entry.table_bucket);
|
||||
let has_state = indexes.iter().any(|object| object.key == state_path);
|
||||
let mut index_reads = stream::iter(indexes.into_iter().filter(|object| object.key != state_path))
|
||||
.map(|object| self.registration_owner(object, false, locked_table_entry, cache.as_ref()))
|
||||
.buffered(TABLE_REGISTRATION_READ_CONCURRENCY);
|
||||
while let Some(index) = index_reads.try_next().await? {
|
||||
if index.table_id == candidate.table_id || index.state != TableCatalogEntryState::Active {
|
||||
continue;
|
||||
}
|
||||
if warehouse_object_prefixes_overlap(&index.warehouse_object_prefix, &candidate.warehouse_object_prefix) {
|
||||
parse_namespace_for_store(&index.namespace)?;
|
||||
parse_table_for_store(&index.table)?;
|
||||
if owners
|
||||
.get(&(index.namespace.clone(), index.table.clone()))
|
||||
.is_some_and(|owner| {
|
||||
owner.table_id == index.table_id && owner.warehouse_object_prefix == index.warehouse_object_prefix
|
||||
})
|
||||
{
|
||||
return Err(TableCatalogStoreError::Conflict(
|
||||
"table warehouse location overlaps an active table".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
if has_state {
|
||||
let (state, _) = self
|
||||
.read_entry::<TableWarehouseIndexStateEntry>(self.catalog_bucket(), &state_path)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
TableCatalogStoreError::Unavailable("warehouse index state changed during registration inventory".to_string())
|
||||
})?;
|
||||
table_warehouse_index_state_ready(&state, &entry.table_bucket)?;
|
||||
}
|
||||
if !publication.holds_table_bucket(&entry.table_bucket) {
|
||||
return Err(TableCatalogStoreError::Unavailable("registration publication fence was lost".to_string()));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::registration_cache_tests::{cache_fixture, registration_entry};
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_capacity_and_oversized_owners_preserve_checks() {
|
||||
let (backend, _, namespace) = cache_fixture().await;
|
||||
let cache = TableCatalogRegistrationCache::with_capacity(1024);
|
||||
let backend = crate::table_catalog::test_support::TestCatalogObjectBackend {
|
||||
registration_cache: cache.clone(),
|
||||
..backend
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let mut oversized = registration_entry(&namespace, "oversized");
|
||||
oversized.warehouse_location = format!("s3://analytics/{}", "directory".repeat(512));
|
||||
store.create_table(oversized.clone()).await.unwrap();
|
||||
for number in 0..12 {
|
||||
store
|
||||
.create_table(registration_entry(&namespace, &format!("table_{number}")))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
cache.entries().run_pending_tasks();
|
||||
assert!(cache.entries().weighted_size() <= 1024);
|
||||
backend.reset_call_counts().await;
|
||||
let mut candidate = registration_entry(&namespace, "conflicting");
|
||||
candidate.warehouse_location = format!("{}/child", oversized.warehouse_location);
|
||||
assert!(matches!(store.create_table(candidate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
assert!(
|
||||
backend.read_call_count().await > 0,
|
||||
"eviction must cause reads instead of accepting an absent owner"
|
||||
);
|
||||
cache.entries().run_pending_tasks();
|
||||
assert!(cache.entries().weighted_size() <= 1024);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,376 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::table_catalog::test_support::{RecordingLeaseClient, TestCatalogObjectBackend};
|
||||
use crate::table_catalog::tests::{test_bucket_entry, test_namespace_entry, test_table_entry};
|
||||
use rustfs_lock::LockClient;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
pub(super) fn registration_entry(namespace: &Namespace, name: &str) -> TableEntry {
|
||||
let table = IdentifierSegment::parse(name).expect("valid test table name");
|
||||
let mut entry = test_table_entry(
|
||||
"analytics",
|
||||
namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = format!("id-{name}");
|
||||
entry.warehouse_location = format!("s3://analytics/tables/id-{name}");
|
||||
entry
|
||||
}
|
||||
|
||||
pub(super) async fn cache_fixture() -> (TestCatalogObjectBackend, ObjectTableCatalogStore<TestCatalogObjectBackend>, Namespace) {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry("analytics")).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry("analytics", &namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
store.create_table(registration_entry(&namespace, "orders")).await.unwrap();
|
||||
store.create_table(registration_entry(&namespace, "warmup")).await.unwrap();
|
||||
(backend, store, namespace)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_binds_projection_to_the_get_etag() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
backend.registration_cache.clear();
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
let key = (RUSTFS_META_BUCKET.to_string(), path.clone());
|
||||
let original = backend.state.lock().await.objects.get(&key).unwrap().clone();
|
||||
let pause = backend.pause_before_next_read(RUSTFS_META_BUCKET, &path).await;
|
||||
let task_store = store.clone();
|
||||
let entry = registration_entry(&namespace, "unrelated");
|
||||
let task = tokio::spawn(async move { task_store.create_table(entry).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.unwrap();
|
||||
let mut changed = registration_entry(&namespace, "orders");
|
||||
changed.warehouse_location = "s3://analytics/changed".to_string();
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&changed).unwrap())
|
||||
.await;
|
||||
pause.release();
|
||||
tokio::time::timeout(Duration::from_secs(5), task)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
backend.state.lock().await.objects.insert(key, original);
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.warehouse_location = "s3://analytics/changed/child".to_string();
|
||||
store
|
||||
.create_table(candidate)
|
||||
.await
|
||||
.expect("a GET projection cannot inherit an earlier inventory ETag");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_observes_changed_and_removed_objects() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
let mut changed = registration_entry(&namespace, "orders");
|
||||
changed.warehouse_location = "s3://analytics/relocated".to_string();
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&changed).unwrap())
|
||||
.await;
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.warehouse_location = "s3://analytics/relocated/child".to_string();
|
||||
assert!(matches!(
|
||||
store.create_table(candidate.clone()).await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
backend.delete_object(RUSTFS_META_BUCKET, &path).await.unwrap();
|
||||
candidate.table_id = changed.table_id;
|
||||
store
|
||||
.create_table(candidate)
|
||||
.await
|
||||
.expect("removed records cannot remain active through the cache");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_revalidates_missing_etags_and_corrupt_changed_indexes() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
backend
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.etagless_objects
|
||||
.insert((RUSTFS_META_BUCKET.to_string(), path.clone()));
|
||||
backend.fail_next_read(RUSTFS_META_BUCKET, &path).await;
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
backend.state.lock().await.etagless_objects.clear();
|
||||
let index = store.paths.warehouse_index_entry_path("analytics", "tables/id-orders/");
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &index, b"invalid json".to_vec())
|
||||
.await;
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
assert!(store.load_table("analytics", "sales", "candidate").await.unwrap().is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_rejects_incomplete_and_versioned_inventories() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let namespace_prefix = store.paths.namespace_entries_prefix("analytics");
|
||||
let index_prefix = store.paths.warehouse_index_entries_prefix("analytics");
|
||||
for prefix in [&namespace_prefix, &index_prefix] {
|
||||
backend.state.lock().await.failed_inventory_prefixes.insert(prefix.clone());
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
backend.state.lock().await.failed_inventory_prefixes.clear();
|
||||
}
|
||||
for (version_id, delete_marker) in [(Some(Uuid::new_v4().to_string()), false), (None, true)] {
|
||||
backend.state.lock().await.inventory_overrides.insert(
|
||||
namespace_prefix.clone(),
|
||||
vec![TableCatalogCleanupObject {
|
||||
key: format!("{namespace_prefix}unexpected"),
|
||||
version_id,
|
||||
etag: Some("etag".to_string()),
|
||||
delete_marker,
|
||||
}],
|
||||
);
|
||||
assert!(store.create_table(registration_entry(&namespace, "candidate")).await.is_err());
|
||||
}
|
||||
backend.state.lock().await.inventory_overrides.clear();
|
||||
assert!(store.load_table("analytics", "sales", "candidate").await.unwrap().is_none());
|
||||
let mut duplicate = registration_entry(&namespace, "candidate");
|
||||
duplicate.table_id = "id-orders".to_string();
|
||||
assert!(matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_cancellation_does_not_publish_a_partial_catalog() {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
backend.registration_cache.clear();
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("warmup").unwrap());
|
||||
let pause = backend.pause_before_next_read(RUSTFS_META_BUCKET, &path).await;
|
||||
let task_store = store.clone();
|
||||
let candidate = registration_entry(&namespace, "cancelled");
|
||||
let task = tokio::spawn(async move { task_store.create_table(candidate).await });
|
||||
tokio::time::timeout(Duration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.unwrap();
|
||||
task.abort();
|
||||
assert!(task.await.unwrap_err().is_cancelled());
|
||||
pause.release();
|
||||
assert!(store.load_table("analytics", "sales", "cancelled").await.unwrap().is_none());
|
||||
let mut changed = registration_entry(&namespace, "warmup");
|
||||
changed.warehouse_location = "s3://analytics/changed".to_string();
|
||||
backend
|
||||
.seed_object(RUSTFS_META_BUCKET, &path, serde_json::to_vec(&changed).unwrap())
|
||||
.await;
|
||||
let mut candidate = registration_entry(&namespace, "after_cancel");
|
||||
candidate.warehouse_location = "s3://analytics/changed/child".to_string();
|
||||
assert!(matches!(store.create_table(candidate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_isolated_runtimes_do_not_share_matching_etags() {
|
||||
let (first, first_store, namespace) = cache_fixture().await;
|
||||
let (second, second_store, _) = cache_fixture().await;
|
||||
let path = first_store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse("orders").unwrap());
|
||||
let mut changed = registration_entry(&namespace, "orders");
|
||||
changed.warehouse_location = "s3://analytics/second-only".to_string();
|
||||
let first_etag = first
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.objects
|
||||
.get(&(RUSTFS_META_BUCKET.to_string(), path.clone()))
|
||||
.unwrap()
|
||||
.etag
|
||||
.clone();
|
||||
second.registration_cache.clear();
|
||||
{
|
||||
let mut state = second.state.lock().await;
|
||||
let object = state.objects.get_mut(&(RUSTFS_META_BUCKET.to_string(), path)).unwrap();
|
||||
object.data = serde_json::to_vec(&changed).unwrap();
|
||||
object.etag = first_etag;
|
||||
}
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.warehouse_location = "s3://analytics/second-only/child".to_string();
|
||||
assert!(matches!(
|
||||
second_store.create_table(candidate.clone()).await,
|
||||
Err(TableCatalogStoreError::Conflict(_))
|
||||
));
|
||||
first_store
|
||||
.create_table(candidate)
|
||||
.await
|
||||
.expect("the first runtime has no such owner");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registration_cache_preserves_identity_after_lost_response_and_failed_rollback() {
|
||||
for lost_response in [false, true] {
|
||||
let (backend, store, namespace) = cache_fixture().await;
|
||||
let entry = registration_entry(&namespace, "ambiguous");
|
||||
let path = store
|
||||
.paths
|
||||
.table_entry_path("analytics", &namespace, &IdentifierSegment::parse(&entry.table).unwrap());
|
||||
let index = store.paths.warehouse_index_entry_path("analytics", "tables/id-ambiguous/");
|
||||
if lost_response {
|
||||
backend.fail_after_next_put(RUSTFS_META_BUCKET, &path).await;
|
||||
} else {
|
||||
backend.fail_next_put(RUSTFS_META_BUCKET, &path).await;
|
||||
}
|
||||
backend.fail_delete_attempt(RUSTFS_META_BUCKET, &index, 1).await;
|
||||
assert!(store.create_table(entry.clone()).await.is_err());
|
||||
let mut candidate = registration_entry(&namespace, "candidate");
|
||||
candidate.table_id = entry.table_id;
|
||||
let result = store.create_table(candidate).await;
|
||||
if lost_response {
|
||||
assert!(matches!(result, Err(TableCatalogStoreError::Conflict(_))));
|
||||
} else {
|
||||
result.expect("failed primary writes cannot reserve identity through cached results");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn leased_registration_publication<'a>(
|
||||
backend: &'a TestCatalogObjectBackend,
|
||||
lock: &rustfs_lock::NamespaceLock,
|
||||
owner: &str,
|
||||
) -> (
|
||||
TableCommitLockPublication<'a, TestCatalogObjectBackend>,
|
||||
Arc<rustfs_lock::distributed_lock::LockLostSignal>,
|
||||
) {
|
||||
let guard = lock
|
||||
.lock_guard(
|
||||
rustfs_lock::ObjectKey::new("analytics", default_table_bucket_publication_lock_path()),
|
||||
owner,
|
||||
Duration::from_secs(5),
|
||||
Duration::from_secs(3),
|
||||
)
|
||||
.await
|
||||
.expect("publication lease acquisition succeeds")
|
||||
.expect("publication lease is acquired");
|
||||
let signal = guard.lock_lost_signal().expect("distributed publication has a loss signal");
|
||||
let publication = TableCommitLockPublication::new(backend);
|
||||
{
|
||||
let mut state = publication.state.lock();
|
||||
state.table_bucket = Some("analytics".to_string());
|
||||
state.guards.push(TableCatalogLockGuard::namespace(guard));
|
||||
}
|
||||
(publication, signal)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn registration_cache_rejects_delayed_table_write_after_publication_lease_loss() {
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let stalled_namespace = Namespace::parse("stalled").unwrap();
|
||||
let active_namespace = Namespace::parse("active").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry("analytics")).await.unwrap();
|
||||
for namespace in [&stalled_namespace, &active_namespace] {
|
||||
store
|
||||
.create_namespace(test_namespace_entry("analytics", namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
let initial = registration_entry(&active_namespace, "initial");
|
||||
store.create_table(initial.clone()).await.unwrap();
|
||||
let client = Arc::new(RecordingLeaseClient {
|
||||
inner: rustfs_lock::LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
)))),
|
||||
acquired: Mutex::new(None),
|
||||
});
|
||||
let lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
|
||||
"registration-publication-loss".to_string(),
|
||||
vec![client.clone() as Arc<dyn LockClient>],
|
||||
1,
|
||||
);
|
||||
let (old_publication, old_signal) = leased_registration_publication(&backend, &lock, "old-writer").await;
|
||||
let old_lease = client.acquired.lock().unwrap().clone().unwrap();
|
||||
let delayed = registration_entry(&stalled_namespace, "delayed");
|
||||
let delayed_path =
|
||||
store
|
||||
.paths
|
||||
.table_entry_path("analytics", &stalled_namespace, &IdentifierSegment::parse(&delayed.table).unwrap());
|
||||
let pause = backend.pause_put_attempt(RUSTFS_META_BUCKET, &delayed_path, 1).await;
|
||||
let replacement = registration_entry(&active_namespace, "replacement");
|
||||
let delayed_write = store.register_table_with_publication(delayed.clone(), &old_publication);
|
||||
let takeover = async {
|
||||
tokio::time::timeout(Duration::from_secs(5), pause.wait_started())
|
||||
.await
|
||||
.expect("old writer reaches the table-entry commit");
|
||||
assert!(client.force_release(&old_lease).await.unwrap());
|
||||
tokio::time::timeout(Duration::from_secs(10), old_signal.notified())
|
||||
.await
|
||||
.expect("heartbeat observes publication lease loss");
|
||||
assert!(!old_publication.holds_table_bucket("analytics"));
|
||||
|
||||
let (new_publication, _) = leased_registration_publication(&backend, &lock, "new-writer").await;
|
||||
backend.reset_call_counts().await;
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(5),
|
||||
store.register_table_with_publication(replacement.clone(), &new_publication),
|
||||
)
|
||||
.await
|
||||
.expect("replacement writer completes while the old put is paused")
|
||||
.expect("replacement writer publishes its table after a fresh inventory");
|
||||
assert!(backend.list_call_count().await > 0, "takeover inventories the catalog");
|
||||
pause.release();
|
||||
};
|
||||
let (delayed_result, ()) = tokio::time::timeout(Duration::from_secs(25), async { tokio::join!(delayed_write, takeover) })
|
||||
.await
|
||||
.expect("both writers finish after the paused put is resumed");
|
||||
assert!(
|
||||
matches!(delayed_result, Err(TableCatalogStoreError::Unavailable(_))),
|
||||
"the delayed storage commit must reject the lost publication fence: {delayed_result:?}"
|
||||
);
|
||||
assert_eq!(backend.put_attempt_count(RUSTFS_META_BUCKET, &delayed_path).await, 1);
|
||||
assert!(
|
||||
backend
|
||||
.read_object(RUSTFS_META_BUCKET, &delayed_path)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
for entry in [&initial, &replacement] {
|
||||
assert_eq!(
|
||||
store.load_table("analytics", &entry.namespace, &entry.table).await.unwrap(),
|
||||
Some(entry.clone())
|
||||
);
|
||||
}
|
||||
|
||||
let subsequent = registration_entry(&active_namespace, "subsequent");
|
||||
store.create_table(subsequent.clone()).await.unwrap();
|
||||
for entry in [&initial, &replacement, &subsequent] {
|
||||
let mut duplicate = registration_entry(&active_namespace, "duplicate");
|
||||
duplicate.table_id = entry.table_id.clone();
|
||||
assert!(matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_))));
|
||||
}
|
||||
store
|
||||
.create_table(delayed)
|
||||
.await
|
||||
.expect("the rejected writer left no active table");
|
||||
}
|
||||
@@ -411,6 +411,7 @@ pub(crate) struct TestCatalogObjectBackend {
|
||||
pub(crate) state: Arc<tokio::sync::Mutex<TestCatalogObjectState>>,
|
||||
pub(crate) locks: TestCatalogObjectLocks,
|
||||
pub(crate) strong_runtime: Option<StrongTableCatalogRuntime>,
|
||||
pub(crate) registration_cache: TableCatalogRegistrationCache,
|
||||
// One-shot, path-keyed injection knobs from the admin handler tests'
|
||||
// former TestTableCatalogObjectBackend (backlog#1837 PR2): each fires
|
||||
// once for the named object and clears itself, mirroring the original
|
||||
@@ -543,8 +544,11 @@ pub(crate) struct TestCatalogObjectState {
|
||||
pub(crate) write_lock_acquisitions: BTreeMap<(String, String), usize>,
|
||||
pub(crate) read_lock_acquisitions: BTreeMap<(String, String), usize>,
|
||||
pub(crate) read_calls: usize,
|
||||
pub(crate) read_bytes: usize,
|
||||
pub(crate) metadata_calls: usize,
|
||||
pub(crate) list_calls: usize,
|
||||
pub(crate) failed_inventory_prefixes: BTreeSet<String>,
|
||||
pub(crate) inventory_overrides: BTreeMap<String, Vec<TableCatalogCleanupObject>>,
|
||||
pub(crate) next_etag: u64,
|
||||
}
|
||||
|
||||
@@ -608,6 +612,7 @@ impl TestCatalogObjectBackend {
|
||||
pub(crate) async fn reset_call_counts(&self) {
|
||||
let mut state = self.state.lock().await;
|
||||
state.read_calls = 0;
|
||||
state.read_bytes = 0;
|
||||
state.metadata_calls = 0;
|
||||
state.list_calls = 0;
|
||||
}
|
||||
@@ -955,6 +960,10 @@ impl TestCatalogObjectState {
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
fn registration_cache(&self) -> Option<TableCatalogRegistrationCache> {
|
||||
Some(self.registration_cache.clone())
|
||||
}
|
||||
|
||||
fn strong_catalog_runtime(&self) -> Option<StrongTableCatalogRuntime> {
|
||||
self.strong_runtime.clone()
|
||||
}
|
||||
@@ -1018,6 +1027,7 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
etag: (!etagless).then(|| record.etag.clone()),
|
||||
mod_time: record.mod_time,
|
||||
});
|
||||
state.read_bytes += result.as_ref().map_or(0, |object| object.data.len());
|
||||
let pause = state
|
||||
.pause_read_attempts
|
||||
.get_mut(&key)
|
||||
@@ -1142,6 +1152,17 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_object_if_match_fenced_for_incarnation(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_etag: &str,
|
||||
_expected_incarnation_id: Uuid,
|
||||
fence: &TableCatalogObjectMutationFence,
|
||||
) -> TableCatalogStoreResult<()> {
|
||||
self.delete_object_if_match_fenced(bucket, object, expected_etag, fence).await
|
||||
}
|
||||
|
||||
async fn list_objects(&self, bucket: &str, prefix: &str) -> TableCatalogStoreResult<Vec<String>> {
|
||||
let mut state = self.state.lock().await;
|
||||
state.list_calls += 1;
|
||||
@@ -1153,6 +1174,38 @@ impl TableCatalogObjectBackend for TestCatalogObjectBackend {
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn list_objects_for_cleanup(
|
||||
&self,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
) -> TableCatalogStoreResult<Vec<TableCatalogCleanupObject>> {
|
||||
let mut state = self.state.lock().await;
|
||||
state.list_calls += 1;
|
||||
if state.failed_inventory_prefixes.contains(prefix) {
|
||||
return Err(TableCatalogStoreError::Unavailable(
|
||||
"injected authoritative inventory failure".to_string(),
|
||||
));
|
||||
}
|
||||
if let Some(objects) = state.inventory_overrides.get(prefix) {
|
||||
return Ok(objects.clone());
|
||||
}
|
||||
Ok(state
|
||||
.objects
|
||||
.iter()
|
||||
.filter(|((entry_bucket, object), _)| entry_bucket == bucket && object.starts_with(prefix))
|
||||
.map(|(key, record)| TableCatalogCleanupObject {
|
||||
key: key.1.clone(),
|
||||
version_id: None,
|
||||
etag: (!state.etagless_objects.contains(key)).then(|| record.etag.clone()),
|
||||
delete_marker: false,
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn ensure_no_multipart_uploads(&self, _bucket: &str) -> TableCatalogStoreResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn acquire_write_lock(&self, bucket: &str, object: &str) -> TableCatalogStoreResult<TableCatalogLockGuard> {
|
||||
self.lock_attempts.lock().await.push((bucket.to_string(), object.to_string()));
|
||||
{
|
||||
|
||||
@@ -7145,6 +7145,228 @@ async fn table_data_plane_resource_does_not_match_sibling_prefix() {
|
||||
assert_eq!(backend.list_call_count().await, 1);
|
||||
}
|
||||
|
||||
async fn cached_registration_fixture() -> (TestCatalogObjectBackend, ObjectTableCatalogStore<TestCatalogObjectBackend>, Namespace)
|
||||
{
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let namespace = Namespace::parse("sales").expect("test namespace is valid");
|
||||
store
|
||||
.put_table_bucket(test_bucket_entry("analytics"))
|
||||
.await
|
||||
.expect("bucket is created");
|
||||
store
|
||||
.create_namespace(test_namespace_entry("analytics", &namespace))
|
||||
.await
|
||||
.expect("namespace is created");
|
||||
(backend, store, namespace)
|
||||
}
|
||||
|
||||
fn cached_registration_entry(namespace: &Namespace, name: &str) -> TableEntry {
|
||||
let table = IdentifierSegment::parse(name).expect("test table name is valid");
|
||||
let mut entry = test_table_entry(
|
||||
"analytics",
|
||||
namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = format!("id-{name}");
|
||||
entry.warehouse_location = format!("s3://analytics/tables/id-{name}");
|
||||
entry
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_bounds_reads_across_store_instances() {
|
||||
let (backend, _, namespace) = cached_registration_fixture().await;
|
||||
for number in 0..256 {
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, &format!("table_{number:04}")))
|
||||
.await
|
||||
.expect("each new store instance shares validated registration projections");
|
||||
if number >= 32 {
|
||||
let state = backend.state.lock().await;
|
||||
assert_eq!(state.list_calls, 2, "each registration inventories both catalog ranges");
|
||||
assert!(
|
||||
state.read_calls < 24,
|
||||
"registration used {} object reads at table {number}",
|
||||
state.read_calls
|
||||
);
|
||||
assert!(
|
||||
state.read_bytes < 256 * 1024,
|
||||
"registration read {} bytes at table {number}",
|
||||
state.read_bytes
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_preserves_identity_and_overlap_checks() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
let mut existing = cached_registration_entry(&namespace, "orders");
|
||||
existing.warehouse_location = "s3://analytics/shared/orders".to_string();
|
||||
store.create_table(existing.clone()).await.expect("initial table is created");
|
||||
backend
|
||||
.delete_object(RUSTFS_META_BUCKET, &store.paths.warehouse_index_entry_path("analytics", "shared/orders/"))
|
||||
.await
|
||||
.expect("simulate a missing materialized warehouse index");
|
||||
|
||||
let mut duplicate = cached_registration_entry(&namespace, "duplicate");
|
||||
duplicate.table_id = existing.table_id.clone();
|
||||
assert_matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
for location in [
|
||||
"s3://analytics/shared",
|
||||
"s3://analytics/shared/orders",
|
||||
"s3://analytics/shared/orders/child",
|
||||
] {
|
||||
let mut conflicting = cached_registration_entry(&namespace, "conflicting");
|
||||
conflicting.warehouse_location = location.to_string();
|
||||
assert_matches!(store.create_table(conflicting).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
}
|
||||
let mut adjacent = cached_registration_entry(&namespace, "adjacent");
|
||||
adjacent.warehouse_location = "s3://analytics/shared/orders2".to_string();
|
||||
store
|
||||
.create_table(adjacent)
|
||||
.await
|
||||
.expect("a different directory remains available");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_rechecks_after_failed_write_and_drop() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "orders"))
|
||||
.await
|
||||
.expect("initial table is created");
|
||||
let returns = IdentifierSegment::parse("returns").expect("test name is valid");
|
||||
let table_path = store.paths.table_entry_path("analytics", &namespace, &returns);
|
||||
backend.fail_next_put(RUSTFS_META_BUCKET, &table_path).await;
|
||||
assert!(
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.load_table("analytics", "sales", "returns")
|
||||
.await
|
||||
.expect("table lookup succeeds")
|
||||
.is_none()
|
||||
);
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.expect("retry reads the authoritative records");
|
||||
store
|
||||
.drop_table("analytics", "sales", "orders")
|
||||
.await
|
||||
.expect("drop removes the active owner");
|
||||
let mut replacement = cached_registration_entry(&namespace, "replacement");
|
||||
replacement.table_id = "id-orders".to_string();
|
||||
replacement.warehouse_location = "s3://analytics/tables/id-orders".to_string();
|
||||
store
|
||||
.create_table(replacement)
|
||||
.await
|
||||
.expect("a dropped owner no longer reserves its identity");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_observes_pointer_commits_and_rename() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
let entry = cached_registration_entry(&namespace, "orders");
|
||||
store.create_table(entry.clone()).await.expect("initial table is created");
|
||||
let metadata_directory = entry
|
||||
.metadata_location
|
||||
.rsplit_once('/')
|
||||
.expect("metadata has a parent directory")
|
||||
.0;
|
||||
let next_metadata = format!("{metadata_directory}/00002.metadata.json");
|
||||
backend
|
||||
.seed_object(
|
||||
"analytics",
|
||||
&next_metadata,
|
||||
serde_json::to_vec(&serde_json::json!({"location": entry.warehouse_location})).unwrap(),
|
||||
)
|
||||
.await;
|
||||
store
|
||||
.commit_table(TableCommitRequest {
|
||||
table_bucket: "analytics".to_string(),
|
||||
namespace: "sales".to_string(),
|
||||
table: "orders".to_string(),
|
||||
commit_id: "pointer-only".to_string(),
|
||||
idempotency_key: None,
|
||||
operation: "append".to_string(),
|
||||
expected_version_token: entry.version_token,
|
||||
expected_metadata_location: entry.metadata_location,
|
||||
new_metadata_location: next_metadata,
|
||||
requirements: Vec::new(),
|
||||
writer: None,
|
||||
})
|
||||
.await
|
||||
.expect("metadata pointer commit succeeds");
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
backend.list_call_count().await,
|
||||
2,
|
||||
"pointer changes are checked through fresh inventories"
|
||||
);
|
||||
|
||||
store
|
||||
.rename_table("analytics", "sales", "orders", "sales", "renamed")
|
||||
.await
|
||||
.unwrap();
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "shipments"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
backend.list_call_count().await > 0,
|
||||
"rename is visible in the next authoritative inventory"
|
||||
);
|
||||
let mut duplicate = cached_registration_entry(&namespace, "duplicate");
|
||||
duplicate.table_id = "id-orders".to_string();
|
||||
assert_matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_cache_observes_an_independent_writer() {
|
||||
let (backend, store, namespace) = cached_registration_fixture().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "orders"))
|
||||
.await
|
||||
.unwrap();
|
||||
let scan_store = ObjectTableCatalogStore::new(TestCatalogObjectBackend {
|
||||
registration_cache: TableCatalogRegistrationCache::default(),
|
||||
..backend.clone()
|
||||
});
|
||||
scan_store
|
||||
.create_table(cached_registration_entry(&namespace, "returns"))
|
||||
.await
|
||||
.unwrap();
|
||||
backend.reset_call_counts().await;
|
||||
store
|
||||
.create_table(cached_registration_entry(&namespace, "shipments"))
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(
|
||||
backend.list_call_count().await > 0,
|
||||
"an independent writer is visible in the next authoritative inventory"
|
||||
);
|
||||
let mut duplicate = cached_registration_entry(&namespace, "duplicate");
|
||||
duplicate.table_id = "id-returns".to_string();
|
||||
assert_matches!(store.create_table(duplicate).await, Err(TableCatalogStoreError::Conflict(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_scans_warehouse_prefixes_once() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
@@ -7173,10 +7395,217 @@ async fn object_catalog_registration_scans_warehouse_prefixes_once() {
|
||||
reads.push(backend.read_call_count().await);
|
||||
assert_eq!(backend.list_call_count().await, 2, "one table scan and one warehouse index scan");
|
||||
}
|
||||
assert_eq!(reads[2] - reads[1], 2, "each existing table and its index should be read once");
|
||||
assert_eq!(reads[2], reads[1], "unchanged validated objects should not require another GET");
|
||||
assert_eq!(store.list_tables(bucket, &namespace.public_name()).await.unwrap().len(), 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_bounds_concurrent_catalog_reads() {
|
||||
const MAX_CONCURRENT_READS: usize = 8;
|
||||
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let bucket = "analytics";
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let mut table_objects = Vec::new();
|
||||
let mut index_objects = Vec::new();
|
||||
for number in 0..MAX_CONCURRENT_READS + 2 {
|
||||
let name = format!("orders_{number:02}");
|
||||
let table = IdentifierSegment::parse(&name).unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = format!("table-{name}");
|
||||
entry.warehouse_location = format!("s3://{bucket}/tables/{name}");
|
||||
store.create_table(entry).await.unwrap();
|
||||
table_objects.push(store.paths.table_entry_path(bucket, &namespace, &table));
|
||||
index_objects.push(store.paths.warehouse_index_entry_path(bucket, &format!("tables/{name}/")));
|
||||
}
|
||||
|
||||
backend.registration_cache.clear();
|
||||
let mut scans = Vec::new();
|
||||
for mut objects in [table_objects, index_objects] {
|
||||
objects.sort();
|
||||
let mut scan = Vec::new();
|
||||
for object in objects {
|
||||
let previous_reads = backend
|
||||
.state
|
||||
.lock()
|
||||
.await
|
||||
.read_attempts
|
||||
.get(&(RUSTFS_META_BUCKET.to_string(), object.clone()))
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
let pause = backend.pause_before_next_read(RUSTFS_META_BUCKET, &object).await;
|
||||
scan.push((object, pause, previous_reads));
|
||||
}
|
||||
scans.push(scan);
|
||||
}
|
||||
|
||||
let table = IdentifierSegment::parse("new_orders").unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = "new-table-id".to_string();
|
||||
entry.warehouse_location = format!("s3://{bucket}/tables/new_orders");
|
||||
let registering_store = store.clone();
|
||||
let registration = tokio::spawn(async move { registering_store.create_table(entry).await });
|
||||
|
||||
for scan in scans {
|
||||
for (_, pause, _) in scan.iter().take(MAX_CONCURRENT_READS) {
|
||||
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, pause.wait_started())
|
||||
.await
|
||||
.expect("independent catalog reads should start concurrently");
|
||||
}
|
||||
let state = backend.state.lock().await;
|
||||
let started = scan
|
||||
.iter()
|
||||
.filter(|(object, _, previous_reads)| {
|
||||
state
|
||||
.read_attempts
|
||||
.get(&(RUSTFS_META_BUCKET.to_string(), object.clone()))
|
||||
.copied()
|
||||
.unwrap_or_default()
|
||||
> *previous_reads
|
||||
})
|
||||
.count();
|
||||
assert_eq!(started, MAX_CONCURRENT_READS, "catalog reads must respect the concurrency limit");
|
||||
drop(state);
|
||||
for (_, pause, _) in scan {
|
||||
pause.release();
|
||||
}
|
||||
}
|
||||
|
||||
tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, registration)
|
||||
.await
|
||||
.expect("registration should finish after its reads are released")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(
|
||||
store
|
||||
.load_table(bucket, &namespace.public_name(), table.as_str())
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_propagates_catalog_scan_read_errors() {
|
||||
for fail_index_read in [false, true] {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let bucket = "analytics";
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
let existing = IdentifierSegment::parse("orders").unwrap();
|
||||
seed_table_for_metadata_maintenance(
|
||||
&store,
|
||||
bucket,
|
||||
&namespace,
|
||||
&existing,
|
||||
default_table_metadata_file_path(&namespace, &existing, "00001.metadata.json"),
|
||||
)
|
||||
.await;
|
||||
let failed_object = if fail_index_read {
|
||||
store.paths.warehouse_index_entry_path(bucket, "tables/table-id/")
|
||||
} else {
|
||||
store.paths.table_entry_path(bucket, &namespace, &existing)
|
||||
};
|
||||
backend.fail_next_read(RUSTFS_META_BUCKET, &failed_object).await;
|
||||
|
||||
let table = IdentifierSegment::parse("returns").unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.table_id = "new-table-id".to_string();
|
||||
entry.warehouse_location = format!("s3://{bucket}/tables/new-table-id");
|
||||
assert_matches!(
|
||||
store.create_table(entry.clone()).await,
|
||||
Err(TableCatalogStoreError::Internal(message)) if message.contains(&failed_object)
|
||||
);
|
||||
assert!(
|
||||
store
|
||||
.load_table(bucket, &namespace.public_name(), table.as_str())
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(
|
||||
backend
|
||||
.read_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&store.paths.warehouse_index_entry_path(bucket, "tables/new-table-id/")
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
store
|
||||
.create_table(entry)
|
||||
.await
|
||||
.expect("a failed scan must release its read locks for retry");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_catalog_registration_reuses_locked_deleted_entry() {
|
||||
let backend = TestCatalogObjectBackend {
|
||||
reject_reads_while_write_locked: true,
|
||||
..Default::default()
|
||||
};
|
||||
let store = ObjectTableCatalogStore::new(backend.clone());
|
||||
let bucket = "analytics";
|
||||
let namespace = Namespace::parse("sales").unwrap();
|
||||
let table = IdentifierSegment::parse("orders").unwrap();
|
||||
store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap();
|
||||
store
|
||||
.create_namespace(test_namespace_entry(bucket, &namespace))
|
||||
.await
|
||||
.unwrap();
|
||||
let mut entry = test_table_entry(
|
||||
bucket,
|
||||
&namespace,
|
||||
&table,
|
||||
default_table_metadata_file_path(&namespace, &table, "00001.metadata.json"),
|
||||
);
|
||||
entry.state = TableCatalogEntryState::Deleted;
|
||||
backend
|
||||
.seed_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
&store.paths.table_entry_path(bucket, &namespace, &table),
|
||||
serde_json::to_vec(&entry).unwrap(),
|
||||
)
|
||||
.await;
|
||||
entry.state = TableCatalogEntryState::Active;
|
||||
entry.table_id = "replacement-table-id".to_string();
|
||||
store.create_table(entry.clone()).await.unwrap();
|
||||
assert_eq!(
|
||||
store
|
||||
.load_table(bucket, &namespace.public_name(), table.as_str())
|
||||
.await
|
||||
.unwrap(),
|
||||
Some(entry)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn catalog_backings_reject_overlapping_registered_warehouse_prefixes() {
|
||||
let backend = TestCatalogObjectBackend::default();
|
||||
@@ -20526,9 +20955,9 @@ async fn object_table_catalog_store_reuses_old_prefix_after_failed_relocation_in
|
||||
let mut next_entry = test_table_entry(bucket, &namespace, &next_table, current_metadata);
|
||||
next_entry.table_id = "next-table-id".to_string();
|
||||
next_entry.warehouse_location = format!("s3://{bucket}/tables/table-id");
|
||||
store
|
||||
.create_table(next_entry)
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), store.create_table(next_entry))
|
||||
.await
|
||||
.expect("stale warehouse prefix reuse completes")
|
||||
.expect("stale old warehouse index should not block prefix reuse");
|
||||
|
||||
let (index, _) = store
|
||||
|
||||
@@ -705,7 +705,7 @@ def check_ilm_build_budget(root: Path) -> list[str]:
|
||||
"--workspace-remap": "$GITHUB_WORKSPACE",
|
||||
"--target-dir-remap": "$GITHUB_WORKSPACE/target",
|
||||
}
|
||||
if ("workspace-lint-and-build" not in job.get("needs", []) or "--archive-file" in tokens
|
||||
if ("test-and-lint" not in job.get("needs", []) or "--archive-file" in tokens
|
||||
or any(tokens.count(flag) != 1 or tokens[tokens.index(flag) + 1:tokens.index(flag) + 2] != [value]
|
||||
for flag, value in reuse_options.items())):
|
||||
return ["ILM integration must consume the workspace producer archive"]
|
||||
@@ -729,7 +729,7 @@ def check_ilm_build_budget(root: Path) -> list[str]:
|
||||
return ["ILM integration must propagate failures"]
|
||||
if 'status=${PIPESTATUS[0]}' not in command or 'exit "${status}"' not in command or "|| true" in command.split('status=${PIPESTATUS[0]}')[0]:
|
||||
return ["ILM integration must retain the actual test exit status"]
|
||||
producer_steps = jobs.get("workspace-lint-and-build", {}).get("steps", [])
|
||||
producer_steps = jobs.get("test-and-lint", {}).get("steps", [])
|
||||
builds = [item for item in producer_steps if "ci_test_artifact.py build --configuration workspace" in item.get("run", "")]
|
||||
if len(builds) != 1:
|
||||
return ["ILM integration requires one workspace archive producer"]
|
||||
|
||||
@@ -33,7 +33,7 @@ set -uo pipefail
|
||||
# GNU date on the runners; the fallback keeps local smoke tests on macOS quiet.
|
||||
now() { date --utc --iso-8601=seconds 2>/dev/null || date -u +%Y-%m-%dT%H:%M:%S+00:00; }
|
||||
|
||||
LOG_DIR="artifacts/workspace"
|
||||
LOG_DIR="artifacts/test-and-lint"
|
||||
LOG_FILE="${LOG_DIR}/sampler.log"
|
||||
PID_FILE="${LOG_DIR}/.sampler.pid"
|
||||
INTERVAL="${SAMPLER_INTERVAL_SECS:-60}"
|
||||
|
||||
+16
-25
@@ -16,29 +16,20 @@ import unittest
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
ALWAYS_JOBS = ("classify-changes", "typos", "quick-checks")
|
||||
CODE_JOBS = (
|
||||
"workspace-lint-and-build", "workspace-tests", "core-contract", "connect-feature-boundaries", "test-and-lint-rio-v2",
|
||||
"test-and-lint", "workspace-tests", "core-contract", "connect-feature-boundaries", "test-and-lint-rio-v2",
|
||||
"test-and-lint-protocols", "build-rustfs-debug-binary", "uring-integration",
|
||||
"e2e-tests", "s3-implemented-tests", "s3-lifecycle-behavior-tests",
|
||||
)
|
||||
# Specialized boundary, hardware and alternate-feature lanes do not gate a
|
||||
# merge. They run only in the scheduled (nightly) and manual full-validation
|
||||
# runs, which report a regression within a day; repeating them on every PR
|
||||
# revision and merge group bought no earlier signal worth the runner occupancy
|
||||
# (Connect alone measured 53.5 median minutes per run and occasionally set the
|
||||
# wall). Workspace lint and tests, the short core contract, the debug build,
|
||||
# E2E smoke and the S3 lanes stay on the pull-request path because they cover
|
||||
# the product's core correctness and S3 compatibility surface.
|
||||
NIGHTLY_JOBS = ("connect-feature-boundaries", "uring-integration", "test-and-lint-rio-v2", "test-and-lint-protocols")
|
||||
# Bump when a plan produced by an older `select` can no longer be checked
|
||||
# against this revision's expected job contract. The workflow refuses a plan
|
||||
# that does not carry the current marker and falls back to the full matrix, so
|
||||
# a base planner predating the contract cannot authorize a narrower selection.
|
||||
PLAN_CONTRACT = "2"
|
||||
# Full alternate-feature tests run on merge groups and on scheduled/manual
|
||||
# runs. Pull requests and main pushes skip them: the nightly run reports a
|
||||
# feature regression within a day, and repeating four builds per PR revision
|
||||
# or per merge bought no earlier signal.
|
||||
EXTENDED_FEATURE_JOBS = ("test-and-lint-rio-v2", "test-and-lint-protocols")
|
||||
# These jobs still exercise the server and its black-box harness when only E2E
|
||||
# Rust sources change. Production code, manifests and shared test configuration
|
||||
# always select the full matrix. No production package depends on e2e_test.
|
||||
E2E_JOBS = (
|
||||
"workspace-lint-and-build", "workspace-tests", "core-contract", "build-rustfs-debug-binary", "e2e-tests",
|
||||
"test-and-lint", "workspace-tests", "core-contract", "build-rustfs-debug-binary", "e2e-tests",
|
||||
"s3-implemented-tests", "s3-lifecycle-behavior-tests",
|
||||
)
|
||||
E2E_SELECTION_FILES = {
|
||||
@@ -176,10 +167,11 @@ def expected_results(mode: str, event: str, ref: str, connect: bool = True, acti
|
||||
raise ValueError("shard rehearsal requires an explicit manual event")
|
||||
expected = {job: "success" for job in ALWAYS_JOBS}
|
||||
expected.update({job: "success" if mode == "full" or (mode == "e2e" and job in E2E_JOBS) else "skipped" for job in CODE_JOBS})
|
||||
nightly = mode == "full" and event in ("schedule", "workflow_dispatch")
|
||||
expected.update({job: "success" if nightly else "skipped" for job in NIGHTLY_JOBS})
|
||||
expected["connect-feature-boundaries"] = "success" if nightly and connect else "skipped"
|
||||
expected.update({job: "success" if nightly else "skipped" for job in OPTIONAL_JOBS[:2]})
|
||||
expected["connect-feature-boundaries"] = "success" if mode == "full" and connect else "skipped"
|
||||
if event in ("pull_request", "push"):
|
||||
expected.update({job: "skipped" for job in EXTENDED_FEATURE_JOBS})
|
||||
rio = mode == "full" and event in ("schedule", "workflow_dispatch")
|
||||
expected.update({job: "success" if rio else "skipped" for job in OPTIONAL_JOBS[:2]})
|
||||
full = mode == "full" and (event in ("merge_group", "workflow_dispatch") or (event == "push" and ref == "refs/heads/main"))
|
||||
expected["e2e-full"] = "success" if full else "skipped"
|
||||
expected.update({job: "success" if rehearse_shards else "skipped" for job in SHARD_JOBS})
|
||||
@@ -200,8 +192,8 @@ def read_policy(root: Path, revision: str) -> dict:
|
||||
|
||||
def configurations(jobs: dict[str, str]) -> list[dict]:
|
||||
values = [
|
||||
("workspace-lint", "workspace-lint-and-build", "dev", []),
|
||||
("workspace-tests", "workspace-lint-and-build", "test", []),
|
||||
("workspace-lint", "test-and-lint", "dev", []),
|
||||
("workspace-tests", "test-and-lint", "test", []),
|
||||
("e2e-harness", "build-rustfs-debug-binary", "test", []),
|
||||
("uring-tests", "uring-integration", "test", []),
|
||||
("server-e2e-hooks", "build-rustfs-debug-binary", "dev", ["e2e-test-hooks"]),
|
||||
@@ -548,7 +540,7 @@ def check_workflow(root: Path) -> list[str]:
|
||||
or verifiers[0].get("shell") != "bash" or "if" in verifiers[0]):
|
||||
errors.append("required-checks must unconditionally verify actual needs and preserve its exit status")
|
||||
lint_commands = ("cargo clippy --all-targets -- -D warnings", "python3 scripts/ci_test_artifact.py cargo lint --configuration workspace")
|
||||
clippy = [step for step in jobs.get("workspace-lint-and-build", {}).get("steps", [])
|
||||
clippy = [step for step in jobs.get("test-and-lint", {}).get("steps", [])
|
||||
if any(command in step.get("run", "") for command in lint_commands)]
|
||||
if not clippy or any("if" in step for step in clippy):
|
||||
errors.append("workspace Clippy must run on every code event, pull requests included")
|
||||
@@ -628,8 +620,7 @@ def main() -> int:
|
||||
for name, value in (("mode", plan["mode"]), ("plan", json.dumps(plan, separators=(",", ":"))),
|
||||
("connect", str(plan["connect"]).lower()), ("packages", json.dumps(plan["packages"], separators=(",", ":"))),
|
||||
("full_validation", str(plan["full_validation"]).lower()),
|
||||
("rehearse_shards", str(plan["rehearse_shards"]).lower()),
|
||||
("plan_contract", PLAN_CONTRACT)):
|
||||
("rehearse_shards", str(plan["rehearse_shards"]).lower())):
|
||||
output.write(f"{name}={value}\n")
|
||||
print(json.dumps(plan, indent=2))
|
||||
if os.environ.get("GITHUB_STEP_SUMMARY"):
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,376 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Contract tests for scripts/release/release_gate.py (rustfs/backlog#2662).
|
||||
|
||||
Every fixture under testdata/release-gate/ is a contract rehearsal only: these
|
||||
tests prove the gate's fail-closed orchestration and must never be read as
|
||||
product release evidence or as R02 live verification results.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
GATE = ROOT / "scripts" / "release" / "release_gate.py"
|
||||
FIXTURES = ROOT / "testdata" / "release-gate"
|
||||
PLAN = FIXTURES / "plan.json"
|
||||
CANDIDATE = FIXTURES / "candidate.json"
|
||||
ARTIFACT = FIXTURES / "artifacts" / "rustfs-linux-x86_64-gnu-v9.9.9-fixture.zip"
|
||||
ARTIFACT_NAME = "rustfs-linux-x86_64-gnu-v9.9.9-fixture.zip"
|
||||
REAL_PLAN = ROOT / ".config" / "release-validation-plan.json"
|
||||
|
||||
|
||||
def run_gate(*arguments):
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(GATE), *map(str, arguments)],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def run_validate(results, output, plan=PLAN, candidate=CANDIDATE):
|
||||
return run_gate(
|
||||
"validate",
|
||||
"--plan", plan,
|
||||
"--candidate", candidate,
|
||||
"--results", results,
|
||||
"--output", output,
|
||||
)
|
||||
|
||||
|
||||
def run_promote(output, plan=PLAN, candidate=CANDIDATE, summary=None, artifact=ARTIFACT,
|
||||
artifact_name=ARTIFACT_NAME, artifact_sha256=None):
|
||||
arguments = [
|
||||
"verify-promotion",
|
||||
"--plan", plan,
|
||||
"--candidate", candidate,
|
||||
"--summary", summary or output,
|
||||
]
|
||||
if artifact is not None:
|
||||
arguments += ["--artifact", artifact]
|
||||
if artifact_name is not None:
|
||||
arguments += ["--artifact-name", artifact_name]
|
||||
if artifact_sha256 is not None:
|
||||
arguments += ["--artifact-sha256", artifact_sha256]
|
||||
if output is not None:
|
||||
arguments += ["--output", output]
|
||||
return run_gate(*arguments)
|
||||
|
||||
|
||||
def load_summary(output):
|
||||
return json.loads((Path(output) / "release-gate-summary.json").read_text())
|
||||
|
||||
|
||||
def finding_codes(summary):
|
||||
return {finding["code"] for finding in summary["findings"]}
|
||||
|
||||
|
||||
class GateFixtureTests(unittest.TestCase):
|
||||
"""Assertion matrix rows from rustfs/backlog#2662 exercised on the fixtures."""
|
||||
|
||||
def setUp(self):
|
||||
self._temporary = tempfile.TemporaryDirectory(prefix="release-gate-test.")
|
||||
self.addCleanup(self._temporary.cleanup)
|
||||
self.output = Path(self._temporary.name)
|
||||
|
||||
def validate_fixture(self, name, expect):
|
||||
output = self.output / name
|
||||
result = run_validate(FIXTURES / name, output)
|
||||
self.assertEqual(result.returncode, expect, result.stderr)
|
||||
return result, load_summary(output)
|
||||
|
||||
def test_all_required_pass_and_identity_consistent(self):
|
||||
# Matrix: all required PASS and identity consistent -> validate and
|
||||
# verify-promotion return 0.
|
||||
result, summary = self.validate_fixture("pass", 0)
|
||||
self.assertEqual(summary["state"], "pass")
|
||||
self.assertTrue(summary["publish_allowed"])
|
||||
self.assertEqual(len(summary["required_cases"]), 5)
|
||||
self.assertEqual(len(summary["observed"]), 5)
|
||||
self.assertEqual(summary["findings"], [])
|
||||
expansion = json.loads((self.output / "pass" / "plan-expansion.json").read_text())
|
||||
self.assertEqual(len(expansion["required_cases"]), 5)
|
||||
promote = run_promote(self.output / "pass")
|
||||
self.assertEqual(promote.returncode, 0, promote.stderr)
|
||||
|
||||
def test_cross_platform_same_case_and_split_shas_accepted(self):
|
||||
# Matrix: the same case name on two platforms, and a workflow SHA that
|
||||
# differs from the (legally bound) product SHA, must not be rejected.
|
||||
result, summary = self.validate_fixture("cross-platform", 0)
|
||||
self.assertEqual(summary["state"], "pass")
|
||||
product_commit = json.loads(CANDIDATE.read_text())["product_commit"]
|
||||
workflows = []
|
||||
for name in sorted((FIXTURES / "cross-platform").glob("*.json")):
|
||||
if name.name == "batch.json":
|
||||
continue
|
||||
envelope = json.loads(name.read_text())
|
||||
workflows.append(envelope["workflow"]["workflow_sha"])
|
||||
self.assertNotEqual(envelope["workflow"]["workflow_sha"], product_commit)
|
||||
self.assertEqual(len(workflows), 5)
|
||||
case_names = [entry["case_id"] for entry in summary["observed"]]
|
||||
self.assertEqual(case_names.count("startup"), 2)
|
||||
self.assertEqual(case_names.count("artifact-identity"), 2)
|
||||
|
||||
def test_missing_case_names_the_case(self):
|
||||
result, summary = self.validate_fixture("missing-case", 1)
|
||||
self.assertIn("missing_case", finding_codes(summary))
|
||||
missing = [f for f in summary["findings"] if f["code"] == "missing_case"]
|
||||
self.assertTrue(any(f.get("case_id") == "startup" for f in missing), missing)
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_duplicate_key_names_the_case(self):
|
||||
result, summary = self.validate_fixture("duplicate-key", 1)
|
||||
self.assertIn("duplicate_key", finding_codes(summary))
|
||||
duplicates = [f for f in summary["findings"] if f["code"] == "duplicate_key"]
|
||||
self.assertTrue(any(f.get("case_id") == "startup" for f in duplicates), duplicates)
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_extra_unplanned_case_names_the_case(self):
|
||||
result, summary = self.validate_fixture("extra-case", 1)
|
||||
self.assertIn("extra_case", finding_codes(summary))
|
||||
extra = [f for f in summary["findings"] if f["code"] == "extra_case"]
|
||||
self.assertTrue(any(f.get("case_id") == "surprise-case" for f in extra), extra)
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_non_pass_required_items_block(self):
|
||||
# Matrix: any skipped / infra_error / quarantined required item blocks
|
||||
# and no official publish write is allowed.
|
||||
for name in ("non-pass-skipped", "non-pass-infra-error", "non-pass-quarantined"):
|
||||
with self.subTest(name=name):
|
||||
result, summary = self.validate_fixture(name, 1)
|
||||
self.assertIn("non_pass", finding_codes(summary))
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_child_workflow_cancellation_blocks(self):
|
||||
result, summary = self.validate_fixture("child-cancelled", 1)
|
||||
self.assertIn("cancelled", finding_codes(summary))
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_batch_timeout_blocks(self):
|
||||
result, summary = self.validate_fixture("timeout", 1)
|
||||
self.assertIn("timeout", finding_codes(summary))
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_same_tag_different_bytes_is_rejected(self):
|
||||
result, summary = self.validate_fixture("same-tag-different-bytes", 1)
|
||||
self.assertIn("digest_mismatch", finding_codes(summary))
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_stale_attempt_cannot_be_reused(self):
|
||||
result, summary = self.validate_fixture("stale-attempt", 1)
|
||||
self.assertIn("stale_attempt", finding_codes(summary))
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_wrong_platform_is_rejected(self):
|
||||
result, summary = self.validate_fixture("wrong-platform", 1)
|
||||
self.assertIn("platform_mismatch", finding_codes(summary))
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_wrong_harness_sha_cannot_be_reused(self):
|
||||
result, summary = self.validate_fixture("wrong-harness", 1)
|
||||
self.assertIn("harness_mismatch", finding_codes(summary))
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_zero_tests_block(self):
|
||||
result, summary = self.validate_fixture("zero-tests", 1)
|
||||
self.assertIn("zero_tests", finding_codes(summary))
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_untrusted_source_blocks(self):
|
||||
# A PR-uploaded or foreign JSON set must never gain release authority.
|
||||
result, summary = self.validate_fixture("untrusted-source", 1)
|
||||
self.assertIn("untrusted_source", finding_codes(summary))
|
||||
self.assertFalse(summary["publish_allowed"])
|
||||
|
||||
def test_swapped_package_after_verification_fails_promotion(self):
|
||||
# Matrix: swapping a published package after verification must fail
|
||||
# verify-promotion.
|
||||
output = self.output / "certified"
|
||||
self.assertEqual(run_validate(FIXTURES / "pass", output).returncode, 0)
|
||||
promote = run_promote(
|
||||
output,
|
||||
artifact=FIXTURES / "promotion-swap" / ARTIFACT_NAME,
|
||||
summary=output,
|
||||
)
|
||||
self.assertEqual(promote.returncode, 1, promote.stdout)
|
||||
self.assertIn("artifact_mismatch", promote.stderr)
|
||||
report = json.loads((output / "promotion-check.json").read_text())
|
||||
self.assertEqual(report["state"], "blocked")
|
||||
self.assertEqual(finding_codes(report), {"artifact_mismatch"})
|
||||
|
||||
def test_edited_plan_after_verification_fails_promotion(self):
|
||||
# Matrix: editing the plan after verification must fail
|
||||
# verify-promotion; the old certification cannot be reused.
|
||||
output = self.output / "certified"
|
||||
self.assertEqual(run_validate(FIXTURES / "pass", output).returncode, 0)
|
||||
promote = run_promote(
|
||||
output,
|
||||
plan=FIXTURES / "promotion-edit" / "plan.json",
|
||||
summary=output,
|
||||
)
|
||||
self.assertEqual(promote.returncode, 1, promote.stdout)
|
||||
self.assertIn("plan_edited", promote.stderr)
|
||||
|
||||
def test_promotion_without_certification_fails(self):
|
||||
output = self.output / "blocked"
|
||||
self.assertEqual(run_validate(FIXTURES / "missing-case", output).returncode, 1)
|
||||
promote = run_promote(output, summary=output)
|
||||
self.assertEqual(promote.returncode, 1, promote.stdout)
|
||||
self.assertIn("certification_missing", promote.stderr)
|
||||
|
||||
def test_summary_junit_and_expansion_are_written(self):
|
||||
output = self.output / "pass"
|
||||
self.assertEqual(run_validate(FIXTURES / "pass", output).returncode, 0)
|
||||
summary = load_summary(output)
|
||||
self.assertEqual(summary["plan_sha256"], json.loads(CANDIDATE.read_text())["plan_sha256"])
|
||||
junit = (output / "release-gate.junit.xml").read_text()
|
||||
self.assertIn("release-gate", junit)
|
||||
self.assertIn("identity/linux-x86_64-gnu/startup", junit)
|
||||
markdown = (output / "release-gate-summary.md").read_text()
|
||||
self.assertIn("state: pass", markdown)
|
||||
|
||||
def test_error_messages_carry_no_credentials(self):
|
||||
for name in ("missing-case", "stale-attempt", "untrusted-source", "zero-tests"):
|
||||
output = self.output / name
|
||||
run_validate(FIXTURES / name, output)
|
||||
text = output.joinpath("release-gate-summary.json").read_text()
|
||||
text += (output / "release-gate-summary.md").read_text()
|
||||
lowered = text.lower()
|
||||
for marker in ("secret", "password", "access_key", "akia", "bearer "):
|
||||
self.assertNotIn(marker, lowered, f"{name} leaked a credential-shaped string")
|
||||
|
||||
|
||||
class PlanContractTests(unittest.TestCase):
|
||||
def test_real_plan_is_explicit_about_sources_and_rollback(self):
|
||||
sys.path.insert(0, str(GATE.parent))
|
||||
import release_gate
|
||||
|
||||
plan = json.loads(REAL_PLAN.read_text())
|
||||
info = release_gate.validate_plan(plan)
|
||||
sources = info["upgrade_sources"]
|
||||
self.assertGreaterEqual(len(sources["sources"]), 1)
|
||||
self.assertFalse(sources["rollback_supported"])
|
||||
self.assertTrue(sources["basis"].strip())
|
||||
self.assertTrue(sources["rollback_basis"].strip())
|
||||
for source in sources["sources"]:
|
||||
self.assertEqual(len(source["artifact_sha256"]), 64)
|
||||
|
||||
def test_plan_requires_trusted_workflows_and_frozen_harness(self):
|
||||
sys.path.insert(0, str(GATE.parent))
|
||||
import release_gate
|
||||
|
||||
plan = json.loads(REAL_PLAN.read_text())
|
||||
plan["trusted_workflows"] = []
|
||||
with self.assertRaises(release_gate.GateError):
|
||||
release_gate.validate_plan(plan)
|
||||
plan = json.loads(REAL_PLAN.read_text())
|
||||
plan["lanes"][0]["harness_sha"] = "not-a-sha"
|
||||
with self.assertRaises(release_gate.GateError):
|
||||
release_gate.validate_plan(plan)
|
||||
plan = json.loads(REAL_PLAN.read_text())
|
||||
plan["upgrade_sources"].pop("rollback_supported")
|
||||
with self.assertRaises(release_gate.GateError):
|
||||
release_gate.validate_plan(plan)
|
||||
|
||||
def test_plan_sha256_is_semantic_and_detects_required_set_edits(self):
|
||||
sys.path.insert(0, str(GATE.parent))
|
||||
import release_gate
|
||||
|
||||
plan = json.loads(PLAN.read_text())
|
||||
baseline = release_gate.canonical_sha256(plan)
|
||||
self.assertEqual(release_gate.canonical_sha256(json.loads(json.dumps(plan))), baseline)
|
||||
edited = json.loads(PLAN.read_text())
|
||||
edited["lanes"][0]["cases"] = ["artifact-identity"]
|
||||
self.assertNotEqual(release_gate.canonical_sha256(edited), baseline)
|
||||
|
||||
|
||||
class PublicationWiringTests(unittest.TestCase):
|
||||
"""Static checks that the real publication exits consume the gate."""
|
||||
|
||||
def read(self, relative):
|
||||
return (ROOT / relative).read_text()
|
||||
|
||||
def job_block(self, workflow, job):
|
||||
match = re.search(
|
||||
rf"^ {re.escape(job)}:\n(.*?)(?=^ [A-Za-z#])",
|
||||
workflow,
|
||||
re.S | re.M,
|
||||
)
|
||||
self.assertIsNotNone(match, f"job {job} not found")
|
||||
return match.group(1)
|
||||
|
||||
def test_build_publish_requires_certification(self):
|
||||
workflow = self.read(".github/workflows/build.yml")
|
||||
self.assertIn("\n release-certification:", workflow)
|
||||
publish_block = self.job_block(workflow, "publish-release")
|
||||
self.assertIn("release-certification", publish_block)
|
||||
self.assertIn("release_gate.py validate", workflow)
|
||||
self.assertIn("release_gate.py verify-promotion", workflow)
|
||||
|
||||
def test_build_time_official_r2_writes_are_staged(self):
|
||||
workflow = self.read(".github/workflows/build.yml")
|
||||
self.assertIn("artifacts/rustfs/staging/", workflow)
|
||||
release_writes = [
|
||||
line for line in workflow.splitlines()
|
||||
if "artifacts/rustfs/release/" in line
|
||||
]
|
||||
# Only the certification-gated promotion job may write the official path.
|
||||
self.assertTrue(release_writes, "the promotion job must write the official R2 path")
|
||||
promotion_block = self.job_block(workflow, "promote-release-artifacts")
|
||||
self.assertIn("verify-promotion", promotion_block)
|
||||
self.assertIn("artifacts/rustfs/release/", promotion_block)
|
||||
build_block = self.job_block(workflow, "build-rustfs")
|
||||
self.assertNotIn('R2_PATH="s3://${R2_BUCKET}/artifacts/rustfs/release/"', build_block)
|
||||
|
||||
def test_downstream_workflows_verify_certification(self):
|
||||
docker = self.read(".github/workflows/docker.yml")
|
||||
self.assertIn("\n release-certification:", docker)
|
||||
build_docker_block = self.job_block(docker, "build-docker")
|
||||
self.assertIn("release-certification", build_docker_block)
|
||||
self.assertIn("verify-promotion", docker)
|
||||
package = self.read(".github/workflows/package.yml")
|
||||
self.assertIn("\n release-certification:", package)
|
||||
package_block = self.job_block(package, "package")
|
||||
self.assertIn("release-certification", package_block)
|
||||
helm = self.read(".github/workflows/helm-package.yml")
|
||||
self.assertIn("\n release-certification:", helm)
|
||||
helm_block = self.job_block(helm, "build-helm-package")
|
||||
self.assertIn("release-certification", helm_block)
|
||||
|
||||
def test_manual_downstream_dispatch_without_certification_fails(self):
|
||||
# Every downstream certification job must fail closed when the trusted
|
||||
# certification artifact is absent (e.g. a manual dispatch without one).
|
||||
for relative in (
|
||||
".github/workflows/docker.yml",
|
||||
".github/workflows/package.yml",
|
||||
".github/workflows/helm-package.yml",
|
||||
):
|
||||
workflow = self.read(relative)
|
||||
certification_block = self.job_block(workflow, "release-certification")
|
||||
self.assertIn("fail", certification_block.lower(), relative)
|
||||
self.assertIn("release-certification", certification_block, relative)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -27,7 +27,7 @@ cleanup() {
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
|
||||
for command in curl docker python3 aws; do
|
||||
for command in curl docker python3 awscurl; do
|
||||
command -v "${command}" >/dev/null || {
|
||||
echo "missing required command: ${command}" >&2
|
||||
exit 1
|
||||
@@ -192,19 +192,12 @@ print("\t".join(values[field] for field in ("AccessKeyId", "SecretAccessKey", "S
|
||||
PY
|
||||
)
|
||||
|
||||
# Sign with the STS session to prove the web-identity credentials are usable.
|
||||
# The AWS CLI replaces awscurl here: the functional fleet's system Python is
|
||||
# PEP 668 managed, so the live gate must not require pip-installed tools. The
|
||||
# security suite already requires the AWS CLI. Strip proxy env explicitly
|
||||
# (the runners define it) so this localhost request never leaves the host.
|
||||
env -u http_proxy -u https_proxy -u HTTP_PROXY -u HTTPS_PROXY \
|
||||
AWS_ACCESS_KEY_ID="${STS_ACCESS_KEY}" \
|
||||
AWS_SECRET_ACCESS_KEY="${STS_SECRET_KEY}" \
|
||||
AWS_SESSION_TOKEN="${STS_SESSION_TOKEN}" \
|
||||
AWS_EC2_METADATA_DISABLED=true \
|
||||
aws --endpoint-url "${RUSTFS_ORIGIN}" --region us-east-1 \
|
||||
s3api list-buckets --output json >"${WORK_DIR}/list-buckets.json"
|
||||
grep -q '"Buckets"' "${WORK_DIR}/list-buckets.json"
|
||||
awscurl --fail-with-body --service s3 --region us-east-1 \
|
||||
--access_key "${STS_ACCESS_KEY}" \
|
||||
--secret_key "${STS_SECRET_KEY}" \
|
||||
--security_token "${STS_SESSION_TOKEN}" \
|
||||
"${RUSTFS_ORIGIN}/" >"${WORK_DIR}/list-buckets.xml"
|
||||
grep -q '<ListAllMyBucketsResult' "${WORK_DIR}/list-buckets.xml"
|
||||
|
||||
TAMPERED_TOKEN="$(python3 - "${GOOD_TOKEN}" <<'PY'
|
||||
import sys
|
||||
|
||||
+9
-29
@@ -116,7 +116,7 @@ class PlannerTests(unittest.TestCase):
|
||||
plan = self.repo.plan(self.repo.commit(), base=baseline)
|
||||
self.assertEqual(plan['packages'], ['consumer', 'core', 'leaf', 'rustfs'])
|
||||
self.assertEqual(plan['jobs']['connect-feature-boundaries'], 'skipped')
|
||||
for job in ('workspace-lint-and-build', 'workspace-tests', 'core-contract', 'e2e-tests',
|
||||
for job in ('test-and-lint', 'workspace-tests', 'core-contract', 'e2e-tests',
|
||||
's3-implemented-tests', 's3-lifecycle-behavior-tests'):
|
||||
self.assertEqual(plan['jobs'][job], 'success')
|
||||
|
||||
@@ -233,8 +233,7 @@ class PlannerTests(unittest.TestCase):
|
||||
result = subprocess.run([sys.executable, '-I', str(script), 'select'], cwd=self.repo.root, env=env, capture_output=True, text=True)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
values = dict(line.split('=', 1) for line in output.read_text().splitlines())
|
||||
self.assertEqual(set(values), {'mode', 'plan', 'connect', 'packages', 'full_validation', 'rehearse_shards', 'plan_contract'})
|
||||
self.assertEqual(values['plan_contract'], '2')
|
||||
self.assertEqual(set(values), {'mode', 'plan', 'connect', 'packages', 'full_validation', 'rehearse_shards'})
|
||||
plan = json.loads(values['plan'])
|
||||
self.assertEqual(plan['policy']['revision'], self.repo.base)
|
||||
self.assertEqual(plan['policy']['rollout'], 'shadow')
|
||||
@@ -296,48 +295,29 @@ class GateTests(unittest.TestCase):
|
||||
needs['classify-changes']['outputs'] = {'mode': 'full', 'plan': '{'}
|
||||
self.assertTrue(gate.verify_results(needs, 'pull_request', 'refs/heads/main'))
|
||||
|
||||
def test_plan_from_a_previous_contract_cannot_authorize_the_new_job_set(self):
|
||||
# A base planner predating the NIGHTLY_JOBS contract still emits
|
||||
# connect-feature-boundaries as required success on a pull request. The
|
||||
# current verifier must reject that plan instead of accepting the old
|
||||
# responsibility, so the workflow's contract-marker fallback is the only
|
||||
# path that can widen or narrow the required set.
|
||||
stale = self.plan()
|
||||
stale['jobs'] = dict(stale['jobs'], **{'connect-feature-boundaries': 'success',
|
||||
'uring-integration': 'success',
|
||||
'test-and-lint-rio-v2': 'success',
|
||||
'test-and-lint-protocols': 'success'})
|
||||
stale['not_applicable'] = {job: 'fixture' for job, state in stale['jobs'].items() if state == 'skipped'}
|
||||
expected = gate.expected_results('full', 'pull_request', 'refs/heads/main')
|
||||
self.assertEqual(expected['connect-feature-boundaries'], 'skipped')
|
||||
needs = {job: {'result': result} for job, result in expected.items()}
|
||||
needs['classify-changes']['outputs'] = {'mode': 'full', 'plan': json.dumps(stale, separators=(',', ':')),
|
||||
'connect': 'true', 'full_validation': 'true', 'rehearse_shards': 'false'}
|
||||
self.assertTrue(gate.verify_results(needs, 'pull_request', 'refs/heads/main'))
|
||||
|
||||
def workflow(self):
|
||||
jobs = {}
|
||||
for job in gate.ALWAYS_JOBS + gate.CODE_JOBS + gate.OPTIONAL_JOBS:
|
||||
# A complete plan lookup is represented by equivalent explicit expressions here.
|
||||
if job in gate.ALWAYS_JOBS:
|
||||
expression = "true"
|
||||
elif job == 'connect-feature-boundaries':
|
||||
expression = "needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect == 'true' && contains(fromJSON('[\"schedule\",\"workflow_dispatch\"]'), github.event_name)"
|
||||
elif job in gate.NIGHTLY_JOBS:
|
||||
expression = "needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('[\"schedule\",\"workflow_dispatch\"]'), github.event_name)"
|
||||
elif job in gate.EXTENDED_FEATURE_JOBS:
|
||||
expression = "needs.classify-changes.outputs.mode == 'full' && contains(fromJSON('[\"merge_group\",\"schedule\",\"workflow_dispatch\"]'), github.event_name)"
|
||||
elif job in gate.OPTIONAL_JOBS[:2]:
|
||||
expression = "contains(fromJSON('[\"schedule\",\"workflow_dispatch\"]'), github.event_name)"
|
||||
elif job == 'e2e-full':
|
||||
expression = "github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/main')"
|
||||
elif job in gate.SHARD_JOBS:
|
||||
expression = "needs.classify-changes.outputs.rehearse_shards == 'true'"
|
||||
elif job == 'connect-feature-boundaries':
|
||||
expression = "needs.classify-changes.outputs.mode == 'full' && needs.classify-changes.outputs.connect == 'true'"
|
||||
elif job in gate.E2E_JOBS:
|
||||
expression = "needs.classify-changes.outputs.mode != 'docs'"
|
||||
else:
|
||||
expression = "needs.classify-changes.outputs.mode == 'full'"
|
||||
expression = f"({expression}) && (github.event_name != 'pull_request' || github.event.action != 'closed')"
|
||||
jobs[job] = {'name': job, 'if': expression, 'steps': []}
|
||||
jobs['workspace-lint-and-build']['steps'] = [{'run': 'cargo clippy --all-targets -- -D warnings'}]
|
||||
jobs['test-and-lint']['steps'] = [{'run': 'cargo clippy --all-targets -- -D warnings'}]
|
||||
jobs['test-and-lint-protocols']['strategy'] = {'matrix': {'features': [
|
||||
{'name': 'swift', 'flags': '--features swift'}, {'name': 'sftp', 'flags': '--features sftp'}, {'name': 'http3', 'flags': '--features http3,swift'}]}}
|
||||
jobs['required-checks'] = {'name': 'Test and Lint', 'if': "always() && (github.event_name != 'pull_request' || github.event.action != 'closed')",
|
||||
@@ -407,7 +387,7 @@ class GateTests(unittest.TestCase):
|
||||
workflow['jobs']['required-checks']['needs'].reverse()
|
||||
for job in ('workspace-tests', 'core-contract', 'e2e-tests', 's3-implemented-tests', 's3-lifecycle-behavior-tests'):
|
||||
del workflow['jobs'][job]['if']
|
||||
workflow['jobs'][job]['needs'] = ['workspace-lint-and-build' if job == 'workspace-tests' else 'build-rustfs-debug-binary']
|
||||
workflow['jobs'][job]['needs'] = ['test-and-lint' if job == 'workspace-tests' else 'build-rustfs-debug-binary']
|
||||
self.assertEqual(self.check(workflow), [])
|
||||
for mutate in (
|
||||
lambda w: w['jobs']['required-checks']['needs'].pop(),
|
||||
@@ -415,7 +395,7 @@ class GateTests(unittest.TestCase):
|
||||
lambda w: w['jobs']['required-checks'].update({'if': "'always()' == 'always()'"}),
|
||||
lambda w: w['jobs']['required-checks']['steps'][0].update({'run': 'python3 scripts/ci_gate.py verify || true'}),
|
||||
lambda w: w['jobs']['typos'].update({'continue-on-error': 'true'}),
|
||||
lambda w: w['jobs']['workspace-lint-and-build']['steps'][0].update({'if': 'false'}),
|
||||
lambda w: w['jobs']['test-and-lint']['steps'][0].update({'if': 'false'}),
|
||||
lambda w: w['jobs']['test-and-lint-protocols']['strategy']['matrix']['features'].pop(),
|
||||
lambda w: w['jobs'].update({'unaccounted': {'name': 'New'}}),
|
||||
lambda w: w['on']['pull_request'].update({'paths': ['README.md']}),
|
||||
|
||||
Vendored
-29
@@ -1,29 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"version": "9.9.9-fixture",
|
||||
"release_profile": "stable",
|
||||
"features": [
|
||||
"pyroscope"
|
||||
],
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"validation_batch": {
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2
|
||||
},
|
||||
"deliverables": [
|
||||
{
|
||||
"kind": "zip",
|
||||
"name": "rustfs-linux-x86_64-gnu-v9.9.9-fixture.zip",
|
||||
"location": "https://example.invalid/artifacts/runs/fixture-batch-1/2/rustfs-linux-x86_64-gnu-v9.9.9-fixture.zip",
|
||||
"sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"size": 36,
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa"
|
||||
}
|
||||
],
|
||||
"image": {
|
||||
"ref": "example.invalid/rustfs:9.9.9-fixture",
|
||||
"digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"status": "completed"
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:02:00Z",
|
||||
"finished_at": "2026-09-26T10:02:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-aarch64-gnu.json",
|
||||
"sha256": "fa138d5891e5098b97137b0edbb4779c4350285fdc76cec7e6214809b34b3075",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:03:00Z",
|
||||
"finished_at": "2026-09-26T10:03:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-aarch64-gnu.json",
|
||||
"sha256": "314b19c4fee5f406cbf7ee0d6a436059115f1ff06f6864830045f9d452ab91c9",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:00:00Z",
|
||||
"finished_at": "2026-09-26T10:00:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-x86_64-gnu.json",
|
||||
"sha256": "37e58ec3fc6fe3ac7ff2fcbeebdb62f6a6aaaa34ec771a0f3306c821da0336f7",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:01:00Z",
|
||||
"finished_at": "2026-09-26T10:01:30Z",
|
||||
"state": "infra_error",
|
||||
"reason_code": "child-cancelled",
|
||||
"reason_class": "ENV",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-x86_64-gnu.json",
|
||||
"sha256": "8b8774175c55c400b337610a89fe725ad2637af8fcc4973a82ab218f9182b80d",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111",
|
||||
"status": "cancelled"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "upgrade",
|
||||
"case_id": "upgrade-smoke",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "2222222222222222222222222222222222222222",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:04:00Z",
|
||||
"finished_at": "2026-09-26T10:04:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/upgrade/upgrade-smoke-linux-x86_64-gnu.json",
|
||||
"sha256": "c1ce56808b52859d2590b34b5990aed7bf560b134e44bd0c09f1873c9f764b38",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-upgrade-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "2222222222222222222222222222222222222222"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"status": "completed"
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:02:00Z",
|
||||
"finished_at": "2026-09-26T10:02:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-aarch64-gnu.json",
|
||||
"sha256": "fa138d5891e5098b97137b0edbb4779c4350285fdc76cec7e6214809b34b3075",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-cross-identity-linux-aarch64-gnu-artifact-identity",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "4444444444444444444444444444444444444444"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:03:00Z",
|
||||
"finished_at": "2026-09-26T10:03:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-aarch64-gnu.json",
|
||||
"sha256": "314b19c4fee5f406cbf7ee0d6a436059115f1ff06f6864830045f9d452ab91c9",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-cross-identity-linux-aarch64-gnu-startup",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "4444444444444444444444444444444444444444"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:00:00Z",
|
||||
"finished_at": "2026-09-26T10:00:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-x86_64-gnu.json",
|
||||
"sha256": "37e58ec3fc6fe3ac7ff2fcbeebdb62f6a6aaaa34ec771a0f3306c821da0336f7",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-cross-identity-linux-x86_64-gnu-artifact-identity",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "4444444444444444444444444444444444444444"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:01:00Z",
|
||||
"finished_at": "2026-09-26T10:01:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-x86_64-gnu.json",
|
||||
"sha256": "8b8774175c55c400b337610a89fe725ad2637af8fcc4973a82ab218f9182b80d",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-cross-identity-linux-x86_64-gnu-startup",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "4444444444444444444444444444444444444444"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "upgrade",
|
||||
"case_id": "upgrade-smoke",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "2222222222222222222222222222222222222222",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:04:00Z",
|
||||
"finished_at": "2026-09-26T10:04:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/upgrade/upgrade-smoke-linux-x86_64-gnu.json",
|
||||
"sha256": "c1ce56808b52859d2590b34b5990aed7bf560b134e44bd0c09f1873c9f764b38",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-cross-upgrade-linux-x86_64-gnu-upgrade-smoke",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "4444444444444444444444444444444444444444"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"status": "completed"
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:02:00Z",
|
||||
"finished_at": "2026-09-26T10:02:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-aarch64-gnu.json",
|
||||
"sha256": "fa138d5891e5098b97137b0edbb4779c4350285fdc76cec7e6214809b34b3075",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:03:00Z",
|
||||
"finished_at": "2026-09-26T10:03:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-aarch64-gnu.json",
|
||||
"sha256": "314b19c4fee5f406cbf7ee0d6a436059115f1ff06f6864830045f9d452ab91c9",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:00:00Z",
|
||||
"finished_at": "2026-09-26T10:00:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-x86_64-gnu.json",
|
||||
"sha256": "37e58ec3fc6fe3ac7ff2fcbeebdb62f6a6aaaa34ec771a0f3306c821da0336f7",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:01:00Z",
|
||||
"finished_at": "2026-09-26T10:01:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-x86_64-gnu.json",
|
||||
"sha256": "8b8774175c55c400b337610a89fe725ad2637af8fcc4973a82ab218f9182b80d",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:01:00Z",
|
||||
"finished_at": "2026-09-26T10:01:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-x86_64-gnu.json",
|
||||
"sha256": "8b8774175c55c400b337610a89fe725ad2637af8fcc4973a82ab218f9182b80d",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "upgrade",
|
||||
"case_id": "upgrade-smoke",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "2222222222222222222222222222222222222222",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:04:00Z",
|
||||
"finished_at": "2026-09-26T10:04:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/upgrade/upgrade-smoke-linux-x86_64-gnu.json",
|
||||
"sha256": "c1ce56808b52859d2590b34b5990aed7bf560b134e44bd0c09f1873c9f764b38",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-upgrade-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "2222222222222222222222222222222222222222"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-6
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"status": "completed"
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:02:00Z",
|
||||
"finished_at": "2026-09-26T10:02:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-aarch64-gnu.json",
|
||||
"sha256": "fa138d5891e5098b97137b0edbb4779c4350285fdc76cec7e6214809b34b3075",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:03:00Z",
|
||||
"finished_at": "2026-09-26T10:03:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-aarch64-gnu.json",
|
||||
"sha256": "314b19c4fee5f406cbf7ee0d6a436059115f1ff06f6864830045f9d452ab91c9",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:00:00Z",
|
||||
"finished_at": "2026-09-26T10:00:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-x86_64-gnu.json",
|
||||
"sha256": "37e58ec3fc6fe3ac7ff2fcbeebdb62f6a6aaaa34ec771a0f3306c821da0336f7",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:01:00Z",
|
||||
"finished_at": "2026-09-26T10:01:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-x86_64-gnu.json",
|
||||
"sha256": "8b8774175c55c400b337610a89fe725ad2637af8fcc4973a82ab218f9182b80d",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "surprise-case",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:05:00Z",
|
||||
"finished_at": "2026-09-26T10:05:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/surprise-case-linux-x86_64-gnu.json",
|
||||
"sha256": "f82eedac16c983bd426ca24a6cbf3acfa032dc722e9c4a5ed7ef21c7eff0bcb1",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "upgrade",
|
||||
"case_id": "upgrade-smoke",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "2222222222222222222222222222222222222222",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:04:00Z",
|
||||
"finished_at": "2026-09-26T10:04:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/upgrade/upgrade-smoke-linux-x86_64-gnu.json",
|
||||
"sha256": "c1ce56808b52859d2590b34b5990aed7bf560b134e44bd0c09f1873c9f764b38",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-upgrade-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "2222222222222222222222222222222222222222"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"status": "completed"
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:02:00Z",
|
||||
"finished_at": "2026-09-26T10:02:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-aarch64-gnu.json",
|
||||
"sha256": "fa138d5891e5098b97137b0edbb4779c4350285fdc76cec7e6214809b34b3075",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:00:00Z",
|
||||
"finished_at": "2026-09-26T10:00:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-x86_64-gnu.json",
|
||||
"sha256": "37e58ec3fc6fe3ac7ff2fcbeebdb62f6a6aaaa34ec771a0f3306c821da0336f7",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:01:00Z",
|
||||
"finished_at": "2026-09-26T10:01:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-x86_64-gnu.json",
|
||||
"sha256": "8b8774175c55c400b337610a89fe725ad2637af8fcc4973a82ab218f9182b80d",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "upgrade",
|
||||
"case_id": "upgrade-smoke",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "2222222222222222222222222222222222222222",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:04:00Z",
|
||||
"finished_at": "2026-09-26T10:04:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/upgrade/upgrade-smoke-linux-x86_64-gnu.json",
|
||||
"sha256": "c1ce56808b52859d2590b34b5990aed7bf560b134e44bd0c09f1873c9f764b38",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-upgrade-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "2222222222222222222222222222222222222222"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"status": "completed"
|
||||
}
|
||||
Vendored
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:02:00Z",
|
||||
"finished_at": "2026-09-26T10:02:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-aarch64-gnu.json",
|
||||
"sha256": "fa138d5891e5098b97137b0edbb4779c4350285fdc76cec7e6214809b34b3075",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:03:00Z",
|
||||
"finished_at": "2026-09-26T10:03:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-aarch64-gnu.json",
|
||||
"sha256": "314b19c4fee5f406cbf7ee0d6a436059115f1ff06f6864830045f9d452ab91c9",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
Vendored
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:00:00Z",
|
||||
"finished_at": "2026-09-26T10:00:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-x86_64-gnu.json",
|
||||
"sha256": "37e58ec3fc6fe3ac7ff2fcbeebdb62f6a6aaaa34ec771a0f3306c821da0336f7",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:01:00Z",
|
||||
"finished_at": "2026-09-26T10:01:30Z",
|
||||
"state": "infra_error",
|
||||
"reason_code": "fixture-infra_error",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-x86_64-gnu.json",
|
||||
"sha256": "8b8774175c55c400b337610a89fe725ad2637af8fcc4973a82ab218f9182b80d",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "upgrade",
|
||||
"case_id": "upgrade-smoke",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "2222222222222222222222222222222222222222",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:04:00Z",
|
||||
"finished_at": "2026-09-26T10:04:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/upgrade/upgrade-smoke-linux-x86_64-gnu.json",
|
||||
"sha256": "c1ce56808b52859d2590b34b5990aed7bf560b134e44bd0c09f1873c9f764b38",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-upgrade-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "2222222222222222222222222222222222222222"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"status": "completed"
|
||||
}
|
||||
Vendored
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:02:00Z",
|
||||
"finished_at": "2026-09-26T10:02:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-aarch64-gnu.json",
|
||||
"sha256": "fa138d5891e5098b97137b0edbb4779c4350285fdc76cec7e6214809b34b3075",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:03:00Z",
|
||||
"finished_at": "2026-09-26T10:03:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-aarch64-gnu.json",
|
||||
"sha256": "314b19c4fee5f406cbf7ee0d6a436059115f1ff06f6864830045f9d452ab91c9",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
Vendored
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:00:00Z",
|
||||
"finished_at": "2026-09-26T10:00:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-x86_64-gnu.json",
|
||||
"sha256": "37e58ec3fc6fe3ac7ff2fcbeebdb62f6a6aaaa34ec771a0f3306c821da0336f7",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "startup",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:01:00Z",
|
||||
"finished_at": "2026-09-26T10:01:30Z",
|
||||
"state": "quarantined",
|
||||
"reason_code": "fixture-quarantined",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/startup-linux-x86_64-gnu.json",
|
||||
"sha256": "8b8774175c55c400b337610a89fe725ad2637af8fcc4973a82ab218f9182b80d",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "upgrade",
|
||||
"case_id": "upgrade-smoke",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "2222222222222222222222222222222222222222",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-x86_64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:04:00Z",
|
||||
"finished_at": "2026-09-26T10:04:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/upgrade/upgrade-smoke-linux-x86_64-gnu.json",
|
||||
"sha256": "c1ce56808b52859d2590b34b5990aed7bf560b134e44bd0c09f1873c9f764b38",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-upgrade-linux-x86_64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "2222222222222222222222222222222222222222"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"status": "completed"
|
||||
}
|
||||
-42
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"evidence_profile": "release-artifact",
|
||||
"run_id": "fixture-batch-1",
|
||||
"run_attempt": 2,
|
||||
"plan_sha256": "578bb57e65df8ba7a63e40aafc254fa644163270b7ec288fe3368bcd7c10ff86",
|
||||
"tier": "release-certification",
|
||||
"lane": "identity",
|
||||
"case_id": "artifact-identity",
|
||||
"product_commit": "3333333333333333333333333333333333333333",
|
||||
"harness_commit": "1111111111111111111111111111111111111111",
|
||||
"target_fingerprint": "rustfs-9.9.9-fixture-linux-aarch64-gnu",
|
||||
"platform": {
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
"started_at": "2026-09-26T10:02:00Z",
|
||||
"finished_at": "2026-09-26T10:02:30Z",
|
||||
"state": "pass",
|
||||
"reason_code": "ok",
|
||||
"reason_class": "NOT_APPLICABLE",
|
||||
"evidence": [
|
||||
{
|
||||
"path": "cases/identity/artifact-identity-linux-aarch64-gnu.json",
|
||||
"sha256": "fa138d5891e5098b97137b0edbb4779c4350285fdc76cec7e6214809b34b3075",
|
||||
"size": 64
|
||||
}
|
||||
],
|
||||
"workflow": {
|
||||
"repository": "rustfs/s3chaos",
|
||||
"workflow_id": "release-gate.yml",
|
||||
"run_id": "child-identity-linux-aarch64-gnu",
|
||||
"run_attempt": 1,
|
||||
"workflow_sha": "1111111111111111111111111111111111111111"
|
||||
},
|
||||
"artifact": {
|
||||
"archive_sha256": "ea3ac1fb0ec4aa8fbec650f64ccdbc2543843c8d5734c3915d1b57d7cbb92ec8",
|
||||
"binary_sha256": "7b77e78cc7b8f63a60f8a7101bebed6502f82e03016f4d4e11450f927e0a52fa",
|
||||
"container_image_digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555"
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user