mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-12 05:04:18 +00:00
Compare commits
83 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 443e8afee1 | |||
| 782e3e5209 | |||
| cfffb1700e | |||
| cd62f3ba4f | |||
| 7bc419e368 | |||
| 7269243de7 | |||
| c4a7423f99 | |||
| 2db4ed244a | |||
| 93a61df1cf | |||
| feec0e9e4a | |||
| 38baeeb11d | |||
| 1af039c972 | |||
| a2fc7c13b4 | |||
| dbe01e5cb1 | |||
| ffc27fa6db | |||
| 7e8edab0d3 | |||
| 22057ad4c5 | |||
| 279a88ab56 | |||
| b4abe64ada | |||
| add363f084 | |||
| 1f2dbe36e5 | |||
| 67fbafa13e | |||
| 1000860cb8 | |||
| b783f404ad | |||
| 11a991c777 | |||
| 3f2b6f690a | |||
| 88b626e7a0 | |||
| 8cf6187ff4 | |||
| a0a0cee9eb | |||
| 80fff4b5c5 | |||
| d91dfcedd2 | |||
| 79c4fedf8c | |||
| 73c846e09c | |||
| 6938c467d0 | |||
| 5323df7049 | |||
| 55b6fbd260 | |||
| dd677cc121 | |||
| 0c2f327974 | |||
| df42cf1ce8 | |||
| d290d46b34 | |||
| 0d297ed8cd | |||
| b8dffa2f1d | |||
| 62b6f5f3fc | |||
| be0986115c | |||
| 1bd91c8dad | |||
| dcb22bd4d5 | |||
| fa17718444 | |||
| 99abffc57a | |||
| fc061c7930 | |||
| d86bf1360f | |||
| f6097d1df2 | |||
| 7e5d32eecf | |||
| 72da0e21c4 | |||
| 78a0cf321c | |||
| cbe38a120f | |||
| db0828aa1c | |||
| 3ca889b099 | |||
| e6112c526f | |||
| c5967686f1 | |||
| 859de565ad | |||
| 1f7cc593e7 | |||
| 1d5a50f094 | |||
| 53245be9f4 | |||
| c47432e353 | |||
| f622c1bce0 | |||
| a99d8f4f89 | |||
| 4d15b81623 | |||
| 836d30192f | |||
| 860c61ee6d | |||
| f600af4704 | |||
| 9e747e8475 | |||
| 96fc067765 | |||
| e809b4899e | |||
| 584f03f300 | |||
| d52d6fade7 | |||
| 3623d6db2d | |||
| 484f1254ca | |||
| c9c16efbb6 | |||
| f2a59073ca | |||
| c1af94d1d1 | |||
| f84bbfe28f | |||
| 378a2ff7f8 | |||
| b88b53aa6a |
@@ -1,38 +1,43 @@
|
||||
---
|
||||
name: rustfs-release-publish
|
||||
description: Publish a RustFS version through frozen Linux candidate acceptance, desktop completion, manifest approval, byte-preserving publication and post-release follow-up. Use only for an explicit release request.
|
||||
description: Publish a RustFS version through Console, release-branch CI, preview acceptance, human confirmation, and final publication. Use only for an explicit release request.
|
||||
---
|
||||
# RustFS Release
|
||||
|
||||
Require the exact final SemVer target before version edits or publication. If unspecified, inspect releases/tags and ask with concrete candidates; do not guess the channel or bump. Reuse the user's confirmed target and authorized delivery scope. Independent read-only preflight may continue.
|
||||
Require the exact final SemVer target before version edits or publication. If unspecified, inspect current releases/tags and ask with concrete candidates; do not guess the channel or bump. A named bump type resolves to an exact version that still needs confirmation. Report the confirmed target verbatim; resolve any later mismatch before proceeding. Independent read-only preflight may continue.
|
||||
|
||||
## Invariants
|
||||
|
||||
- Cargo and every candidate executable contain the final product version. `<target>-preview.<run_number>` is an immutable candidate identifier, not a public tag or displayed binary version.
|
||||
- Freeze the exact source, reviewed workflow recipe, toolchain, Console, CLI, dependency lock, previous stable version and validation plan. Newer unrelated main or Console releases do not change those inputs.
|
||||
- Retain `release/<target>` history when a release branch is used; backport only reviewed fixes. Pin `CANDIDATE_SHA` to its exact CI-validated commit. Never force-push a release branch or move a public tag.
|
||||
- Build Linux first. Complete field acceptance before macOS/Windows and packaging. Full certification and final acceptance are required before publication; a partial Linux PASS grants no release authority.
|
||||
- Promote the accepted bytes. Final publication never recompiles binaries or rebuilds packages/images. Infrastructure retries reuse successful artifacts; changed source or inputs require a new candidate and affected acceptance.
|
||||
- Phase failures block dependents. No latest/default-channel update before versioned delivery. Use the [pipeline runbook](../../../docs/operations/release-pipeline.md) for executable entry points, locks and recovery.
|
||||
- Cargo stores `<target>`, never `-preview.N`. Before publication, installation defaults retain the previous available deliverable. Final and preview binaries embed their own tag names, so version/asset names differ despite identical source.
|
||||
- Use annotated tags without `v`. Preview names are exactly `<target>-preview.<digits>`; choose the next unused iteration after fetching tags. Preview Releases are prereleases, never Latest; publish only versioned assets/exact Docker tags, never latest assets, R2, Helm, moving Docker channels, or installation/banner updates.
|
||||
- `release/<target>` owns the candidate. Retain its history, backport only reviewed fixes, and never merge main wholesale or force-push it. Pin `PREVIEW_HASH` to its exact commit with complete passing CI. Preview and final tags both point there; never tag a later main/branch HEAD or insert another bump.
|
||||
- Phase failures block dependents. Keep preview Releases until final publication and notes generation finish; CI removes Releases while retaining tags. Never move an existing preview or final tag.
|
||||
|
||||
## Read by phase
|
||||
## Read by Phase
|
||||
|
||||
Read only the current phase's procedure; retain completed evidence across turns. Follow root verification, Git/PR, and authorization rules.
|
||||
|
||||
| Phase | Procedure |
|
||||
|---|---|
|
||||
| 0 | Clean tree, fetch main/tags, verify access, then complete the mandatory [Console gate](references/console-gate.md). |
|
||||
| 1–2 | [Prepare the exact source and run branch CI](references/release-candidate.md). Version edits use `rustfs-release-version-bump`. |
|
||||
| 3 | [Build and verify the Linux candidate](references/preview-release.md). |
|
||||
| 4–5 | [Run artifact, field, Console and rc acceptance](references/preview-acceptance.md), then complete desktop/package/image/chart validation. |
|
||||
| Confirmation | Report the complete manifest and obtain the confirmation below. |
|
||||
| 6 | [Publish the accepted manifest](references/final-release.md). |
|
||||
| 7 | [Reconcile advisories, milestones, installation references, website and release branch](references/post-release-updates.md). |
|
||||
| 0 | Clean tree, fetch main/tags, verify GitHub access, then complete the mandatory [Console release gate](references/console-gate.md). A green Console build alone is insufficient. |
|
||||
| 1–2 | [Prepare/reuse the release candidate and run exact-SHA branch CI](references/release-candidate.md). Source version changes use `rustfs-release-version-bump` with authorized delivery scope. |
|
||||
| 3 | [Verify the preview build, assets, channels, and notes](references/preview-release.md). |
|
||||
| 4–5 | [Run the downloaded binary, Console CRUD, and latest-rc matrix](references/preview-acceptance.md). Every check must pass. |
|
||||
| Confirmation | Complete the gate below and end the turn. |
|
||||
| 6 | After fresh confirmation, [publish and verify the final tag](references/final-release.md) at `PREVIEW_HASH`. |
|
||||
| 7 | [Reconcile Security Advisories, milestones, installation references, and the website; reconcile main and remove the completed release branch](references/post-release-updates.md) only after complete publication. |
|
||||
|
||||
## Human confirmation
|
||||
## Human Confirmation
|
||||
|
||||
After all acceptance passes, report the target, source SHA, source CI run, Linux and completion run URLs, final manifest SHA-256, accepted Console/CLI identities, compatibility/performance results and backlog report. Explicitly ask whether to publish this exact manifest and end the turn without dispatching publication.
|
||||
After Phases 3–5 pass, recheck that the release branch still equals `PREVIEW_HASH` and that the current Console asset tag/digest match the preview's recorded build inputs. Report the target, branch, preview tag/hash, branch CI run, preview Release URL, accepted Console asset identity/result, and rc matrix. Explicitly ask whether to publish the final tag and end the turn without creating/pushing it.
|
||||
|
||||
Only a new affirmative reply to that exact report authorizes Phase 6. An earlier general release request, silence or automatic continuation does not count. Reuse confirmation for retries of that same manifest; do not ask again merely because a destination failed. A changed source, input, manifest or acceptance result invalidates it and requires renewed acceptance/confirmation.
|
||||
Only a new affirmative user reply to that exact report authorizes Phase 6. The original release request, earlier confirmation, silence, or automated follow-up does not count. Confirmation binds the target, preview tag, hash, and accepted build inputs; a changed value or repeated/failed acceptance cycle invalidates it. Revalidate and obtain fresh confirmation before continuing.
|
||||
|
||||
## Recovery and reporting
|
||||
## Recovery
|
||||
|
||||
Resume failed infrastructure jobs without rebuilding successful units. Fix product defects on main, backport when needed, and start a new candidate. After a final tag exists, retry the same bytes or use a new patch version; never move the tag. Do not conflate implemented, CI-passed, field-accepted, approved, published and post-release-complete states. Report incomplete work and deviations with evidence; a tag or green build alone is not a completed release.
|
||||
- Before the final tag exists, land fixes on main, backport to the candidate, verify target versions, and restart branch CI/acceptance. Use a new preview iteration once a preview was tagged; main may advance independently.
|
||||
- If only an external build input changed, record it, create a new preview iteration even when the source SHA is unchanged, and repeat the affected build and artifact acceptance before fresh confirmation. Reuse source checks whose inputs remain valid.
|
||||
- Once the final tag exists, retry publication jobs for that exact tag. A source fix requires a newly confirmed target, not retagging or a newer preview. Phase 7 retries recheck artifact availability and superseding releases.
|
||||
- If abandoned after the bump merged, report the unpublished version on main and record whether to revert or let the next release overwrite it. Deleting unpublished candidate branches or preview tags requires an explicit decision. Completed release branches follow Phase 7 cleanup; keep release tags.
|
||||
|
||||
Report completed phase results and incomplete work with evidence: Console previous/latest tags, release need, hash and run/Release URLs; target, branch CI/SHA, previews and hash, confirmation state; preview/final assets, latest-channel state, Console/rc results and cleanup; advisory reconciliation results and pending disclosure decisions; both milestone states, installation PR checks/merge state, and website text/link/PR/deployment state. Keep artifact publication separate from follow-up delivery; do not call an open PR a live update. Report deviations and their authorization.
|
||||
|
||||
@@ -4,7 +4,7 @@ Read during Phase 0, before changing RustFS version files or tags. Follow the pa
|
||||
|
||||
### Console release gate
|
||||
|
||||
Complete this gate before changing any RustFS version file or creating any RustFS tag. Candidate creation resolves `repos/rustfs/console/releases/latest` once and freezes the exact asset for every build, so a successful Console build alone is insufficient. After freezing, later Console releases do not silently change or invalidate the accepted inputs.
|
||||
Complete this gate before changing any RustFS version file or creating any RustFS tag. RustFS `build.yml` downloads the asset returned by `repos/rustfs/console/releases/latest`, so a successful Console build alone is insufficient.
|
||||
|
||||
1. Read the latest published Console tag and compare it with Console `main`:
|
||||
|
||||
|
||||
@@ -1,13 +1,22 @@
|
||||
# Final release publication
|
||||
# Final Release Publication
|
||||
|
||||
Read only after fresh confirmation of the complete candidate report. The [parent confirmation gate](../SKILL.md#human-confirmation) applies.
|
||||
Read only after fresh user confirmation of the preview acceptance report. The [parent confirmation gate](../SKILL.md#human-confirmation) applies.
|
||||
|
||||
Dispatch `release-publish.yml` on main with the successful completion run ID and approved `release-manifest.json` SHA-256. Do not create the tag manually. The workflow revalidates source, bytes and trusted certification, then creates the annotated final tag at `CANDIDATE_SHA`. A recipe change fails closed and requires a new candidate; unrelated main changes do not require a rebuild.
|
||||
## Phase 6 — Publish the final tag on the validated commit
|
||||
|
||||
Require both versioned publication and channel publication jobs to succeed. Verify the final tag's commit and manifest identity; all six platform ZIPs and compatibility aliases; DEB/RPM, checksums, SBOM, provenance and evidence bundle; both multi-architecture image variants in Docker Hub, GHCR and Quay; and the accepted Helm chart in the chart repository. Download a host-compatible final artifact and confirm its digest is exactly the accepted candidate digest and its version/source output agrees.
|
||||
No second source version bump. Before creating the final tag, recheck the remote `release/<target>` HEAD equals `PREVIEW_HASH` and the current Console release tag/asset digest still match the accepted preview inputs. If either changed after confirmation, the approval no longer applies; validate the changed inputs with a new preview and obtain fresh confirmation. The final tag goes on the exact commit the preview validated:
|
||||
|
||||
For stable releases verify GitHub Latest, R2 download aliases and `latest.json` unless a newer stable release already owns those channels. Prereleases must remain GitHub prereleases and leave stable pointers unchanged. Verify generated release notes compare against the frozen previous stable release, excluding internal previews.
|
||||
```bash
|
||||
git fetch origin --tags
|
||||
git rev-parse "<preview-tag>^{commit}" # must equal PREVIEW_HASH — abort if not
|
||||
git tag -a "<target>" -m "Release <target>" "$PREVIEW_HASH"
|
||||
git push origin "<target>"
|
||||
```
|
||||
|
||||
On infrastructure failure retry the same workflow/seal; matching destinations are reused. Never clobber an asset, force a tag, rebuild the image or silently accept a changed digest. A source fix after the final tag requires a new patch version. Cross-registry writes are not atomic: report the unfinished destination rather than claiming publication complete.
|
||||
- CI rebuilds from the same source with a different tag name, so the binary self-reports `<target>`. Other inputs can change: `build.yml` reads Console `releases/latest` during each build. Compare the actual Console tag, URL, and verified digest in every final build job with the accepted preview record, and inspect available build provenance for other relevant changes. A pre-tag check cannot prevent a later input change. On a mismatch or missing evidence, repeat affected acceptance against the final artifacts and report the limitation; failed acceptance leaves release validation incomplete even if publication already succeeded. Never move the final tag to conceal a failure.
|
||||
- Verify the final tag's complete publication path: all matrix and release jobs green; `gh release view "<target>"` shows the full versioned and `-latest` asset set plus checksums, SBOM, and provenance; Docker and Helm workflows succeed; `latest.json` points to `<target>`. A stable target must have `isPrerelease=false` and `isLatest=true`. An alpha/beta/rc target must have `isPrerelease=true`; GitHub does not permit prereleases to be Latest, but the project `latest.json` still advances to the final non-preview target.
|
||||
- Verify the final Release body contains `## What's Changed` and a Full Changelog link. When `PREVIOUS_DELIVERABLE` exists, the link MUST be `https://github.com/rustfs/rustfs/compare/<PREVIOUS_DELIVERABLE>...<target>` and the baseline MUST equal the preview Release baseline; for example, both `1.0.0-beta.12-preview.1` and `1.0.0-beta.12` compare from `1.0.0-beta.11`.
|
||||
- Verify the preview cleanup: `cleanup-preview-releases` must succeed and `gh release view "<preview-tag>"` must then report `release not found` for every preview iteration of this target. Each surviving preview tag must still resolve to its own recorded commit; the accepted preview resolves to `PREVIEW_HASH`. If the job failed, delete the leftover Releases manually with `gh release delete "<preview-tag>" --yes` and report it.
|
||||
- Verify a host-compatible final artifact's checksum and `./rustfs --version`; it must report `<target>` and the expected source SHA. This identity check does not replace affected acceptance when build inputs changed.
|
||||
|
||||
New candidates create no preview Releases to clean up. For a legacy preview release, preserve its tag and remove its Release only after final delivery and notes are verified. Do not pass `--cleanup-tag`. Complete the [post-release follow-up](post-release-updates.md) separately.
|
||||
Never pass `--cleanup-tag` when removing preview Releases. Remove them manually only if `cleanup-preview-releases` failed, after final publication and notes generation. Record deleted Releases and surviving tags.
|
||||
|
||||
@@ -24,7 +24,7 @@ Reconcile advisories after every completed final release, including Console-only
|
||||
|
||||
1. Re-read the published deliverable state before edits or a retry. If a newer release has superseded this target, do not downgrade installation references or the website announcement. Report the superseding release and leave those defaults intact.
|
||||
2. Invoke `rustfs-release-version-bump` with stage `post-release`, the exact target, and the authorized delivery scope. Prepare the follow-up on current main; do not edit or move either validated release tag. Cargo versions may already be preparing the next target and must remain untouched.
|
||||
3. Verify the exact Docker image manifest, Release download links, RPM source archive, and Helm repository entry before publishing the installation PR. The chart is packaged and tested before sealing; publication uploads those same bytes only after versioned Docker images are available.
|
||||
3. Verify the exact Docker image manifest, Release download links, RPM source archive, and Helm repository entry before publishing the installation PR. The chart workflow overrides chart/app versions from the final tag, pins the source checkout to that release, and waits for its default Docker image before packaging.
|
||||
4. Report the installation PR and its merge state separately from the completed release. Reuse existing merge authorization and repository checks; do not infer new merge authority from this reference.
|
||||
|
||||
## rustfs.com top banner
|
||||
@@ -54,4 +54,4 @@ Use the existing announcement configuration from [website PR #112](https://githu
|
||||
1. Compare the exact published tag with current main. Audit every release-side commit since their merge base, including inherited backports. An ahead count is a history difference, not proof of missing changes. Use ancestry and patch equivalence, then inspect current code and regression evidence for squashed, adapted, superseded, or reverted changes. Record each commit as present, superseded, release-only, or missing; a positive `git cherry` result alone does not prove a missing fix.
|
||||
2. Bring missing shared changes to main through a reviewed PR and the required checks before closing the follow-up. Preserve main's later changes; decide version bumps and pinned Console configuration separately. If a history merge is needed, use a merge commit and verify the release commit is an ancestor of main; squash does not establish that ancestry. Do not merge main into the published release branch or move its tags. Report unresolved changes instead of claiming reconciliation is complete.
|
||||
3. After publication and authorized follow-ups are complete, delete the remote `release/<target>` branch. Immediately before deletion, verify its HEAD still equals the published tag's commit and no open PR targets or originates from it. Preserve a branch with newer work or open PRs and report the blocker. Use a deletion with an expected-old-SHA check to avoid removing a concurrently advanced branch.
|
||||
4. Verify the remote branch is absent and the final tag and any legacy preview tags still resolve to their recorded commits. Remove a local release branch only if it is task-owned, unchecked out, and has no unpreserved work. Record cleanup separately from artifact publication; an explicitly authorized deletion before reconciliation does not make unresolved main changes complete.
|
||||
4. Verify the remote branch is absent and the final and preview tags still resolve to their recorded commits. Remove a local release branch only if it is task-owned, unchecked out, and has no unpreserved work. Record cleanup separately from artifact publication; an explicitly authorized deletion before reconciliation does not make unresolved main changes complete.
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
# Candidate Artifact Acceptance
|
||||
# Preview Artifact Acceptance
|
||||
|
||||
Read after Phase 3 succeeds. Complete every check below before the parent skill's manual confirmation gate. These checks cover the downloaded artifact, embedded Console, and the rc release frozen in the candidate inputs.
|
||||
Read after Phase 3 succeeds. Complete every check below before the parent skill's manual confirmation gate. These checks cover the downloaded artifact, embedded Console, and latest rc client.
|
||||
|
||||
## Phase 4 — Run the artifact locally, verify the console
|
||||
|
||||
Use a unique task-owned scratch directory. Select the versioned archive matching the acceptance host, and verify it against the sealed manifest before extraction. Before desktop completion, use the Linux acceptance environment rather than rebuilding for the local desktop. Do not substitute a different build when a matching artifact is unavailable; report that acceptance gap. The shell example below assumes the verified executable is at `RUSTFS_BINARY` (`rustfs.exe` on Windows), `SCRATCH` is the task directory, and `S3_PORT` and `CONSOLE_PORT` are distinct available ports. Use equivalent native commands when the shell differs.
|
||||
Use a unique task-owned scratch directory. Select the versioned archive matching the host OS, architecture, and runtime, download its checksum file, and verify the archive before extraction. Do not substitute a different build when a matching artifact is unavailable; report that acceptance gap. The shell example below assumes the verified executable is at `RUSTFS_BINARY` (`rustfs.exe` on Windows), `SCRATCH` is the task directory, and `S3_PORT` and `CONSOLE_PORT` are distinct available ports. Use equivalent native commands when the shell differs.
|
||||
|
||||
Bind both listeners to loopback. Choose task-only credentials, set loopback proxy exclusions for the server/client processes, and record the server PID. A port probe does not reserve a port; retry startup with fresh ports on a bind conflict rather than connecting to an existing service.
|
||||
|
||||
@@ -24,16 +24,16 @@ PREVIEW_SERVER_PID=$!
|
||||
|
||||
Checks (all must pass):
|
||||
|
||||
- `"$RUSTFS_BINARY" --version` reports the exact final `<target>` and full `CANDIDATE_SHA`. A preview suffix, another version or another source is a failure.
|
||||
- `"$RUSTFS_BINARY" --version` reports the preview tag name and the short SHA of `PREVIEW_HASH`. Reporting `<target>` without the `-preview.N` suffix means the build did not embed the tag — treat as FAIL and investigate before proceeding.
|
||||
- Wait with a bounded readiness deadline while checking the recorded server process remains alive; `curl --noproxy '*' -fsS "$S3_ENDPOINT/health/ready"` must return ready from this process's listener.
|
||||
- Startup log shows the embedded console being served (this was the regression that `fix(release): require embedded console assets` guards).
|
||||
- Open `CONSOLE_ENDPOINT` in the browser and log in with the task credentials. The dashboard must render without JS console errors; create a bucket, upload a file, download it back (byte-identical), and delete the object and bucket. Keep the server running for Phase 5.
|
||||
|
||||
## Phase 5 — Validate with the pinned rc client
|
||||
## Phase 5 — Validate with the latest rc client
|
||||
|
||||
`rc` is the RustFS CLI client from <https://github.com/rustfs/cli>.
|
||||
|
||||
- Read the CLI release and asset digests from `inputs.json`; verify the local client's version or download a host-compatible asset from that exact release into the task directory. If the host is a desktop, resolve its asset within the same pinned CLI release and record the digest; do not independently follow `latest`. Set `RC_BIN` to that executable and record its version/digest. Avoid changing the user's installed client for this check.
|
||||
- Resolve the latest release with `gh api repos/rustfs/cli/releases/latest --jq .tag_name`; verify the local client's version or download and verify a host-compatible client into the task directory. Set `RC_BIN` to that executable and record its version/digest. Avoid changing the user's installed client for this check.
|
||||
- Inspect the client's current help for an isolated configuration option and use it when supported. Otherwise choose a unique `PREVIEW_ALIAS` and confirm it is absent before creating it; never overwrite an existing alias. Use task-owned bucket/user names and input/output files. The names below are safe only inside the newly started isolated server.
|
||||
- Point the client at `S3_ENDPOINT` with loopback proxy exclusions and run the command matrix, recording PASS/FAIL per command:
|
||||
|
||||
@@ -60,9 +60,3 @@ cmp <local-file> "$SCRATCH/out"
|
||||
```
|
||||
|
||||
- Any FAIL blocks acceptance. On success or failure, stop only the recorded task process and remove only task-created aliases/data after retaining the needed failure evidence. Do not erase an existing client configuration or unrelated server data.
|
||||
|
||||
## Field acceptance and desktop completion
|
||||
|
||||
Before accepting the Linux manifest, record Scanner/Heal, multi-pool, memory, major features, high-concurrency 503, performance and previous-version integrity/upgrade results in backlog. Preserve the release-specific agreed limits and baseline; a PASS label without artifact identities or test results does not satisfy acceptance.
|
||||
|
||||
Dispatch `release-complete.yml` only after that report passes, using the exact Linux manifest digest and backlog URL. Require native macOS/Windows checks, package extraction, image startup/scans, chart tests and complete certification to pass. Complete any desktop Console/client checks applicable to the release against these same archives. Record the completion run and final manifest SHA-256 before entering the parent confirmation gate. Partial Linux acceptance does not authorize final publication.
|
||||
|
||||
@@ -1,9 +1,25 @@
|
||||
# Linux candidate verification
|
||||
# Preview Release Verification
|
||||
|
||||
Read for Phase 3 after the source passed exact-SHA CI. The [parent invariants](../SKILL.md) apply; commands and recovery live in the [pipeline runbook](../../../../docs/operations/release-pipeline.md).
|
||||
Read for Phase 3 after the candidate passed exact-SHA CI. The [parent release invariants](../SKILL.md) apply.
|
||||
|
||||
Dispatch `release-candidate.yml` on main with the confirmed final version and `CANDIDATE_SHA`. Record the run ID, frozen `inputs.json`, every Linux archive digest and `linux-manifest.json` SHA-256. Require the four Linux builds, native startup/S3 checks, previous-release upgrade matrix and Linux seal to succeed. Missing, skipped, cancelled or failed required work is incomplete acceptance.
|
||||
## Preview channels
|
||||
|
||||
Verify the frozen Console identity equals the one accepted in Phase 0. Every matrix job consumes that same digest and exact toolchain. The binary must report the final product version and source commit; its candidate ID is separate. No public tag, Release, registry tag, R2 release path, Helm repository or latest pointer should change during this stage.
|
||||
- Use `<target>-preview.N` for every target, e.g. `1.0.0-beta.10-preview.3` or `1.1.0-preview.1`.
|
||||
- The canonical suffix is exactly `-preview.<digits>`. `build.yml` recognizes it before alpha/beta/rc classification and routes it to the preview-only path; any other tag containing `-preview` fails closed instead of being treated as a release.
|
||||
- A preview Release MUST be published with `isPrerelease=true` and `isLatest=false`. Any `*-latest` preview asset or preview-triggered `latest.json`, R2, Helm, or moving Docker channel tag is a pipeline failure. Docker images may use the exact preview version tag, including its existing variant suffixes.
|
||||
- Preview Releases are cleaned up by the `cleanup-preview-releases` job after `publish-release` succeeds for the deliverable tag. It deletes every Release whose tag is exactly `<target>-preview.<digits>` and never passes `--cleanup-tag`, so the tags survive.
|
||||
|
||||
A green Linux run is ready for field acceptance, not publication. The partial certification must have `publish_allowed: false`. Retain the exact Linux manifest for the next stage; a new source/build-input candidate must not inherit the old approval. A newer unrelated main commit does not invalidate this evidence.
|
||||
## Phase 3 — CI and preview Release verification
|
||||
|
||||
- Find and watch the tag build: `gh run list --workflow build.yml --branch "<preview-tag>" --limit 1` then `gh run watch <run-id>`. Every build matrix target must succeed (linux x86_64/aarch64 × musl/gnu, macos-aarch64, windows-x86_64).
|
||||
- Confirm the preview tag resolves to the Phase 2 CI-validated `PREVIEW_HASH`. If the release branch advances during acceptance, repeat Phases 2–5 with a new preview tag.
|
||||
- Confirm the Release publication jobs (`create-release`, `upload-release-assets`, and `publish-release`) succeed while `update-latest-version` is skipped.
|
||||
- Record the inputs actually consumed by each build job: source SHA and the embedded Console release tag, asset URL, and verified SHA256 from the download logs. The workflow reads Console `releases/latest` separately for each build; confirm the matrix used the Console asset accepted in Phase 0. A missing identity or differing asset blocks acceptance until resolved. Retain available toolchain and build provenance with the artifact checksums; the same source SHA alone does not establish equivalent builds.
|
||||
- Verify `gh release view "<preview-tag>" --json isPrerelease,assets,url`: `isPrerelease` must be `true`, and the Release must contain all 6 versioned platform zips, checksums, SBOM, and provenance with no `-latest` assets. Confirm `gh api repos/{owner}/{repo}/releases/latest --jq .tag_name` does not return `<preview-tag>`.
|
||||
- Record `PREVIOUS_DELIVERABLE`, selected from published Releases by `publishedAt` after excluding the current tag and every `-preview.N` tag. Verify `gh release view "<preview-tag>" --json body --jq .body` contains `## What's Changed` and, when `PREVIOUS_DELIVERABLE` exists, `**Full Changelog**: https://github.com/rustfs/rustfs/compare/<PREVIOUS_DELIVERABLE>...<preview-tag>`. For a repository with no previous deliverable, verify a Full Changelog link exists and record the GitHub baseline fallback.
|
||||
- Confirm Helm is skipped. If preview Docker images are published, confirm they use only exact preview tags (and variant suffixes), with no `latest`, `alpha`, `beta`, or `rc` channel updates. Preview validation covers the built RustFS binaries, embedded console, and rc compatibility; it does not authorize advancing any installation default.
|
||||
|
||||
## Release notes
|
||||
|
||||
- When a previous deliverable exists, GitHub Release notes for the preview and final tags MUST use it as their shared comparison baseline: the most recently published non-preview Release before the target. Internal `-preview.N` Releases are explicitly excluded from that selection, even when they point at the same commit as the final tag — cleanup runs after the notes are generated, so the preview Release is still present and would otherwise be picked as the baseline. If no previous deliverable exists, omit `previous_tag_name` and record that GitHub's default baseline fallback was used.
|
||||
- Generated Release notes carry a workflow-management marker so retries can repair them. Before manually curating a generated body, remove that marker; unmarked non-placeholder notes are preserved by later workflow runs.
|
||||
|
||||
@@ -16,13 +16,22 @@ Read for Phases 1–2. The [parent release invariants](../SKILL.md) and Console
|
||||
- Only when neither the release branch, an unfinished preview tag, nor current main already has `<target>`: if main is at a later version, select and verify a historical commit containing `<target>` rather than downgrading main; block if none exists. Otherwise invoke the `rustfs-release-version-bump` skill with stage `prepare`, the final `<target>` (NOT a preview version), and full GitHub flow (commit/push/PR).
|
||||
- If a bump PR was needed, get it merged into main and record its merge commit. For a new release branch when main already has `<target>`, identify the main commit that introduced those versions. Phase 2 chooses a release candidate containing that commit; an existing release branch keeps its own history and a newer main HEAD is not required.
|
||||
|
||||
Do not wait for the moving latest main HEAD to become green. The release-branch CI in Phase 2 validates the chosen commit before a candidate build is dispatched.
|
||||
Do not wait for the moving latest main HEAD to become green. The release-branch CI in Phase 2 validates the chosen commit before a preview tag is created.
|
||||
|
||||
## Phase 2 — Pin and validate the release source
|
||||
## Phase 2 — Pin and validate the release candidate, then publish the preview tag
|
||||
|
||||
- Use `release/<target>`. On the first attempt, create it from a selected main commit containing the version bump and intended changes; it need not be the latest main HEAD. On a restart, retain the existing branch and backport only reviewed fixes from main. Verify the branch's Cargo versions still equal `<target>`. A legacy preview can anchor source selection, but its tag and old artifacts do not constitute a candidate in the new protocol; freeze and verify a new candidate after the release recipe is aligned.
|
||||
- The current `ci.yml` runs automatically on main pushes, not release-branch pushes. Dispatch it explicitly with `gh workflow run ci.yml --ref "release/<target>"`. Locate that dispatch run, verify its `head_sha` equals the release branch's remote HEAD, and require the full expected CI job set to pass, including `Quick Checks`, `Test and Lint`, and `End-to-End Tests (full merge gate)`. Treat a missing or skipped required lane as incomplete. A successful PR check or CI run for a different SHA does not qualify. If the branch advances or a run is cancelled, validate the new SHA again. Do not dispatch a candidate with failing or incomplete CI.
|
||||
- Use `release/<target>`. On the first attempt, create it from a selected main commit containing the version bump and intended changes; it need not be the latest main HEAD. On a restart, retain the existing branch and backport only reviewed fixes from main. Verify the branch's Cargo versions still equal `<target>`. If adopting an existing preview, run this branch CI before continuing its acceptance; reuse the tag only when it points to the validated SHA.
|
||||
- The current `ci.yml` runs automatically on main pushes, not release-branch pushes. Dispatch it explicitly with `gh workflow run ci.yml --ref "release/<target>"`. Locate that dispatch run, verify its `head_sha` equals the release branch's remote HEAD, and require the full expected CI job set to pass, including `Quick Checks`, `Test and Lint`, and `End-to-End Tests (full merge gate)`. Treat a missing or skipped required lane as incomplete. A successful PR check or CI run for a different SHA does not qualify. If the branch advances or a run is cancelled, validate the new SHA again. Do not tag a candidate with failing or incomplete CI.
|
||||
- Prefer protecting `release/*` against force pushes and deletion and requiring reviewed backport PRs. Check live rulesets rather than assuming the main ruleset covers release branches; report a missing release-branch rule as a gap.
|
||||
- Set `CANDIDATE_SHA` to the CI-validated release-branch commit. Record the branch, exact commit and CI run URL. The new candidate protocol starts from a commit whose release recipe matches the reviewed main dispatcher; backport the recipe before freezing an older release branch if needed.
|
||||
- Dispatch the Linux candidate as described in [candidate verification](preview-release.md). Do not create a preview or final tag here. The final executable already contains `<target>`; publication later promotes those same bytes.
|
||||
- After a source or build-input change, run the affected source checks and create a new candidate. Reuse the frozen candidate after an infrastructure failure. Do not restart acceptance for unrelated main advances.
|
||||
- Set `PREVIEW_HASH` to the CI-validated release-branch commit and recheck the remote branch HEAD immediately before tagging. Report the branch, commit, and CI run URL; do not derive `PREVIEW_HASH` from a later `origin/main` fetch. When adopting an existing preview tag at this SHA, skip tag creation and continue Phase 3.
|
||||
|
||||
```bash
|
||||
git tag -a "<preview-tag>" -m "Release <preview-tag>" "$PREVIEW_HASH"
|
||||
git push origin "<preview-tag>"
|
||||
```
|
||||
|
||||
Pushing the tag triggers `.github/workflows/build.yml` ("Build and Release"); `docker.yml` chains off it via `workflow_run`.
|
||||
|
||||
The preview run builds versioned artifacts and publishes them in a GitHub prerelease. Docker may publish exact preview image tags. Latest-channel, R2, and Helm publication must be skipped; installation references and the website announcement stay unchanged.
|
||||
|
||||
On a restart after a backport, re-run release-branch CI before setting the new `PREVIEW_HASH`. Use the next unused preview iteration when the validated commit differs from an existing preview tag; never move an existing tag.
|
||||
|
||||
@@ -4,18 +4,18 @@ description: Prepare an exact RustFS Cargo version or align installation referen
|
||||
---
|
||||
# Release Version Files
|
||||
|
||||
Require an exact target; ask if missing or ambiguous. Reject targets containing `-preview`: those identify candidates, not product versions or new public tags. Infer `post-release` only from an explicit request or parent workflow; otherwise use `prepare`. Delivery defaults to local edit/verify; commit, push, PR, and merge follow existing authorization and [Git rules](../../references/pull-requests.md).
|
||||
Require an exact target; ask if missing or ambiguous. Reject targets containing `-preview`: those are tag-only. Infer `post-release` only from an explicit request or parent workflow; otherwise use `prepare`. Delivery defaults to local edit/verify; commit, push, PR, and merge follow existing authorization and [Git rules](../../references/pull-requests.md).
|
||||
|
||||
| Stage | Files and constraints |
|
||||
|---|---|
|
||||
| `prepare` | Only `Cargo.toml` and `Cargo.lock`: workspace package, internal dependency, and workspace member versions become `<target>`. External dependencies stay unchanged. |
|
||||
| `post-release` | Only `README.md`, `README_ZH.md`, `flake.nix`, `helm/rustfs/Chart.yaml`, and `rustfs.spec`. Cargo may already target the next release; leave it untouched. |
|
||||
|
||||
Before installation edits, verify the exact non-preview Release, downloadable assets/source archive, pullable `rustfs/rustfs:<target>` image manifest, and published Helm chart. Block on missing artifacts. Do not downgrade defaults if a newer deliverable superseded the target. Candidate preparation retains the previous published installation references; follow-up commits never replace validated tags or sealed artifacts.
|
||||
Before installation edits, verify the exact non-preview Release, downloadable assets/source archive, pullable `rustfs/rustfs:<target>` image manifest, and published Helm chart. Block on missing artifacts. Do not downgrade defaults if a newer deliverable superseded the target. Preview preparation retains the previous published installation references; follow-up commits never replace either validated tag.
|
||||
|
||||
Packaging policy:
|
||||
|
||||
- Docker image tags have no `v` prefix. Derive chart/app versions with `scripts/helm_chart_version.sh <target>`; both equal the final product SemVer, including prerelease suffixes. Helm CI uses the frozen candidate version before sealing, so no early installation chart bump is required. Previously published chart versions remain unchanged.
|
||||
- Docker image tags have no `v` prefix. Derive chart/app versions with `scripts/helm_chart_version.sh <target>` (`beta.N -> 0.N.0`; other versions unchanged). Helm CI derives versions from the final tag, so no early chart bump is required.
|
||||
- RPM `Version` is numeric; `Release` is the prerelease suffix or `1` for stable. Expanded `Source0` and the unpack directory must resolve to the exact target including its suffix. Add the changelog entry using current git identity, date, and target; never copy an example identity/date.
|
||||
- Changing these mappings requires explicit confirmation.
|
||||
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=21e9a1561ac20382503c38353bbb9b830658534a4f277c254a42d0afc6ebed8b
|
||||
sha256=fbabc660e6e01fd8ff6b9d1de7d68e58242d60c942f85aed5074b0ec2918c154
|
||||
|
||||
@@ -71,15 +71,10 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
|
||||
./scripts/check_s3s_footprint.sh
|
||||
|
||||
.PHONY: test-count-ratchet-check
|
||||
test-count-ratchet-check: ## Check no workspace crate fell below its test floor or rose above its #[ignore] ceiling in the committed baseline
|
||||
test-count-ratchet-check: ## Check no workspace crate lost tests or gained #[ignore] against the committed baseline
|
||||
@echo "🧪 Checking per-crate test-count ratchet..."
|
||||
./scripts/check_test_count_ratchet.sh
|
||||
|
||||
.PHONY: test-seams-check
|
||||
test-seams-check: ## Check tests outside an operation's owner call its use-case only through rustfs/src/app/test_seams
|
||||
@echo "🧪 Checking cross-owner test seams..."
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py
|
||||
|
||||
.PHONY: fips-wording-check
|
||||
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
|
||||
@echo "📣 Checking cryptographic capability wording guard..."
|
||||
|
||||
@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
|
||||
./scripts/check_no_planning_docs.sh
|
||||
|
||||
.PHONY: pre-commit
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check test-seams-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
|
||||
@echo "✅ All pre-commit checks passed!"
|
||||
|
||||
.PHONY: pre-pr
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check test-seams-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check test-count-ratchet-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
|
||||
@echo "✅ All pre-PR checks passed!"
|
||||
|
||||
.PHONY: dev-check
|
||||
|
||||
@@ -22,14 +22,6 @@ TEST_THREADS ?= 1
|
||||
# differ from CI and .config/nextest.toml test-groups will NOT apply.
|
||||
|
||||
.PHONY: script-tests
|
||||
.PHONY: release-package-tests
|
||||
release-package-tests: ## Run release package integration on Linux with fpm, dpkg and rpm installed
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_packages.py
|
||||
|
||||
.PHONY: release-image-tests
|
||||
release-image-tests: ## Run OCI integration with Docker, Go and skopeo; set RELEASE_IMAGE_VARIANT and RELEASE_IMAGE_OUTPUT
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_images.py
|
||||
|
||||
script-tests: ## Run shell script tests
|
||||
@echo "Running script tests..."
|
||||
trap 'status=$$?; printf "ERROR: script-tests failed (exit %s): %s\n" "$$status" "$$BASH_COMMAND" >&2; exit "$$status"' ERR
|
||||
@@ -61,19 +53,12 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_check_doc_paths.py
|
||||
./scripts/check_embedded_secrets.sh --self-test
|
||||
./scripts/check_s3s_footprint.sh --self-test
|
||||
./scripts/test_check_test_count_ratchet.sh
|
||||
./scripts/test_guard_source_set.sh
|
||||
./scripts/test_split_guard_rules.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_lifecycle_transition_api_test.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py --self-test
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_reference_binary.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/release -p "test_*.py"
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_upgrade_workflow.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_migration_gate_evidence.py
|
||||
@@ -94,11 +79,8 @@ script-tests: ## Run shell script tests
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_validation_runner.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_api_coverage.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_reference_parity.py
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
./scripts/validate_object_data_cache_cold_stampede.sh --self-test
|
||||
|
||||
@@ -946,10 +946,6 @@ path = "junit.xml"
|
||||
# parallel-safe — the same property e2e-smoke relies on. The exceptions are the
|
||||
# 4-disk reliability / degraded-read fault-injection tests, multi-node heal
|
||||
# clusters, and the fixed-port Vault tests, all serialized below.
|
||||
# `dual_stack_differential::reference::` holds hermetic unit tests of how the
|
||||
# differential picks side A (shell-script stand-ins, no server, no reference
|
||||
# binary). They already run on every pull request in e2e-smoke, so e2e-full
|
||||
# excludes them to keep its selection digest unchanged.
|
||||
[profile.e2e-full]
|
||||
default-filter = """
|
||||
package(e2e_test)
|
||||
@@ -960,7 +956,6 @@ default-filter = """
|
||||
& !test(/^replication_target_matrix_test::/)
|
||||
& !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)
|
||||
& !test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)
|
||||
& !test(/^dual_stack_differential::reference::/)
|
||||
"""
|
||||
fail-fast = false
|
||||
|
||||
|
||||
@@ -1,20 +0,0 @@
|
||||
# Pinned RustFS release used as the "reference" target of parity lanes
|
||||
# (.github/workflows/reference-parity.yml). This is the single place that names
|
||||
# the reference release; .github/actions/reference-binary resolves, downloads and
|
||||
# verifies it, and fails on a missing pin, a missing asset or a digest mismatch.
|
||||
#
|
||||
# Bumping the reference is a reviewed event: every per-test difference between
|
||||
# the candidate and the reference is measured against this exact build.
|
||||
#
|
||||
# Each sha256 is the value published in the release's SHA256SUMS asset, which
|
||||
# matches the digest GitHub reports for the same asset. The action checks both.
|
||||
|
||||
schema = 1
|
||||
repository = "rustfs/rustfs"
|
||||
version = "1.0.5"
|
||||
tag = "1.0.5"
|
||||
|
||||
[platforms.linux-x86_64-gnu]
|
||||
asset = "rustfs-linux-x86_64-gnu-v1.0.5.zip"
|
||||
sha256 = "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
|
||||
executable = "rustfs"
|
||||
@@ -1,24 +0,0 @@
|
||||
# Targets of the s3-tests reference parity lane (.github/workflows/reference-parity.yml).
|
||||
#
|
||||
# Exactly one target uses `binary = "reference"` (the release pinned in
|
||||
# .config/reference-binary.toml). Every other target uses `binary = "candidate"`
|
||||
# (the binary built from the branch under test) and is compared per node id
|
||||
# against the reference. All targets run the same harness, test list and server
|
||||
# configuration; `env` adds only the target's own RUSTFS_* settings.
|
||||
#
|
||||
# Adding a target is one more [[targets]] table, for example:
|
||||
#
|
||||
# [[targets]]
|
||||
# name = "candidate-gateway"
|
||||
# binary = "candidate"
|
||||
# env = { RUSTFS_S3_STACK = "gateway" }
|
||||
|
||||
schema = 1
|
||||
|
||||
[[targets]]
|
||||
name = "reference"
|
||||
binary = "reference"
|
||||
|
||||
[[targets]]
|
||||
name = "candidate-legacy"
|
||||
binary = "candidate"
|
||||
@@ -1,12 +0,0 @@
|
||||
# Tolerated per-test differences of the s3-tests reference parity lane.
|
||||
#
|
||||
# One entry per line, four fields separated by " | ":
|
||||
#
|
||||
# <target> | <pytest node id> | <tracking issue URL> | <reason>
|
||||
#
|
||||
# <target> is a candidate name from .config/reference-parity-targets.toml, the
|
||||
# node id is exactly as the lane prints it (s3tests/functional/test_s3.py::...),
|
||||
# and the issue URL is https://github.com/rustfs/<repo>/issues/<number>.
|
||||
#
|
||||
# Same semantics as scripts/junit_diff.py --allow: an entry whose difference no
|
||||
# longer occurs is stale and fails the lane, so remove it when it is fixed.
|
||||
@@ -1,93 +1,44 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"plan_id": "rustfs-release-validation",
|
||||
"description": "Certification for immutable candidates. All shipped platforms require native identity and startup; stable and prerelease require the pinned upgrade matrix. Linux-only acceptance never authorizes publication.",
|
||||
"release_types": [
|
||||
"preview",
|
||||
"prerelease",
|
||||
"stable"
|
||||
],
|
||||
"description": "Versioned certification plan for every public RustFS release exit (rustfs/backlog#2662, R01). preview, prerelease and stable all require this explicit plan. Scripts/release/release_gate.py fail-closes publication on any missing, duplicate, stale, substituted or non-PASS required item.",
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"targets": [
|
||||
{
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "musl"
|
||||
},
|
||||
{
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "musl"
|
||||
},
|
||||
{
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
{
|
||||
"os": "linux",
|
||||
"arch": "aarch64",
|
||||
"abi": "gnu"
|
||||
},
|
||||
{
|
||||
"os": "macos",
|
||||
"arch": "aarch64",
|
||||
"abi": "none"
|
||||
},
|
||||
{
|
||||
"os": "windows",
|
||||
"arch": "x86_64",
|
||||
"abi": "msvc"
|
||||
}
|
||||
{"os": "linux", "arch": "x86_64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "musl"},
|
||||
{"os": "linux", "arch": "x86_64", "abi": "gnu"},
|
||||
{"os": "linux", "arch": "aarch64", "abi": "gnu"},
|
||||
{"os": "macos", "arch": "aarch64", "abi": "none"},
|
||||
{"os": "windows", "arch": "x86_64", "abi": "msvc"}
|
||||
],
|
||||
"targets_basis": "Platform matrix of .github/workflows/build.yml (prepare-platform-matrix). gnu and macOS targets build with the pyroscope feature; musl and Windows targets build with no extra features.",
|
||||
"features": [
|
||||
"pyroscope"
|
||||
],
|
||||
"features": ["pyroscope"],
|
||||
"trusted_workflows": [
|
||||
{
|
||||
"repository": "rustfs/rustfs",
|
||||
"workflow_id": "release-candidate.yml"
|
||||
},
|
||||
{
|
||||
"repository": "rustfs/rustfs",
|
||||
"workflow_id": "release-complete.yml"
|
||||
}
|
||||
{"repository": "rustfs/s3chaos", "workflow_id": "release-gate.yml"}
|
||||
],
|
||||
"trusted_workflows_basis": "Receipts are produced by the reviewed main workflows and fetched by exact run ID with GitHub artifact digest verification. Candidate inputs freeze the harness commit separately from product source.",
|
||||
"trusted_workflows_basis": "The shared evidence envelope producer lives in rustfs/s3chaos (G01, fault/release_gate.rs) and its plan/live split workflow is .github/workflows/release-gate.yml there. Only results from these identities may enter a release certification; PR-uploaded JSON never gains release authority. R02 must extend this list when additional verifier workflows land, never widen it to untrusted sources.",
|
||||
"lanes": [
|
||||
{
|
||||
"lane": "identity",
|
||||
"harness_repository": "rustfs/rustfs",
|
||||
"harness_sha": "candidate.workflow_sha",
|
||||
"harness_basis": "The exact workflow commit is frozen in candidate inputs; no mutable branch is accepted as a harness identity.",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Fixed G01 evidence-envelope producer revision (rustfs/s3chaos PR #102 head recorded in rustfs/backlog#2658). R02 re-pins this when the live verifier ships.",
|
||||
"timeout_seconds": 1800,
|
||||
"environment_capabilities": [
|
||||
"process-exec"
|
||||
],
|
||||
"cases": [
|
||||
"artifact-identity",
|
||||
"startup"
|
||||
],
|
||||
"environment_capabilities": ["process-exec"],
|
||||
"cases": ["artifact-identity", "startup"],
|
||||
"applies_to": {
|
||||
"release_types": [
|
||||
"preview",
|
||||
"prerelease",
|
||||
"stable"
|
||||
],
|
||||
"release_types": ["preview", "prerelease", "stable"],
|
||||
"platforms": "all"
|
||||
},
|
||||
"applies_basis": "Artifact identity and startup checks are mandatory for every release type, preview included, on every shipped target (rustfs/backlog#2662 frozen design item 5)."
|
||||
},
|
||||
{
|
||||
"lane": "upgrade",
|
||||
"harness_repository": "rustfs/rustfs",
|
||||
"harness_sha": "candidate.workflow_sha",
|
||||
"harness_basis": "The exact workflow commit is frozen in candidate inputs; no mutable branch is accepted as a harness identity.",
|
||||
"harness_repository": "rustfs/s3chaos",
|
||||
"harness_sha": "9a48f55c1d6d58ce63a8fb1ea7317e024f32f11d",
|
||||
"harness_basis": "Same fixed envelope producer as the identity lane; the workflow SHA and the product build SHA are bound separately and need not be equal.",
|
||||
"timeout_seconds": 5400,
|
||||
"environment_capabilities": [
|
||||
"process-exec",
|
||||
"previous-release-artifact"
|
||||
],
|
||||
"environment_capabilities": ["process-exec", "previous-release-artifact"],
|
||||
"cases": [
|
||||
"direct-upgrade",
|
||||
"mixed-version-upgrade",
|
||||
@@ -98,17 +49,8 @@
|
||||
],
|
||||
"cases_basis": "Case IDs of the pinned previous-release matrix in .github/workflows/e2e-upgrade.yml (source.version/asset/sha256 and the case list mapped there).",
|
||||
"applies_to": {
|
||||
"release_types": [
|
||||
"prerelease",
|
||||
"stable"
|
||||
],
|
||||
"platforms": [
|
||||
{
|
||||
"os": "linux",
|
||||
"arch": "x86_64",
|
||||
"abi": "gnu"
|
||||
}
|
||||
]
|
||||
"release_types": ["prerelease", "stable"],
|
||||
"platforms": [{"os": "linux", "arch": "x86_64", "abi": "gnu"}]
|
||||
},
|
||||
"applies_basis": "preview releases use the pre-declared smaller set (identity lane only). The upgrade lane is only exercised on linux-x86_64-gnu, the platform of the pinned source assets."
|
||||
}
|
||||
|
||||
@@ -12,6 +12,6 @@
|
||||
# s3_error_lines lines holding s3_error!, same scope, minus those the script
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=63
|
||||
s3_error_lines=217
|
||||
files=65
|
||||
s3_error_lines=219
|
||||
ecstore_files=0
|
||||
|
||||
@@ -4,17 +4,15 @@
|
||||
# One row per workspace member: <package>, tests, ignored, tab-separated.
|
||||
# tests counts #[test], #[tokio::test], #[tokio::test(...)] and #[test_case(...)]
|
||||
# attribute lines under the package directory; ignored counts #[ignore] and
|
||||
# #[ignore = "..."]. The tests column is a floor and the ignored column is a
|
||||
# ceiling: CI fails when a crate's tests fall below its row or its ignored count
|
||||
# rises above it. Extra tests and fewer ignores pass; --update tightens the rows.
|
||||
# Lowering a tests row or raising an ignored row is a reviewed statement that
|
||||
# tests were intentionally removed or ignored; justify it in the PR description.
|
||||
# Regenerate with:
|
||||
# #[ignore = "..."]. Both columns are exact: CI fails when a crate's tests fall
|
||||
# below or rise above its row, or when its ignored count changes. Raise a row in
|
||||
# the PR that adds tests; lowering one is a reviewed statement that tests were
|
||||
# intentionally removed. Regenerate with:
|
||||
# scripts/check_test_count_ratchet.sh --update
|
||||
e2e_test 1026 20
|
||||
rustfs 6449 64
|
||||
rustfs 6438 64
|
||||
rustfs-audit 70 0
|
||||
rustfs-checksums 28 1
|
||||
rustfs-checksums 26 0
|
||||
rustfs-common 36 0
|
||||
rustfs-concurrency 87 0
|
||||
rustfs-config 43 0
|
||||
@@ -25,7 +23,6 @@ rustfs-data-usage 58 0
|
||||
rustfs-ecstore 7858 2
|
||||
rustfs-extension-schema 23 0
|
||||
rustfs-filemeta 361 0
|
||||
rustfs-gateway-adapter 74 0
|
||||
rustfs-heal 1128 3
|
||||
rustfs-heal-contracts 8 0
|
||||
rustfs-iam 415 0
|
||||
@@ -46,7 +43,7 @@ rustfs-policy 758 0
|
||||
rustfs-protocols 770 1
|
||||
rustfs-protos 75 0
|
||||
rustfs-replication 209 0
|
||||
rustfs-rio 225 1
|
||||
rustfs-rio 223 0
|
||||
rustfs-rio-v2 34 6
|
||||
rustfs-s3-client 71 0
|
||||
rustfs-s3-ops 8 0
|
||||
@@ -62,6 +59,6 @@ rustfs-targets 612 26
|
||||
rustfs-test-utils 1 0
|
||||
rustfs-tls-runtime 44 0
|
||||
rustfs-trusted-proxies 72 0
|
||||
rustfs-utils 253 1
|
||||
rustfs-utils 251 0
|
||||
rustfs-zip 30 0
|
||||
zstd 11 0
|
||||
|
||||
@@ -101,10 +101,6 @@ runs:
|
||||
shell: bash
|
||||
run: ./scripts/check_test_count_ratchet.sh
|
||||
|
||||
- name: Check cross-owner tests use test seams
|
||||
shell: bash
|
||||
run: python3 ./scripts/check_test_seams.py
|
||||
|
||||
- name: Check cryptographic capability wording
|
||||
shell: bash
|
||||
run: ./scripts/check_fips_wording.sh
|
||||
|
||||
@@ -1,96 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: "Reference RustFS binary"
|
||||
description: "Download the pinned reference RustFS release, verify its sha256 and expose the executable path. A missing pin, missing asset or digest mismatch fails the step."
|
||||
|
||||
inputs:
|
||||
pin-file:
|
||||
description: "Pin file naming the reference release (repository-relative)"
|
||||
required: false
|
||||
default: ".config/reference-binary.toml"
|
||||
platform:
|
||||
description: "Platform table in the pin file"
|
||||
required: false
|
||||
default: "linux-x86_64-gnu"
|
||||
destination:
|
||||
description: "Directory that receives the executable; defaults to $RUNNER_TEMP/reference-binary"
|
||||
required: false
|
||||
default: ""
|
||||
|
||||
outputs:
|
||||
path:
|
||||
description: "Absolute path of the verified executable"
|
||||
value: ${{ steps.unpack.outputs.path }}
|
||||
version:
|
||||
description: "Pinned reference version"
|
||||
value: ${{ steps.pin.outputs.version }}
|
||||
sha256:
|
||||
description: "Pinned and verified archive sha256"
|
||||
value: ${{ steps.pin.outputs.sha256 }}
|
||||
|
||||
runs:
|
||||
using: "composite"
|
||||
steps:
|
||||
- name: Resolve reference pin
|
||||
id: pin
|
||||
shell: bash
|
||||
env:
|
||||
PIN_TOOL: ${{ github.action_path }}/reference_binary.py
|
||||
PIN_FILE: ${{ inputs.pin-file }}
|
||||
PIN_PLATFORM: ${{ inputs.platform }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
resolved="$(python3 "$PIN_TOOL" resolve --pin "$PIN_FILE" --platform "$PIN_PLATFORM")"
|
||||
printf '%s\n' "$resolved"
|
||||
printf '%s\n' "$resolved" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download and verify reference archive
|
||||
id: download
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
ASSET_DOWNLOADER: ${{ github.action_path }}/../setup/download_release_asset.py
|
||||
REF_REPOSITORY: ${{ steps.pin.outputs.repository }}
|
||||
REF_TAG: ${{ steps.pin.outputs.tag }}
|
||||
REF_ASSET: ${{ steps.pin.outputs.asset }}
|
||||
REF_SHA256: ${{ steps.pin.outputs.sha256 }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
archive="$RUNNER_TEMP/reference-archive/$REF_ASSET"
|
||||
# Fails unless the release lists exactly one asset of this name whose
|
||||
# published digest equals the pin and whose downloaded bytes match it.
|
||||
python3 "$ASSET_DOWNLOADER" --repository "$REF_REPOSITORY" --tag "$REF_TAG" \
|
||||
--asset "$REF_ASSET" --output "$archive" --sha256 "$REF_SHA256"
|
||||
echo "archive=$archive" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Unpack reference executable
|
||||
id: unpack
|
||||
shell: bash
|
||||
env:
|
||||
PIN_TOOL: ${{ github.action_path }}/reference_binary.py
|
||||
REF_ARCHIVE: ${{ steps.download.outputs.archive }}
|
||||
REF_SHA256: ${{ steps.pin.outputs.sha256 }}
|
||||
REF_EXECUTABLE: ${{ steps.pin.outputs.executable }}
|
||||
REF_DESTINATION: ${{ inputs.destination }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
destination="${REF_DESTINATION:-$RUNNER_TEMP/reference-binary}"
|
||||
unpacked="$(python3 "$PIN_TOOL" unpack --archive "$REF_ARCHIVE" --sha256 "$REF_SHA256" \
|
||||
--executable "$REF_EXECUTABLE" --destination "$destination")"
|
||||
path="${unpacked#path=}"
|
||||
test -x "$path"
|
||||
echo "Reference executable: $path"
|
||||
"$path" --version
|
||||
echo "path=$path" >> "$GITHUB_OUTPUT"
|
||||
@@ -1,144 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Resolve and unpack the pinned reference RustFS release binary.
|
||||
|
||||
`resolve` reads the pin file and prints the selected platform's identity as
|
||||
`key=value` lines (GitHub step-output format). `unpack` re-verifies the
|
||||
downloaded archive digest against the pin and extracts exactly one executable.
|
||||
Every missing or malformed input is a failure; nothing is skipped.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import tomllib
|
||||
import zipfile
|
||||
|
||||
REPOSITORY = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*")
|
||||
TAG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*")
|
||||
PLATFORM = re.compile(r"[a-z0-9][a-z0-9_-]*")
|
||||
ASSET = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]*\.zip")
|
||||
SHA256 = re.compile(r"[0-9a-f]{64}")
|
||||
EXECUTABLE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
|
||||
MAX_EXECUTABLE_BYTES = 1024 * 1024 * 1024
|
||||
|
||||
|
||||
class PinError(Exception):
|
||||
"""A pin, archive or argument that cannot be trusted."""
|
||||
|
||||
|
||||
def field(table, key, pattern, where):
|
||||
value = table.get(key)
|
||||
if not isinstance(value, str) or not pattern.fullmatch(value):
|
||||
raise PinError(f"{where}: missing or invalid '{key}'")
|
||||
return value
|
||||
|
||||
|
||||
def resolve(pin_path, platform):
|
||||
if not PLATFORM.fullmatch(platform or ""):
|
||||
raise PinError("invalid platform name")
|
||||
try:
|
||||
data = tomllib.loads(Path(pin_path).read_text(encoding="utf-8"))
|
||||
except FileNotFoundError as exc:
|
||||
raise PinError(f"{pin_path}: reference pin file is missing") from exc
|
||||
except (OSError, UnicodeDecodeError, tomllib.TOMLDecodeError) as exc:
|
||||
raise PinError(f"{pin_path}: cannot read reference pin: {exc}") from exc
|
||||
if data.get("schema") != 1 or type(data.get("schema")) is not int:
|
||||
raise PinError(f"{pin_path}: unsupported pin schema")
|
||||
identity = {
|
||||
"repository": field(data, "repository", REPOSITORY, pin_path),
|
||||
"version": field(data, "version", TAG, pin_path),
|
||||
"tag": field(data, "tag", TAG, pin_path),
|
||||
}
|
||||
platforms = data.get("platforms")
|
||||
if not isinstance(platforms, dict) or not platforms:
|
||||
raise PinError(f"{pin_path}: no pinned platforms")
|
||||
entry = platforms.get(platform)
|
||||
if not isinstance(entry, dict):
|
||||
raise PinError(f"{pin_path}: no pin for platform '{platform}'")
|
||||
where = f"{pin_path} [platforms.{platform}]"
|
||||
identity["asset"] = field(entry, "asset", ASSET, where)
|
||||
identity["sha256"] = field(entry, "sha256", SHA256, where)
|
||||
identity["executable"] = field(entry, "executable", EXECUTABLE, where)
|
||||
if identity["version"] not in identity["asset"]:
|
||||
raise PinError(f"{where}: asset name does not carry version {identity['version']}")
|
||||
unknown = set(entry) - {"asset", "sha256", "executable"}
|
||||
if unknown:
|
||||
raise PinError(f"{where}: unknown keys {sorted(unknown)}")
|
||||
return identity
|
||||
|
||||
|
||||
def file_sha256(path):
|
||||
with open(path, "rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
|
||||
|
||||
def unpack(archive, sha256, executable, destination):
|
||||
if not SHA256.fullmatch(sha256 or "") or not EXECUTABLE.fullmatch(executable or ""):
|
||||
raise PinError("invalid expected digest or executable name")
|
||||
try:
|
||||
actual = file_sha256(archive)
|
||||
except OSError as exc:
|
||||
raise PinError(f"{archive}: reference archive is missing or unreadable") from exc
|
||||
if actual != sha256:
|
||||
raise PinError(f"{archive}: sha256 {actual} does not match the pinned digest")
|
||||
try:
|
||||
with zipfile.ZipFile(archive) as bundle:
|
||||
matches = [info for info in bundle.infolist() if info.filename == executable]
|
||||
if len(matches) != 1:
|
||||
raise PinError(f"{archive}: must contain exactly one top-level '{executable}'")
|
||||
info = matches[0]
|
||||
if info.is_dir() or not 0 < info.file_size <= MAX_EXECUTABLE_BYTES:
|
||||
raise PinError(f"{archive}: '{executable}' is not a plausible executable")
|
||||
destination = Path(destination)
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
target = destination / executable
|
||||
with bundle.open(info) as source, open(target, "wb") as sink:
|
||||
while chunk := source.read(1024 * 1024):
|
||||
sink.write(chunk)
|
||||
except zipfile.BadZipFile as exc:
|
||||
raise PinError(f"{archive}: not a valid zip archive") from exc
|
||||
target.chmod(0o755)
|
||||
return target
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = parser.add_subparsers(dest="mode", required=True)
|
||||
res = sub.add_parser("resolve")
|
||||
res.add_argument("--pin", required=True)
|
||||
res.add_argument("--platform", required=True)
|
||||
unp = sub.add_parser("unpack")
|
||||
unp.add_argument("--archive", required=True)
|
||||
unp.add_argument("--sha256", required=True)
|
||||
unp.add_argument("--executable", required=True)
|
||||
unp.add_argument("--destination", required=True)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
if args.mode == "resolve":
|
||||
for key, value in resolve(args.pin, args.platform).items():
|
||||
print(f"{key}={value}")
|
||||
else:
|
||||
print(f"path={unpack(args.archive, args.sha256, args.executable, args.destination)}")
|
||||
except PinError as exc:
|
||||
print(f"reference binary: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -350,10 +350,10 @@
|
||||
"roles": [
|
||||
"periodic",
|
||||
"manual",
|
||||
"reusable"
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"workflow_call",
|
||||
"push",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
@@ -367,7 +367,7 @@
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 36,
|
||||
"result_owner": "build.yml",
|
||||
"disposition": "Builds artifacts only; public release writes belong to release-publish.yml.",
|
||||
"disposition": "retained independent entry",
|
||||
"failure_owner": "workflow",
|
||||
"origin": "baseline",
|
||||
"alert_contract": "scheduled-validation"
|
||||
@@ -745,13 +745,14 @@
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/docker.yml",
|
||||
"name": "Build candidate images",
|
||||
"name": "Docker Images",
|
||||
"roles": [
|
||||
"release",
|
||||
"reusable"
|
||||
"manual",
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"workflow_call"
|
||||
"workflow_run",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "event",
|
||||
"timezone": "UTC",
|
||||
@@ -760,7 +761,7 @@
|
||||
"policy": "active",
|
||||
"monitor": "none",
|
||||
"result_owner": "docker.yml",
|
||||
"disposition": "Builds artifacts only; public release writes belong to release-publish.yml.",
|
||||
"disposition": "retained independent entry",
|
||||
"monitor_reason": "Event-driven or operator-requested work has no periodic freshness obligation.",
|
||||
"origin": "baseline"
|
||||
},
|
||||
@@ -869,33 +870,6 @@
|
||||
"failure_owner": "watchdog",
|
||||
"origin": "baseline"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/reference-parity.yml",
|
||||
"name": "Reference Parity",
|
||||
"roles": [
|
||||
"pr",
|
||||
"periodic",
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"pull_request",
|
||||
"schedule",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "nightly",
|
||||
"timezone": "UTC",
|
||||
"schedule": [
|
||||
"37 1 * * *"
|
||||
],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "workflow",
|
||||
"max_age_hours": 36,
|
||||
"result_owner": "reference-parity.yml",
|
||||
"disposition": "s3-tests per-node-id parity of branch candidates against the pinned reference release",
|
||||
"failure_owner": "watchdog",
|
||||
"origin": "added"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/functional-chain-health.yml",
|
||||
"name": "Functional chain health",
|
||||
@@ -980,13 +954,14 @@
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/helm-package.yml",
|
||||
"name": "Build candidate Helm chart",
|
||||
"name": "Publish helm chart to artifacthub",
|
||||
"roles": [
|
||||
"release",
|
||||
"reusable"
|
||||
"manual",
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"workflow_call"
|
||||
"workflow_run",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "event",
|
||||
"timezone": "UTC",
|
||||
@@ -995,7 +970,7 @@
|
||||
"policy": "active",
|
||||
"monitor": "none",
|
||||
"result_owner": "helm-package.yml",
|
||||
"disposition": "Builds artifacts only; public release writes belong to release-publish.yml.",
|
||||
"disposition": "retained independent entry",
|
||||
"monitor_reason": "Event-driven or operator-requested work has no periodic freshness obligation.",
|
||||
"origin": "baseline"
|
||||
},
|
||||
@@ -1021,26 +996,6 @@
|
||||
"policy_reason": "Preserve operator-disabled state; enabling requires a separate policy decision.",
|
||||
"origin": "baseline"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/manual-main-packages.yml",
|
||||
"name": "Manual main packages",
|
||||
"roles": [
|
||||
"manual"
|
||||
],
|
||||
"events": [
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "event",
|
||||
"timezone": "UTC",
|
||||
"schedule": [],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "none",
|
||||
"result_owner": "manual-main-packages.yml",
|
||||
"disposition": "Packages the origin/main tip as workflow artifacts only. No GitHub Release or download-channel writes.",
|
||||
"monitor_reason": "Operator-requested packaging has no periodic freshness obligation.",
|
||||
"origin": "manual-main-packages"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/minio-interop.yml",
|
||||
"name": "minio-interop",
|
||||
@@ -1227,13 +1182,14 @@
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/package.yml",
|
||||
"name": "Build candidate packages",
|
||||
"name": "Package DEB/RPM",
|
||||
"roles": [
|
||||
"release",
|
||||
"reusable"
|
||||
"manual",
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"workflow_call"
|
||||
"workflow_run",
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "event",
|
||||
"timezone": "UTC",
|
||||
@@ -1242,7 +1198,7 @@
|
||||
"policy": "active",
|
||||
"monitor": "none",
|
||||
"result_owner": "package.yml",
|
||||
"disposition": "Builds artifacts only; public release writes belong to release-publish.yml.",
|
||||
"disposition": "retained independent entry",
|
||||
"monitor_reason": "Event-driven or operator-requested work has no periodic freshness obligation.",
|
||||
"origin": "baseline"
|
||||
},
|
||||
@@ -1771,69 +1727,6 @@
|
||||
"disposition": "retained independent entry",
|
||||
"monitor_reason": "Event-driven or operator-requested work has no periodic freshness obligation.",
|
||||
"origin": "baseline"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/release-candidate.yml",
|
||||
"name": "Release candidate (Linux)",
|
||||
"roles": [
|
||||
"manual",
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "event",
|
||||
"timezone": "UTC",
|
||||
"schedule": [],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "none",
|
||||
"result_owner": "release-candidate.yml",
|
||||
"disposition": "Explicit immutable candidate stage",
|
||||
"monitor_reason": "Operator-requested release stage has no periodic freshness obligation.",
|
||||
"origin": "release-candidate-migration"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/release-complete.yml",
|
||||
"name": "Release candidate (complete)",
|
||||
"roles": [
|
||||
"manual",
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "event",
|
||||
"timezone": "UTC",
|
||||
"schedule": [],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "none",
|
||||
"result_owner": "release-complete.yml",
|
||||
"disposition": "Explicit immutable candidate stage",
|
||||
"monitor_reason": "Operator-requested release stage has no periodic freshness obligation.",
|
||||
"origin": "release-candidate-migration"
|
||||
},
|
||||
{
|
||||
"workflow": ".github/workflows/release-publish.yml",
|
||||
"name": "Publish accepted release",
|
||||
"roles": [
|
||||
"manual",
|
||||
"release"
|
||||
],
|
||||
"events": [
|
||||
"workflow_dispatch"
|
||||
],
|
||||
"cadence": "event",
|
||||
"timezone": "UTC",
|
||||
"schedule": [],
|
||||
"owner": "@overtrue",
|
||||
"policy": "active",
|
||||
"monitor": "none",
|
||||
"result_owner": "release-publish.yml",
|
||||
"disposition": "Explicit immutable candidate stage",
|
||||
"monitor_reason": "Operator-requested release stage has no periodic freshness obligation.",
|
||||
"origin": "release-candidate-migration"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
## Billing facts
|
||||
|
||||
This repository is public. For a public repository, GitHub's standard hosted runners (`ubuntu-latest`, `windows-latest`, `macos-latest`) cost nothing and have no minute cap, whatever plan the organisation is on. GitHub bills only two things: larger hosted runners, including the `macos-latest-xlarge` release lane used here, and standard runners in private repositories.
|
||||
This repository is public. For a public repository, GitHub's standard hosted runners (`ubuntu-latest`, `windows-latest`, `macos-latest`) cost nothing and have no minute cap, whatever plan the organisation is on. GitHub bills only two things: larger hosted runners, which have custom labels and are not used here, and standard runners in private repositories.
|
||||
|
||||
Every other label in this repository is capacity RustFS pays for. ARC pods and lab machines accrue cost while the machine exists. A Ubicloud label is metered for the minutes a job holds its VM:
|
||||
|
||||
@@ -13,7 +13,6 @@ Every other label in this repository is capacity RustFS pays for. ARC pods and l
|
||||
|---|---|---|---|
|
||||
| `ubuntu-latest` | nobody | GitHub-hosted, Ubuntu 24.04, 4 vCPU, 16 GB RAM, about 14 GB of free disk. Ships a C toolchain, `docker`, `gh`, `curl`, `wget`, `jq`, Python. Can `mount` tmpfs and loop devices; io_uring works | the default for every job |
|
||||
| `windows-latest`, `macos-latest` | nobody | GitHub-hosted standard runners | Windows and macOS targets |
|
||||
| `macos-latest-xlarge` | RustFS | GitHub-hosted larger Apple Silicon runner | macOS release builds, deferred until Linux acceptance for candidates |
|
||||
| `sm-standard-2` | RustFS | ARC pod, Ubuntu 24.04, minimal image: no `gh`, no C toolchain; Docker only on the `dind-` label | only a job that a hosted runner cannot run (see the placement rule) |
|
||||
| `sm-standard-4` | RustFS | ARC pod, Ubuntu 24.04, 14 CPUs and 28 GB (the label is not the core count), large disk | workspace-scale Cargo builds |
|
||||
| `ubicloud-standard-16-arm` | RustFS | Ubicloud VM, alias of `ubicloud-standard-16-arm-ubuntu-2404`: Ubuntu 24.04 arm64, 16 vCPU, 48 GB, 300 GB disk. Reached through the `build.yml` matrix, so `actionlint.yaml` does not list it | native aarch64 release builds (`linux-aarch64-musl`, `linux-aarch64-gnu`) |
|
||||
@@ -33,7 +32,7 @@ A job runs on `ubuntu-latest` unless it has one of these reasons, and the reason
|
||||
|
||||
Jobs that do only the following have no reason to be self-hosted: shell or Python scripts, `cargo fmt`, `cargo deny`, typos, actionlint, API calls (`gh`, dependency review, issue and stale bots, CLA checks), downloading an artifact and running a prebuilt binary, or a `cargo build -p rustfs` with its own cache key. `e2e-distributed.yml` and `e2e-upgrade.yml` build the server and the `e2e_test` harness on `ubuntu-latest` inside their budgets, so "it compiles something" alone is not a reason.
|
||||
|
||||
The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection uses `ubuntu-latest` so a lab outage cannot also stop its independent monitor. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it wraps verified candidate binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
|
||||
The functional chain's `prepare` and `complete-chain` jobs, the performance runner availability check, and runner hygiene use `ubuntu-latest`: they inspect APIs or artifacts, not the lab hosts. The actual functional and performance suites remain on their lab runners. Functional-chain health collection uses `ubuntu-latest` so a lab outage cannot also stop its independent monitor. Docker image packaging also uses `ubuntu-latest` for both musl and glibc variants; it downloads release binaries rather than compiling the workspace, and still sets up QEMU and Buildx explicitly.
|
||||
|
||||
Moving these jobs off AKS reduces capacity demand, not necessarily the Azure bill immediately. VM costs fall only when nodes scale down or smaller capacity replaces them.
|
||||
|
||||
@@ -53,7 +52,7 @@ A hosted runner is also the only choice for a few things the pods cannot do: `mo
|
||||
These are the patterns that have cost real money here without validating anything more:
|
||||
|
||||
- An `if: always()` summary or alert job on a self-hosted label inside a `workflow_run`-triggered workflow. The upstream workflow completes many times a day, the real jobs skip, and the summary still starts a pod each time. Gate it on the upstream job having run, and keep it on `ubuntu-latest`.
|
||||
- Per-push workflows on `main`. A merge triggers the applicable `ci.yml` matrix and cache warming; `build.yml` development builds are scheduled or manual; with tens of merges a day that is as much self-hosted time as all pull requests together. Anything that does not verify the merged tree itself belongs on a schedule or behind a `paths` filter.
|
||||
- Per-push workflows on `main`. A merge triggers the full `ci.yml` matrix, `cache-warm.yml` and the `build.yml` development build; with tens of merges a day that is as much self-hosted time as all pull requests together. Anything that does not verify the merged tree itself belongs on a schedule or behind a `paths` filter.
|
||||
- `apt-get install` of tools the hosted image already ships. It is only needed on the minimal pod images.
|
||||
- A throttled `CARGO_BUILD_JOBS` on a 14-CPU pod. The limit exists for link-storm reasons documented in `ci.yml`; measure before raising it, and measure before lowering it further.
|
||||
- Tiny jobs on ARC pods. Each one pays roughly a minute of pod start-up before its first step; a compile-free job that takes ten seconds costs more to schedule than to run.
|
||||
@@ -67,7 +66,7 @@ gh api "repos/rustfs/rustfs/actions/runs/<run_id>/jobs?per_page=100" \
|
||||
--jq '.jobs[] | "\(.conclusion)\t\(.labels | join(","))\t\(.started_at)\t\(.completed_at)\t\(.name)"'
|
||||
```
|
||||
|
||||
Rows using `macos-latest-xlarge` or a label starting with `sm-`, `dind-`, `ubicloud-`, `smoke-` or `pf-` use paid capacity. At the October 2026 review, a full pull-request run of `ci.yml` spent about 150 self-hosted minutes, a third of it in jobs that never compiled; treat that as the scale of what a placement mistake costs, not as a current figure.
|
||||
Only rows whose label starts with `sm-`, `dind-`, `ubicloud-`, `smoke-` or `pf-` cost anything. At the October 2026 review, a full pull-request run of `ci.yml` spent about 150 self-hosted minutes, a third of it in jobs that never compiled; treat that as the scale of what a placement mistake costs, not as a current figure.
|
||||
|
||||
## Checklist for a new or moved job
|
||||
|
||||
@@ -77,9 +76,3 @@ Rows using `macos-latest-xlarge` or a label starting with `sm-`, `dind-`, `ubicl
|
||||
- A self-hosted `runs-on` carries a comment naming its reason from the placement rule.
|
||||
- Every `./.github/actions/setup` call states `cache-save-if`, and a key read from one pool is not written from the other.
|
||||
- Moving a job does not change its check name, so the `main` ruleset and `docs/testing/ci-gates.md` stay as they are; renaming it does, and then that document's change checklist applies.
|
||||
|
||||
## Release stages
|
||||
|
||||
The [candidate release pipeline](../../docs/operations/release-pipeline.md) builds Linux first and defers paid desktop capacity until acceptance. Daily development builds also select Linux; use manual platform selection for desktop development artifacts. All six platforms remain required before publication. The pipeline records and promotes the exact tested bytes. Builder workflows have no public-release credentials. Keep periodic development builds, PR verification and field-test chains separate when their contracts differ; do not re-enable disabled workflows as part of a source-file migration.
|
||||
|
||||
Larger runners remain billed for public repositories; see [GitHub runner billing](https://docs.github.com/en/actions/concepts/runners/larger-runners#billing). Actual spending comes from organization billing, not runner labels or an estimated build duration.
|
||||
|
||||
+8
-149
@@ -23,16 +23,9 @@ on:
|
||||
- 'deny.toml'
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/**'
|
||||
- 'scripts/release/**'
|
||||
- 'scripts/security/generate_release_supply_chain_assets.py'
|
||||
- '.config/release-validation-plan.json'
|
||||
- '.github/scheduled-validations.json'
|
||||
- 'scripts/test_helm_release_workflow.py'
|
||||
- 'scripts/test_release_packages.py'
|
||||
- 'scripts/test_release_images.py'
|
||||
- 'Dockerfile*'
|
||||
- 'entrypoint.sh'
|
||||
- 'bootstrap/**'
|
||||
- 'scripts/release/create_or_update_release.sh'
|
||||
- 'scripts/release/package_service_scripts.sh'
|
||||
- 'scripts/release/package_versions.sh'
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/test_docker_workflow.py'
|
||||
@@ -48,16 +41,9 @@ on:
|
||||
- 'deny.toml'
|
||||
- '.github/actions/**'
|
||||
- '.github/workflows/**'
|
||||
- 'scripts/release/**'
|
||||
- 'scripts/security/generate_release_supply_chain_assets.py'
|
||||
- '.config/release-validation-plan.json'
|
||||
- '.github/scheduled-validations.json'
|
||||
- 'scripts/test_helm_release_workflow.py'
|
||||
- 'scripts/test_release_packages.py'
|
||||
- 'scripts/test_release_images.py'
|
||||
- 'Dockerfile*'
|
||||
- 'entrypoint.sh'
|
||||
- 'bootstrap/**'
|
||||
- 'scripts/release/create_or_update_release.sh'
|
||||
- 'scripts/release/package_service_scripts.sh'
|
||||
- 'scripts/release/package_versions.sh'
|
||||
- 'scripts/test_package_service_scripts.sh'
|
||||
- 'scripts/test_package_versions.sh'
|
||||
- 'scripts/test_docker_workflow.py'
|
||||
@@ -146,38 +132,12 @@ jobs:
|
||||
if: github.repository == 'rustfs/rustfs' && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
release_changed: ${{ steps.scope.outputs.release_changed }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Scope native packaging checks to release changes
|
||||
id: scope
|
||||
env:
|
||||
BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }}
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import os, re, subprocess
|
||||
event, base = os.environ['GITHUB_EVENT_NAME'], os.environ.get('BASE_SHA', '')
|
||||
changed = event == 'workflow_dispatch'
|
||||
if event in ('push', 'pull_request'):
|
||||
if not re.fullmatch(r'[0-9a-f]{40}', base) or base == '0' * 40:
|
||||
changed = True
|
||||
else:
|
||||
paths = subprocess.check_output(['git', 'diff', '--name-only', base, os.environ['GITHUB_SHA']], text=True).splitlines()
|
||||
pattern = r'^(scripts/release/|scripts/test_release_|Dockerfile|entrypoint.sh$|bootstrap/|\.dockerignore$|\.config/release-validation-plan.json$|\.github/workflows/(audit|build|docker|package|helm-package|release-[^.]+)\.yml$)'
|
||||
changed = any(re.match(pattern, path) for path in paths)
|
||||
with open(os.environ['GITHUB_OUTPUT'], 'a') as output:
|
||||
output.write(f'release_changed={str(changed).lower()}\n')
|
||||
PY
|
||||
|
||||
- name: Report unpinned GitHub Actions
|
||||
run: ./scripts/security/check_workflow_pins.sh --enforce
|
||||
|
||||
@@ -190,35 +150,8 @@ jobs:
|
||||
- name: Check checkouts clear their credentials
|
||||
run: ./scripts/security/check_persist_credentials.sh
|
||||
|
||||
- name: Check release candidate and promotion contracts
|
||||
run: python3 -m unittest discover -s scripts/release -p 'test_*.py'
|
||||
|
||||
- name: Prepare artifact transport rehearsal
|
||||
run: |
|
||||
mkdir -p "$RUNNER_TEMP/release-transport"
|
||||
printf '{"purpose":"transport contract, not release evidence"}\n' > "$RUNNER_TEMP/release-transport/transport.json"
|
||||
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-transport-contract-${{ github.run_attempt }}
|
||||
path: ${{ runner.temp }}/release-transport/transport.json
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
- name: Verify actual GitHub artifact metadata and downloaded bytes
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import os, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, 'scripts/release')
|
||||
import candidate as c
|
||||
run = c.github(f"repos/{c.REPOSITORY}/actions/runs/{os.environ['GITHUB_RUN_ID']}")
|
||||
destination = Path(os.environ['RUNNER_TEMP']) / 'release-transport-downloaded'
|
||||
c.download_artifact(run, f"release-transport-contract-{os.environ['GITHUB_RUN_ATTEMPT']}", destination)
|
||||
c.require(c.read_json(destination / 'transport.json') == {'purpose': 'transport contract, not release evidence'}, 'artifact round trip changed')
|
||||
PY
|
||||
- name: Check preview release workflow policy
|
||||
run: ./scripts/security/check_preview_release_workflow.sh
|
||||
|
||||
- name: Check tier evidence workflow isolation
|
||||
run: ./scripts/security/check_tier_artifact_workflow.sh
|
||||
@@ -229,80 +162,6 @@ jobs:
|
||||
- name: Check package service lifecycle contract
|
||||
run: ./scripts/test_package_service_scripts.sh
|
||||
|
||||
release-package-contract:
|
||||
name: Release package contract
|
||||
needs: workflow-pin-report
|
||||
if: needs.workflow-pin-report.outputs.release_changed == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install real package managers
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ruby ruby-dev build-essential rpm fakeroot cpio
|
||||
sudo gem install fpm --version 1.17.0 --no-document
|
||||
- name: Exercise accepted and substituted payloads through both recipes
|
||||
run: python3 scripts/test_release_packages.py
|
||||
|
||||
release-image-contract:
|
||||
name: Release image contract (${{ matrix.variant }})
|
||||
needs: workflow-pin-report
|
||||
if: needs.workflow-pin-report.outputs.release_changed == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 35
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
variant: [musl, gnu]
|
||||
env:
|
||||
RELEASE_IMAGE_VARIANT: ${{ matrix.variant }}
|
||||
NO_PROXY: 127.0.0.1,localhost,::1
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Install OCI tools
|
||||
run: |
|
||||
echo "RELEASE_IMAGE_OUTPUT=$RUNNER_TEMP/image-contract" >> "$GITHUB_ENV"
|
||||
echo 'no_proxy=127.0.0.1,localhost,::1' >> "$GITHUB_ENV"
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y skopeo
|
||||
go version
|
||||
- uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
- name: Exercise real OCI packaging, startup, registry retry and conflicts
|
||||
run: python3 scripts/test_release_images.py
|
||||
- name: Exercise amd64 OCI archive scanning
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25
|
||||
env:
|
||||
TRIVY_PLATFORM: linux/amd64
|
||||
with:
|
||||
input: ${{ runner.temp }}/image-contract-scan/image-${{ matrix.variant }}-amd64.tar
|
||||
format: sarif
|
||||
output: ${{ runner.temp }}/image-contract/scan-${{ matrix.variant }}-amd64.sarif
|
||||
exit-code: "0"
|
||||
- name: Exercise arm64 OCI archive scanning
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25
|
||||
env:
|
||||
TRIVY_PLATFORM: linux/arm64
|
||||
with:
|
||||
input: ${{ runner.temp }}/image-contract-scan/image-${{ matrix.variant }}-arm64.tar
|
||||
format: sarif
|
||||
output: ${{ runner.temp }}/image-contract/scan-${{ matrix.variant }}-arm64.sarif
|
||||
exit-code: "0"
|
||||
- name: Verify scan receipts before recording the archive
|
||||
run: |
|
||||
python3 - <<'PY'
|
||||
import os, sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, 'scripts/release')
|
||||
import deliver, test_candidate
|
||||
deliver.image_record(test_candidate.inputs(), os.environ['RELEASE_IMAGE_VARIANT'], Path(os.environ['RELEASE_IMAGE_OUTPUT']))
|
||||
PY
|
||||
|
||||
dependency-review:
|
||||
name: Dependency Review
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
+764
-183
File diff suppressed because it is too large
Load Diff
@@ -96,7 +96,7 @@ env:
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
jobs:
|
||||
# Readers: workspace-clippy, workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
|
||||
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
|
||||
# e2e-tests, e2e-full.
|
||||
warm-ci-dev:
|
||||
name: Warm ci-dev
|
||||
|
||||
+14
-65
@@ -115,7 +115,7 @@ jobs:
|
||||
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
|
||||
echo "Base CI policy unavailable; running the full matrix."
|
||||
fi
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=4$' "$GITHUB_OUTPUT"; then
|
||||
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=3$' "$GITHUB_OUTPUT"; then
|
||||
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
|
||||
echo "Base policy does not cover this workflow contract; retaining the full matrix."
|
||||
fi
|
||||
@@ -170,67 +170,6 @@ jobs:
|
||||
- name: Run shared quick checks
|
||||
uses: ./.github/actions/quick-checks
|
||||
|
||||
# Clippy runs beside the test-binary build instead of in front of it. The two
|
||||
# share no compiler output: Clippy writes check-mode metadata, the archive
|
||||
# needs codegen and linked test binaries, and both restore the third-party
|
||||
# artifacts of their own mode from ci-dev. As one job the lint held the
|
||||
# archive, and every test that consumes it, back by its whole duration.
|
||||
# scripts/ci_gate.py keeps this job in the Test and Lint aggregate and out
|
||||
# of the archive consumer's dependency chain.
|
||||
workspace-clippy:
|
||||
name: Workspace Clippy
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
needs: [ quick-checks, classify-changes ]
|
||||
# Reads the ci-dev cache that cache-warm.yml writes on this image, and the
|
||||
# all-targets check of the workspace does not fit a hosted runner's disk.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
# Checkout otherwise writes the token into .git/config, where a PR's
|
||||
# own build.rs or proc-macro could read it back out.
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
# Read-only, like every other ci-dev reader; cache-warm.yml warms the
|
||||
# check artifacts of this exact invocation.
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
# Clippy gates every event, pull requests included. When it ran only after
|
||||
# merge, a lint regression merged freely and then failed every main run
|
||||
# until someone fixed it (10 of 12 main runs on 2026-10-17/18). cache-warm
|
||||
# writes the same invocation's check artifacts into ci-dev, so a pull
|
||||
# request re-checks only the crates its diff touches. Main keeps the step
|
||||
# because pull requests can merge without being up to date with main.
|
||||
- name: Run clippy lints
|
||||
run: |
|
||||
./scripts/ci/resource_sampler.sh start clippy
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
python3 scripts/ci_test_artifact.py cargo lint --configuration workspace
|
||||
|
||||
- name: Upload lint diagnostics
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: workspace-clippy-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
|
||||
path: artifacts/workspace
|
||||
retention-days: 3
|
||||
if-no-files-found: error
|
||||
|
||||
# Keeps its job ID and check name: it is the archive producer that
|
||||
# workspace-tests and scripts/check_test_wiring.py bind to. The lint half
|
||||
# is workspace-clippy above; this job builds the test archive, then runs
|
||||
# the doctests and the log-analyzer anchor check on the same build output.
|
||||
workspace-lint-and-build:
|
||||
name: Workspace Lint and Build
|
||||
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
|
||||
@@ -272,6 +211,18 @@ jobs:
|
||||
echo "started_at=$(date --utc --iso-8601=seconds)"
|
||||
} > artifacts/workspace/run-metadata.txt
|
||||
|
||||
# Clippy gates every event, pull requests included. When it ran only after
|
||||
# merge, a lint regression merged freely and then failed every main run
|
||||
# until someone fixed it (10 of 12 main runs on 2026-10-17/18). cache-warm
|
||||
# writes the same invocation's check artifacts into ci-dev, so a pull
|
||||
# request re-checks only the crates its diff touches. Main keeps the step
|
||||
# because pull requests can merge without being up to date with main.
|
||||
- name: Run clippy lints
|
||||
run: |
|
||||
./scripts/ci/resource_sampler.sh start clippy
|
||||
trap './scripts/ci/resource_sampler.sh stop' EXIT
|
||||
python3 scripts/ci_test_artifact.py cargo lint --configuration workspace
|
||||
|
||||
- name: Archive workspace test binaries
|
||||
env:
|
||||
# Workspace test compilation has high per-process memory use.
|
||||
@@ -322,7 +273,7 @@ jobs:
|
||||
# rustfs/backlog#1289: fail if a seed rule's log anchor no longer exists
|
||||
# verbatim in the source tree (log message drifted without updating the
|
||||
# rule). Placed here where the workspace — including the la-dump-anchors
|
||||
# bin's dependencies — is already built by the test archive step above.
|
||||
# bin — is already built by the clippy/test steps above.
|
||||
- name: Check log-analyzer rule anchors
|
||||
run: ./scripts/check_log_analyzer_rules.sh
|
||||
|
||||
@@ -1619,7 +1570,6 @@ jobs:
|
||||
- classify-changes
|
||||
- typos
|
||||
- quick-checks
|
||||
- workspace-clippy
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- workspace-shards
|
||||
@@ -1658,7 +1608,6 @@ jobs:
|
||||
- required-checks
|
||||
- typos
|
||||
- quick-checks
|
||||
- workspace-clippy
|
||||
- workspace-lint-and-build
|
||||
- workspace-tests
|
||||
- core-contract
|
||||
|
||||
+754
-74
@@ -1,99 +1,779 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
name: Build candidate images
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
candidate_manifest:
|
||||
type: string
|
||||
required: true
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Docker Images Workflow
|
||||
#
|
||||
# This workflow builds Docker images using pre-built binaries from the build workflow.
|
||||
#
|
||||
# Trigger Types:
|
||||
# 1. workflow_run: Automatically triggered when "Build and Release" workflow completes
|
||||
# 2. workflow_dispatch: Manual trigger for standalone Docker builds
|
||||
#
|
||||
# Key Features:
|
||||
# - Only triggers when Linux builds (x86_64 + aarch64) are successful
|
||||
# - Independent of macOS/Windows build status
|
||||
# - Uses workflow_run event for precise control
|
||||
# - Only builds Docker images for releases and prereleases (development builds are skipped)
|
||||
|
||||
name: Docker Images
|
||||
|
||||
# Permissions needed for workflow_run event and Docker registry access
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
packages: write
|
||||
|
||||
on:
|
||||
# Automatically triggered when build workflow completes
|
||||
workflow_run:
|
||||
workflows: [ "Build and Release" ]
|
||||
types: [ completed ]
|
||||
# Manual trigger with same parameters for consistency
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
push_images:
|
||||
description: "Push images to registries"
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
version:
|
||||
description: "Version to build (latest, v1.0.0, v1.0.0-alpha1, or v1.0.1-preview.1)"
|
||||
required: false
|
||||
default: "latest"
|
||||
type: string
|
||||
force_rebuild:
|
||||
description: "Force rebuild even if binary exists (useful for testing)"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
|
||||
env:
|
||||
CONCLUSION: ${{ github.event.workflow_run.conclusion }}
|
||||
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
||||
TRIGGERING_EVENT: ${{ github.event.workflow_run.event }}
|
||||
DOCKERHUB_USERNAME: rustfs
|
||||
CARGO_TERM_COLOR: always
|
||||
REGISTRY_DOCKERHUB: rustfs/rustfs
|
||||
REGISTRY_GHCR: ghcr.io/${{ github.repository }}
|
||||
REGISTRY_QUAY: quay.io/rustfs/rustfs
|
||||
DOCKER_PLATFORMS: linux/amd64,linux/arm64
|
||||
|
||||
jobs:
|
||||
build-docker:
|
||||
# Check if we should build Docker images.
|
||||
# Short-circuit at job level so per-merge development builds (workflow_run
|
||||
# from a push to main) skip the whole run without spawning a checkout job:
|
||||
# images are only published for tag builds — build.yml push triggers cover
|
||||
# only main and tags, so a non-main head_branch is a tag name — and for
|
||||
# manual dispatch.
|
||||
build-check:
|
||||
name: Docker Build Check
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push' &&
|
||||
github.event.workflow_run.head_branch != 'main')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
timeout-minutes: 30
|
||||
outputs:
|
||||
should_build: ${{ steps.check.outputs.should_build }}
|
||||
should_push: ${{ steps.check.outputs.should_push }}
|
||||
build_type: ${{ steps.check.outputs.build_type }}
|
||||
version: ${{ steps.check.outputs.version }}
|
||||
short_sha: ${{ steps.check.outputs.short_sha }}
|
||||
is_prerelease: ${{ steps.check.outputs.is_prerelease }}
|
||||
create_latest: ${{ steps.check.outputs.create_latest }}
|
||||
source_ref: ${{ steps.check.outputs.source_ref }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
# For workflow_run events, checkout the specific commit that triggered the workflow
|
||||
ref: ${{ github.event.workflow_run.head_sha || github.sha }}
|
||||
|
||||
- name: Check build conditions
|
||||
id: check
|
||||
env:
|
||||
# dispatch inputs via env, not `${{ }}` interpolation: they are
|
||||
# free-form strings and would otherwise be evaluated by bash.
|
||||
INPUT_VERSION: ${{ github.event.inputs.version }}
|
||||
INPUT_PUSH_IMAGES: ${{ github.event.inputs.push_images }}
|
||||
INPUT_FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }}
|
||||
run: |
|
||||
should_build=false
|
||||
should_push=false
|
||||
build_type="none"
|
||||
version=""
|
||||
short_sha=""
|
||||
is_prerelease=false
|
||||
create_latest=false
|
||||
source_ref="$GITHUB_SHA"
|
||||
|
||||
# Pre-GA policy: until the first stable (vX.Y.Z) tag exists, every
|
||||
# prerelease (alpha/beta/rc) also moves `latest`, so users pulling
|
||||
# `latest` get the newest test build. Once a stable tag is published
|
||||
# this returns false and `latest` follows stable releases only.
|
||||
prerelease_moves_latest() {
|
||||
local stable_tags
|
||||
stable_tags=$(git ls-remote --tags --refs origin 2>/dev/null \
|
||||
| awk '{print $2}' \
|
||||
| grep -E '^refs/tags/v?[0-9]+\.[0-9]+\.[0-9]+$' || true)
|
||||
if [[ -z "$stable_tags" ]]; then
|
||||
return 0
|
||||
fi
|
||||
echo "ℹ️ Stable release tag(s) already exist; prereleases no longer update latest"
|
||||
return 1
|
||||
}
|
||||
|
||||
if [[ "${{ github.event_name }}" == "workflow_run" ]]; then
|
||||
# Triggered by build workflow completion
|
||||
echo "🔗 Triggered by build workflow completion"
|
||||
|
||||
# Check if the triggering workflow was successful
|
||||
# If the workflow succeeded, it means ALL builds (including Linux x86_64 and aarch64) succeeded
|
||||
if [[ "$CONCLUSION" == "success" ]]; then
|
||||
echo "✅ Build workflow succeeded, all builds including Linux are successful"
|
||||
should_build=true
|
||||
should_push=true
|
||||
else
|
||||
echo "❌ Build workflow failed (conclusion: $CONCLUSION), skipping Docker build"
|
||||
should_build=false
|
||||
fi
|
||||
|
||||
# Extract version info from commit message or use commit SHA
|
||||
# Use Git to generate consistent short SHA (ensures uniqueness like build.yml)
|
||||
short_sha=$(git rev-parse --short "$HEAD_SHA")
|
||||
source_ref="$HEAD_SHA"
|
||||
|
||||
# Determine build type based on triggering workflow event and ref
|
||||
triggering_event="$TRIGGERING_EVENT"
|
||||
head_branch="$HEAD_BRANCH"
|
||||
|
||||
echo "🔍 Analyzing triggering workflow:"
|
||||
echo " 📋 Event: $triggering_event"
|
||||
echo " 🌿 Head branch: $head_branch"
|
||||
echo " 📎 Head SHA: $HEAD_SHA"
|
||||
|
||||
# Check if this was triggered by a tag push
|
||||
if [[ "$triggering_event" == "push" ]]; then
|
||||
# For tag pushes, head_branch will be like "refs/tags/v1.0.0" or just "v1.0.0"
|
||||
if [[ "$head_branch" == refs/tags/* ]]; then
|
||||
# Extract tag name from refs/tags/TAG_NAME
|
||||
tag_name="${head_branch#refs/tags/}"
|
||||
version="$tag_name"
|
||||
elif [[ "$head_branch" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+ ]]; then
|
||||
# Direct tag name like "v1.0.0" or "1.0.0-alpha.1"
|
||||
version="$head_branch"
|
||||
elif [[ "$head_branch" == "main" ]]; then
|
||||
# Regular branch push to main
|
||||
build_type="development"
|
||||
version="dev-${short_sha}"
|
||||
should_build=false
|
||||
echo "⏭️ Skipping Docker build for development version (main branch push)"
|
||||
else
|
||||
# Other branch push
|
||||
build_type="development"
|
||||
version="dev-${short_sha}"
|
||||
should_build=false
|
||||
echo "⏭️ Skipping Docker build for development version (branch: $head_branch)"
|
||||
fi
|
||||
|
||||
# If we extracted a version (tag), determine release type
|
||||
if [[ -n "$version" ]] && [[ "$version" != "dev-${short_sha}" ]]; then
|
||||
# Remove 'v' prefix if present for consistent version format
|
||||
if [[ "$version" == v* ]]; then
|
||||
version="${version#v}"
|
||||
fi
|
||||
|
||||
if [[ "$version" =~ -preview\.[0-9]+$ ]]; then
|
||||
build_type="preview"
|
||||
is_prerelease=true
|
||||
echo "🔍 Building Docker image for preview: $version (version tags only)"
|
||||
elif [[ "$version" == *"-preview"* ]]; then
|
||||
echo "❌ Invalid preview tag: $version (expected suffix: -preview.<number>)" >&2
|
||||
exit 1
|
||||
elif [[ "$version" == *"alpha"* ]] || [[ "$version" == *"beta"* ]] || [[ "$version" == *"rc"* ]]; then
|
||||
build_type="prerelease"
|
||||
is_prerelease=true
|
||||
# Pre-GA policy: prereleases update latest until the first stable tag exists.
|
||||
if prerelease_moves_latest; then
|
||||
create_latest=true
|
||||
echo "🧪 Building Docker image for prerelease: $version (creating latest tag)"
|
||||
else
|
||||
echo "🧪 Building Docker image for prerelease: $version"
|
||||
fi
|
||||
else
|
||||
build_type="release"
|
||||
create_latest=true
|
||||
echo "🚀 Building Docker image for release: $version"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
# Non-push events
|
||||
build_type="development"
|
||||
version="dev-${short_sha}"
|
||||
should_build=false
|
||||
echo "⏭️ Skipping Docker build for development version (event: $triggering_event)"
|
||||
fi
|
||||
|
||||
echo "🔄 Build triggered by workflow_run:"
|
||||
echo " 📋 Conclusion: $CONCLUSION"
|
||||
echo " 🌿 Branch: $HEAD_BRANCH"
|
||||
echo " 📎 SHA: $HEAD_SHA"
|
||||
echo " 🎯 Event: $TRIGGERING_EVENT"
|
||||
|
||||
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
|
||||
# Manual trigger
|
||||
input_version="$INPUT_VERSION"
|
||||
version="${input_version}"
|
||||
should_push="$INPUT_PUSH_IMAGES"
|
||||
should_build=true
|
||||
|
||||
# Get short SHA
|
||||
short_sha=$(git rev-parse --short HEAD)
|
||||
|
||||
echo "🎯 Manual Docker build triggered:"
|
||||
echo " 📋 Requested version: $input_version"
|
||||
echo " 🔧 Force rebuild: $INPUT_FORCE_REBUILD"
|
||||
echo " 🚀 Push images: $should_push"
|
||||
|
||||
case "$input_version" in
|
||||
"latest")
|
||||
build_type="release"
|
||||
create_latest=true
|
||||
echo "🚀 Building with latest stable release version"
|
||||
;;
|
||||
*-preview*)
|
||||
if [[ ! "$input_version" =~ ^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?-preview\.[0-9]+$ ]]; then
|
||||
echo "❌ Invalid preview version: $input_version" >&2
|
||||
exit 1
|
||||
fi
|
||||
build_type="preview"
|
||||
is_prerelease=true
|
||||
version="${input_version#v}"
|
||||
echo "🔍 Building with preview version: $version (version tags only)"
|
||||
;;
|
||||
# Prerelease versions (must match first, more specific)
|
||||
v*alpha*|v*beta*|v*rc*|*alpha*|*beta*|*rc*)
|
||||
build_type="prerelease"
|
||||
is_prerelease=true
|
||||
# Pre-GA policy: prereleases update latest until the first stable tag exists.
|
||||
if prerelease_moves_latest; then
|
||||
create_latest=true
|
||||
echo "🧪 Building with prerelease version: $input_version (creating latest tag)"
|
||||
else
|
||||
echo "🧪 Building with prerelease version: $input_version"
|
||||
fi
|
||||
;;
|
||||
# Release versions (match after prereleases, more general)
|
||||
v[0-9]*|[0-9]*.*.*)
|
||||
build_type="release"
|
||||
create_latest=true
|
||||
echo "📦 Building with specific release version: $input_version"
|
||||
;;
|
||||
*)
|
||||
# Invalid version for Docker build
|
||||
should_build=false
|
||||
echo "❌ Invalid version for Docker build: $input_version"
|
||||
echo "⚠️ Only release versions (latest, v1.0.0, 1.0.0) and prereleases (v1.0.0-alpha1, 1.0.0-beta2) are supported"
|
||||
;;
|
||||
esac
|
||||
|
||||
if [[ "$should_build" == true && "$input_version" != "latest" ]]; then
|
||||
tag_ref="refs/tags/$input_version"
|
||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||
if [[ "$input_version" == v* ]]; then
|
||||
tag_ref="refs/tags/${input_version#v}"
|
||||
else
|
||||
tag_ref="refs/tags/v$input_version"
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! git ls-remote --exit-code origin "$tag_ref" >/dev/null 2>&1; then
|
||||
echo "❌ Release tag not found for Docker build: $input_version"
|
||||
exit 1
|
||||
fi
|
||||
source_ref="$tag_ref"
|
||||
fi
|
||||
fi
|
||||
|
||||
{
|
||||
echo "should_build=$should_build"
|
||||
echo "should_push=$should_push"
|
||||
echo "build_type=$build_type"
|
||||
echo "version=$version"
|
||||
echo "short_sha=$short_sha"
|
||||
echo "is_prerelease=$is_prerelease"
|
||||
echo "create_latest=$create_latest"
|
||||
echo "source_ref=$source_ref"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "🐳 Docker Build Summary:"
|
||||
echo " - Should build: $should_build"
|
||||
echo " - Should push: $should_push"
|
||||
echo " - Build type: $build_type"
|
||||
echo " - Version: $version"
|
||||
echo " - Short SHA: $short_sha"
|
||||
echo " - Is prerelease: $is_prerelease"
|
||||
echo " - Create latest: $create_latest"
|
||||
echo " - Source ref: $source_ref"
|
||||
|
||||
# Consume the trusted release certification for the exact version this
|
||||
# workflow wraps and verify the certified input digests. A manual dispatch
|
||||
# without that certification fails here and pushes no official tag; only
|
||||
# results from the trusted verification sources carry release authority.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: build-check
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$run_id" ]]; then
|
||||
echo "No certified Build and Release run found for ${VERSION}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$run_id" -n release-certification -D certification; then
|
||||
echo "Build run ${run_id} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Input digest binding: every zip this image wraps must be the
|
||||
# certified byte-for-byte content, never a mutable latest pointer.
|
||||
while IFS=$'\t' read -r name location expected; do
|
||||
echo "Verifying certified input ${name}"
|
||||
actual="$(curl -fsSL "$location" | sha256sum | awk '{print $1}')"
|
||||
if [[ "$actual" != "$expected" ]]; then
|
||||
echo "Certified input ${name} does not match its recorded digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
done < <(jq -r '.deliverables[] | select(.name | contains("linux")) | [.name, .location, .sha256] | @tsv' certification/candidate.json)
|
||||
|
||||
# Build multi-arch Docker images
|
||||
# Strategy: Build images using pre-built binaries from dl.rustfs.com
|
||||
# Supports both release and dev channel binaries based on build context
|
||||
# Only runs when should_build is true (which includes workflow success check)
|
||||
build-docker:
|
||||
name: Build Docker Images
|
||||
needs: [ build-check, release-certification ]
|
||||
if: needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- variant: musl
|
||||
file: Dockerfile
|
||||
- variant: gnu
|
||||
suffix: ""
|
||||
- variant: glibc
|
||||
file: Dockerfile.glibc
|
||||
env:
|
||||
CANDIDATE_MANIFEST: ${{ inputs.candidate_manifest }}
|
||||
VARIANT: ${{ matrix.variant }}
|
||||
suffix: "-glibc"
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ fromJson(inputs.candidate_manifest).source_sha }}
|
||||
persist-credentials: false
|
||||
- name: Reuse completed image
|
||||
id: reuse
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: python3 scripts/release/deliver.py reuse --kind image --target "$VARIANT"
|
||||
- name: Install OCI tools
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
ref: ${{ needs.build-check.outputs.source_ref }}
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
username: ${{ env.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_PASSWORD }}
|
||||
|
||||
- name: Login to Quay.io
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: quay.io
|
||||
username: ${{ secrets.QUAY_USERNAME }}
|
||||
password: ${{ secrets.QUAY_PASSWORD }}
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
|
||||
- name: Extract metadata and generate tags
|
||||
id: meta
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y skopeo
|
||||
- uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
- uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
- name: Supply accepted binaries and pinned CLI
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: python3 scripts/release/deliver.py image-prepare --target "$VARIANT"
|
||||
- name: Build OCI archive without registry writes
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
|
||||
VERSION="${{ needs.build-check.outputs.version }}"
|
||||
CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}"
|
||||
VARIANT_SUFFIX="${{ matrix.suffix }}"
|
||||
|
||||
# Convert version format for Dockerfile compatibility. The former
|
||||
# DOCKER_CHANNEL was "release" down every branch and was passed as a
|
||||
# build-arg no Dockerfile declares, so it is gone.
|
||||
case "$VERSION" in
|
||||
"latest")
|
||||
DOCKER_RELEASE="latest"
|
||||
;;
|
||||
v*)
|
||||
# For versioned releases (v1.0.0), remove 'v' prefix for Dockerfile
|
||||
DOCKER_RELEASE="${VERSION#v}"
|
||||
;;
|
||||
*)
|
||||
# For other versions, pass as-is
|
||||
DOCKER_RELEASE="${VERSION}"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "docker_release=$DOCKER_RELEASE" >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "🐳 Docker build parameters:"
|
||||
echo " - Original version: $VERSION"
|
||||
echo " - Docker RELEASE: $DOCKER_RELEASE"
|
||||
|
||||
# Generate tags based on build type
|
||||
# Only support release and prerelease builds (no development builds)
|
||||
TAG_BASE="${VERSION}${VARIANT_SUFFIX}"
|
||||
TAGS="${{ env.REGISTRY_DOCKERHUB }}:$TAG_BASE,${{ env.REGISTRY_GHCR }}:$TAG_BASE,${{ env.REGISTRY_QUAY }}:$TAG_BASE"
|
||||
|
||||
# Add latest when requested (stable releases, and prereleases before GA)
|
||||
if [[ "$CREATE_LATEST" == "true" ]]; then
|
||||
TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_GHCR }}:latest${VARIANT_SUFFIX},${{ env.REGISTRY_QUAY }}:latest${VARIANT_SUFFIX}"
|
||||
fi
|
||||
|
||||
# Always add the channel tag for prereleases, independent of latest
|
||||
if [[ "$BUILD_TYPE" == "prerelease" ]]; then
|
||||
# Prerelease channel tags (alpha, beta, rc)
|
||||
if [[ "$VERSION" == *"alpha"* ]]; then
|
||||
CHANNEL="alpha"
|
||||
elif [[ "$VERSION" == *"beta"* ]]; then
|
||||
CHANNEL="beta"
|
||||
elif [[ "$VERSION" == *"rc"* ]]; then
|
||||
CHANNEL="rc"
|
||||
fi
|
||||
|
||||
if [[ -n "$CHANNEL" ]]; then
|
||||
TAGS="$TAGS,${{ env.REGISTRY_DOCKERHUB }}:${CHANNEL}${VARIANT_SUFFIX},${{ env.REGISTRY_GHCR }}:${CHANNEL}${VARIANT_SUFFIX},${{ env.REGISTRY_QUAY }}:${CHANNEL}${VARIANT_SUFFIX}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Output tags
|
||||
echo "tags=$TAGS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Staging identity for this run's promotion: official tags are only
|
||||
# created after the staged image verifies against the certification,
|
||||
# and they point at this same digest (no rebuild-after-verify).
|
||||
STAGING_TAG="staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${VARIANT_SUFFIX}"
|
||||
STAGING_TAGS="${{ env.REGISTRY_DOCKERHUB }}:${STAGING_TAG},${{ env.REGISTRY_GHCR }}:${STAGING_TAG},${{ env.REGISTRY_QUAY }}:${STAGING_TAG}"
|
||||
echo "staging_tags=$STAGING_TAGS" >> "$GITHUB_OUTPUT"
|
||||
echo "staging_tag=$STAGING_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Generate labels
|
||||
LABELS="org.opencontainers.image.title=RustFS"
|
||||
LABELS="$LABELS,org.opencontainers.image.description=RustFS distributed object storage system"
|
||||
LABELS="$LABELS,org.opencontainers.image.version=$VERSION"
|
||||
SOURCE_REVISION="$(git rev-parse HEAD)"
|
||||
LABELS="$LABELS,org.opencontainers.image.revision=$SOURCE_REVISION"
|
||||
LABELS="$LABELS,org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}"
|
||||
LABELS="$LABELS,org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
||||
LABELS="$LABELS,org.opencontainers.image.build-type=$BUILD_TYPE"
|
||||
|
||||
echo "labels=$LABELS" >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "🐳 Generated Docker tags:"
|
||||
echo "$TAGS" | tr ',' '\n' | sed 's/^/ - /'
|
||||
echo "📋 Build type: $BUILD_TYPE"
|
||||
echo "🔖 Version: $VERSION"
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
|
||||
with:
|
||||
context: .
|
||||
file: ${{ matrix.file }}
|
||||
platforms: linux/amd64,linux/arm64
|
||||
push: false
|
||||
provenance: false
|
||||
outputs: type=oci,dest=delivery/image-${{ matrix.variant }}.tar
|
||||
platforms: ${{ env.DOCKER_PLATFORMS }}
|
||||
push: ${{ needs.build-check.outputs.should_push == 'true' }}
|
||||
# Push to staging tags only: official tags are created by the
|
||||
# certification-gated promotion step below from this same digest.
|
||||
tags: ${{ steps.meta.outputs.staging_tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
# No layer cache. This build compiles nothing — it downloads a
|
||||
# release zip and runs apk/apt — so the cache could only save the
|
||||
# minute or two those take, while creating a correctness problem: with
|
||||
# RELEASE=latest the binary URL is resolved by curl *inside* a RUN
|
||||
# layer, and the layer key does not include what that resolved to. A
|
||||
# rebuild at the same RELEASE value (dispatch with version=latest, or
|
||||
# a re-run of the same version) would hit the old layer and ship the
|
||||
# previous release's binary. mode=max also consumed the same 10GB
|
||||
# Actions cache quota the Rust lanes are fighting over.
|
||||
#
|
||||
# Only RELEASE is passed: it is the sole build-arg the Dockerfiles
|
||||
# declare besides TARGETARCH. BUILDTIME, VERSION, BUILD_TYPE, REVISION
|
||||
# and CHANNEL were never read by any stage (and BUILDTIME's $(date ...)
|
||||
# was a literal here, not a shell substitution). BUILD_DATE and VCS_REF
|
||||
# are declared by the Dockerfiles but deliberately left unset —
|
||||
# supplying them would change the published image labels.
|
||||
build-args: |
|
||||
BINARY_SOURCE=supplied
|
||||
RELEASE=${{ fromJson(inputs.candidate_manifest).version }}
|
||||
VCS_REF=${{ fromJson(inputs.candidate_manifest).source_sha }}
|
||||
- name: Verify wrapped binaries and startup on both architectures
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
run: python3 scripts/release/deliver.py image-smoke --target "$VARIANT"
|
||||
- name: Scan amd64 image before sealing
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25
|
||||
RELEASE=${{ steps.meta.outputs.docker_release }}
|
||||
provenance: true
|
||||
sbom: true
|
||||
# Add retry mechanism by splitting the build process
|
||||
no-cache: false
|
||||
pull: true
|
||||
|
||||
# A later-built wrapper: the staged image must complete this platform's
|
||||
# own binary identity verification before any official tag is written.
|
||||
# It may not inherit an unverified package's conclusion.
|
||||
- name: Verify staged image against certification
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
env:
|
||||
TRIVY_PLATFORM: linux/amd64
|
||||
with:
|
||||
input: delivery-scan/image-${{ matrix.variant }}-amd64.tar
|
||||
format: sarif
|
||||
output: delivery/scan-${{ matrix.variant }}-amd64.sarif
|
||||
exit-code: "0" # Keep the repository's report-only vulnerability policy.
|
||||
- name: Scan arm64 image before sealing
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
case "${{ matrix.variant }}" in
|
||||
musl) libc="musl" ;;
|
||||
glibc) libc="gnu" ;;
|
||||
esac
|
||||
for platform in linux/amd64 linux/arm64; do
|
||||
case "$platform" in
|
||||
linux/amd64) arch_sub="x86_64" ;;
|
||||
linux/arm64) arch_sub="aarch64" ;;
|
||||
esac
|
||||
docker pull --platform "$platform" "$STAGING_REF"
|
||||
container="$(docker create --platform "$platform" "$STAGING_REF")"
|
||||
docker cp "$container:/usr/bin/rustfs" "$RUNNER_TEMP/staged-rustfs"
|
||||
docker rm "$container" >/dev/null
|
||||
actual="$(sha256sum "$RUNNER_TEMP/staged-rustfs" | awk '{print $1}')"
|
||||
expected="$(jq -r --arg needle "linux-${arch_sub}-${libc}" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" || "$actual" != "$expected" ]]; then
|
||||
echo "Staged image binary for ${platform} does not match the certified binary digest" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/staged-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Promote staged image to official tags
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
IFS=',' read -r -a official_tags <<< "${{ steps.meta.outputs.tags }}"
|
||||
for tag in "${official_tags[@]}"; do
|
||||
docker buildx imagetools create -t "$tag" "$STAGING_REF"
|
||||
echo "Promoted $tag -> $STAGING_REF"
|
||||
done
|
||||
|
||||
# Downstream consumers (Helm) verify this record instead of inheriting
|
||||
# an unverified wrapper's conclusion.
|
||||
- name: Record image promotion
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
shell: bash
|
||||
env:
|
||||
TRIVY_PLATFORM: linux/arm64
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
VERSION: ${{ needs.build-check.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
run_id="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$run_id" ]]; then
|
||||
run_id="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$VERSION" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
gh run download "$run_id" -n release-certification -D certification
|
||||
STAGING_REF="${{ env.REGISTRY_DOCKERHUB }}:staging-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}${{ matrix.suffix }}"
|
||||
digest="$(docker buildx imagetools inspect "$STAGING_REF" --format '{{json .Manifest}}' | jq -r '.digest')"
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
jq -n \
|
||||
--arg version "$VERSION" \
|
||||
--arg digest "$digest" \
|
||||
--arg staging_ref "$STAGING_REF" \
|
||||
--arg candidate_sha256 "$candidate_sha" \
|
||||
--arg tags "${{ steps.meta.outputs.tags }}" \
|
||||
--slurpfile candidate certification/candidate.json \
|
||||
'{schema_version: 1, version: $version, image_digest: $digest, staging_ref: $staging_ref,
|
||||
promoted_tags: ($tags | split(",")), candidate_sha256: $candidate_sha256,
|
||||
binary_sha256s: ($candidate[0].deliverables | map(.binary_sha256) | unique)}' \
|
||||
> image-promotion-record.json
|
||||
|
||||
- name: Upload image promotion record
|
||||
if: needs.build-check.outputs.should_push == 'true'
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
input: delivery-scan/image-${{ matrix.variant }}-arm64.tar
|
||||
format: sarif
|
||||
output: delivery/scan-${{ matrix.variant }}-arm64.sarif
|
||||
exit-code: "0"
|
||||
- name: Record verified OCI identity
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
run: python3 scripts/release/deliver.py image-record --target "$VARIANT"
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
with:
|
||||
name: image-${{ matrix.variant }}
|
||||
path: delivery/*
|
||||
name: image-promotion-record${{ matrix.suffix }}
|
||||
path: image-promotion-record.json
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
compression-level: 0
|
||||
|
||||
# Note: Manifest creation is no longer needed as we only build one variant
|
||||
# Multi-arch manifests are automatically created by docker/build-push-action
|
||||
|
||||
# Report-only container image vulnerability scan
|
||||
scan-docker-image:
|
||||
name: Scan Docker Images
|
||||
needs: [ build-check, build-docker ]
|
||||
if: needs.build-check.outputs.should_build == 'true' && needs.build-check.outputs.should_push == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
include:
|
||||
- variant: musl
|
||||
suffix: ""
|
||||
- variant: glibc
|
||||
suffix: "-glibc"
|
||||
steps:
|
||||
- name: Login to GitHub Container Registry
|
||||
# docker/login-action v3
|
||||
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_PASSWORD }}
|
||||
|
||||
- name: Scan image with Trivy
|
||||
# aquasecurity/trivy-action v0.36.0
|
||||
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25
|
||||
with:
|
||||
image-ref: ${{ env.REGISTRY_GHCR }}:${{ needs.build-check.outputs.version }}${{ matrix.suffix }}
|
||||
format: sarif
|
||||
output: trivy-${{ matrix.variant }}.sarif
|
||||
ignore-unfixed: true
|
||||
vuln-type: os,library
|
||||
severity: CRITICAL,HIGH
|
||||
exit-code: "0"
|
||||
|
||||
# Surface findings in the Security tab; an artifact alone is write-only
|
||||
# reporting nobody reviews.
|
||||
- name: Upload scan results to code scanning
|
||||
uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4.37.0
|
||||
with:
|
||||
sarif_file: trivy-${{ matrix.variant }}.sarif
|
||||
category: container-image-${{ matrix.variant }}
|
||||
|
||||
- name: Upload container scan report
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: container-image-scan-${{ matrix.variant }}
|
||||
path: trivy-${{ matrix.variant }}.sarif
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
# Docker build summary
|
||||
docker-summary:
|
||||
name: Docker Build Summary
|
||||
needs: [ build-check, build-docker ]
|
||||
if: always() && needs.build-check.outputs.should_build == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Docker build completion summary
|
||||
run: |
|
||||
BUILD_TYPE="${{ needs.build-check.outputs.build_type }}"
|
||||
VERSION="${{ needs.build-check.outputs.version }}"
|
||||
CREATE_LATEST="${{ needs.build-check.outputs.create_latest }}"
|
||||
|
||||
echo "🐳 Docker build completed successfully!"
|
||||
echo "📦 Build type: $BUILD_TYPE"
|
||||
echo "🔢 Version: $VERSION"
|
||||
echo "🚀 Strategy: Images using pre-built binaries (release channel only)"
|
||||
echo ""
|
||||
|
||||
case "$BUILD_TYPE" in
|
||||
"preview")
|
||||
echo "🔍 Preview Docker image has been built with ${VERSION} tags"
|
||||
echo "⏭️ Preview images do not update latest or prerelease channels"
|
||||
;;
|
||||
"release")
|
||||
echo "🚀 Release Docker image has been built with ${VERSION} tags"
|
||||
echo "✅ This image is ready for production use"
|
||||
if [[ "$CREATE_LATEST" == "true" ]]; then
|
||||
echo "🏷️ Latest tag has been created for stable release"
|
||||
fi
|
||||
;;
|
||||
"prerelease")
|
||||
echo "🧪 Prerelease Docker image has been built with ${VERSION} tags"
|
||||
echo "⚠️ This is a prerelease image - use with caution"
|
||||
# Prereleases move latest until the first stable tag exists (pre-GA policy).
|
||||
if [[ "$CREATE_LATEST" == "true" ]]; then
|
||||
echo "🏷️ Latest tag has been created for prerelease: $VERSION"
|
||||
else
|
||||
echo "🚫 Latest tag NOT created for prerelease"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "❌ Unexpected build type: $BUILD_TYPE"
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -1,26 +1,229 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
name: Build candidate Helm chart
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: Publish helm chart to artifacthub
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
workflow_run:
|
||||
workflows: [ "Build and Release" ]
|
||||
types: [ completed ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
candidate_manifest:
|
||||
type: string
|
||||
version:
|
||||
description: "Release version to publish, e.g. 1.0.0-beta.1 or v1.0.0-beta.1"
|
||||
required: true
|
||||
default: "1.0.0-beta.1"
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
package:
|
||||
# Consume the same trusted release certification as every other publication
|
||||
# exit and verify the image promotion record of the Docker workflow. The
|
||||
# chart wraps no binary of its own, so it may only reference an image whose
|
||||
# platform verification already passed; it must not inherit an unverified
|
||||
# package's conclusion. A run without that evidence fails closed here and
|
||||
# never reaches the chart repository.
|
||||
release-certification:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.raw_tag }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ fromJson(inputs.candidate_manifest).source_sha }}
|
||||
persist-credentials: false
|
||||
- uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
|
||||
- name: Install pinned yq
|
||||
|
||||
- name: Normalize release version
|
||||
id: version
|
||||
env:
|
||||
RAW_INPUT: ${{ github.event.inputs.version }}
|
||||
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
RAW="$RAW_INPUT"
|
||||
else
|
||||
RAW="$RAW_BRANCH"
|
||||
fi
|
||||
case "$RAW" in
|
||||
refs/tags/*) RAW_TAG="${RAW#refs/tags/}" ;;
|
||||
*) RAW_TAG="$RAW" ;;
|
||||
esac
|
||||
echo "raw_tag=$RAW_TAG" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download and verify release certification and image record
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
TRIGGERING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
RAW_TAG: ${{ steps.version.outputs.raw_tag }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification image-record
|
||||
|
||||
build_run="$TRIGGERING_RUN_ID"
|
||||
if [[ -z "$build_run" ]]; then
|
||||
build_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
fi
|
||||
if [[ -z "$build_run" ]]; then
|
||||
echo "No certified Build and Release run found for ${RAW_TAG}; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! gh run download "$build_run" -n release-certification -D certification; then
|
||||
echo "Build run ${build_run} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
docker_run="$(gh api --method GET \
|
||||
"repos/${GITHUB_REPOSITORY}/actions/workflows/docker.yml/runs" \
|
||||
-f branch="$RAW_TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' || true)"
|
||||
if [[ -z "$docker_run" ]] || ! gh run download "$docker_run" -n image-promotion-record -D image-record; then
|
||||
echo "No image promotion record for ${RAW_TAG}; a chart may not wrap an unverified image" >&2
|
||||
exit 1
|
||||
fi
|
||||
candidate_sha="$(python3 -c "import json,hashlib,sys; sys.path.insert(0,'scripts/release'); import release_gate; print(release_gate.canonical_sha256(json.load(open('certification/candidate.json'))))")"
|
||||
record_sha="$(jq -r '.candidate_sha256 // empty' image-record/image-promotion-record.json)"
|
||||
if [[ "$record_sha" != "$candidate_sha" ]]; then
|
||||
echo "Image promotion record does not belong to this certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
overlap="$(jq -r --slurpfile candidate certification/candidate.json \
|
||||
'[.binary_sha256s[]] - ($candidate[0].deliverables | map(.binary_sha256)) | length' \
|
||||
image-record/image-promotion-record.json)"
|
||||
if [[ "$overlap" -ne 0 ]]; then
|
||||
echo "Image promotion record carries binary digests outside the certification" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build-helm-package:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ release-certification ]
|
||||
if: |
|
||||
(github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) ||
|
||||
(
|
||||
github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push' &&
|
||||
contains(github.event.workflow_run.head_branch, '.') &&
|
||||
!contains(github.event.workflow_run.head_branch, '-preview')
|
||||
)
|
||||
|
||||
outputs:
|
||||
raw_tag: ${{ steps.version.outputs.raw_tag }}
|
||||
app_version: ${{ steps.version.outputs.app_version }}
|
||||
chart_version: ${{ steps.version.outputs.chart_version }}
|
||||
|
||||
steps:
|
||||
- name: Checkout release source
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ github.event.workflow_run.head_sha || (startsWith(inputs.version, 'refs/tags/') && inputs.version || format('refs/tags/{0}', inputs.version)) }}
|
||||
persist-credentials: false
|
||||
|
||||
# Both inputs reach the shell through env rather than `${{ }}`
|
||||
# interpolation. A git ref name may contain `$(...)` — anything without a
|
||||
# space is a legal tag — and interpolation pastes it into the script
|
||||
# verbatim, where bash would run it. Reading "$RAW_INPUT" instead makes it
|
||||
# data.
|
||||
- name: Normalize release version
|
||||
id: version
|
||||
env:
|
||||
RAW_INPUT: ${{ github.event.inputs.version }}
|
||||
RAW_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
run: |
|
||||
set -eux
|
||||
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
RAW="$RAW_INPUT"
|
||||
else
|
||||
RAW="$RAW_BRANCH"
|
||||
fi
|
||||
|
||||
case "$RAW" in
|
||||
refs/tags/*)
|
||||
RAW_TAG="${RAW#refs/tags/}"
|
||||
;;
|
||||
*)
|
||||
RAW_TAG="$RAW"
|
||||
;;
|
||||
esac
|
||||
|
||||
./scripts/helm_chart_version.sh "$RAW_TAG"
|
||||
|
||||
# Docker and Helm share the binary-build trigger. Do not publish a chart
|
||||
# until the image selected by its default appVersion is pullable.
|
||||
- name: Wait for release image
|
||||
timeout-minutes: 20
|
||||
env:
|
||||
APP_VERSION: ${{ steps.version.outputs.app_version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ ! "$APP_VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?$ ]] ||
|
||||
[[ "$APP_VERSION" == *-preview* ]]; then
|
||||
echo "Invalid deliverable image version: $APP_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for attempt in {1..60}; do
|
||||
if token=$(curl --fail --silent --show-error --connect-timeout 10 --max-time 20 \
|
||||
'https://auth.docker.io/token?service=registry.docker.io&scope=repository:rustfs/rustfs:pull' \
|
||||
| jq -er '.token // empty') &&
|
||||
curl --fail --silent --show-error --head --connect-timeout 10 --max-time 20 \
|
||||
--header "Authorization: Bearer $token" \
|
||||
--header 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json' \
|
||||
"https://registry-1.docker.io/v2/rustfs/rustfs/manifests/$APP_VERSION" > /dev/null; then
|
||||
echo "Release image is available: rustfs/rustfs:$APP_VERSION"
|
||||
exit 0
|
||||
fi
|
||||
echo "Waiting for rustfs/rustfs:$APP_VERSION ($attempt/60)"
|
||||
if (( attempt < 60 )); then
|
||||
sleep 15
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Release image is unavailable; refusing to publish Helm chart" >&2
|
||||
exit 1
|
||||
|
||||
- name: Replace chart version and app version
|
||||
env:
|
||||
CHART_VERSION: ${{ steps.version.outputs.chart_version }}
|
||||
APP_VERSION: ${{ steps.version.outputs.app_version }}
|
||||
run: |
|
||||
set -eux
|
||||
sed -i -E "s/^version:.*/version: \"${CHART_VERSION}\"/" helm/rustfs/Chart.yaml
|
||||
sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" helm/rustfs/Chart.yaml
|
||||
|
||||
- name: Set up Helm
|
||||
uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
|
||||
|
||||
- name: Install yq
|
||||
run: |
|
||||
set -euo pipefail
|
||||
yq_version="4.54.1"
|
||||
@@ -35,43 +238,66 @@ jobs:
|
||||
echo "$tools_dir" >> "$GITHUB_PATH"
|
||||
"$tools_dir/yq" --version
|
||||
|
||||
- name: Test and package the exact application version
|
||||
id: chart
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CANDIDATE_MANIFEST: ${{ inputs.candidate_manifest }}
|
||||
- name: Test Helm Chart Templates
|
||||
run: ./scripts/test_helm_templates.sh
|
||||
|
||||
- name: Package Helm Chart
|
||||
run: |
|
||||
python3 - <<'PYCHART'
|
||||
import sys, subprocess, shutil
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, 'scripts/release')
|
||||
import candidate as c, pipeline
|
||||
inputs = pipeline.manifest_env()
|
||||
current = pipeline.current_run('release-complete.yml')
|
||||
if any(a['name'] == 'helm-package' for a in c.artifact_list(current['id'])):
|
||||
c.download_artifact(current, 'helm-package', 'chart')
|
||||
record = c.read_json('chart/chart.json')
|
||||
c.require(record['inputs_sha256'] == c.digest(inputs), 'chart belongs to another candidate')
|
||||
c.emit_output('reused', 'true')
|
||||
else:
|
||||
chart = Path('helm/rustfs/Chart.yaml')
|
||||
import re
|
||||
value = chart.read_text()
|
||||
for key in ('version', 'appVersion'):
|
||||
value, count = re.subn(rf'(?m)^{key}:.*$', f'{key}: "{inputs["version"]}"', value)
|
||||
c.require(count == 1, 'ambiguous chart metadata')
|
||||
chart.write_text(value)
|
||||
subprocess.run(['bash', 'scripts/test_helm_templates.sh'], check=True)
|
||||
shutil.copyfile('helm/README.md', 'helm/rustfs/README.md')
|
||||
subprocess.run(['helm', 'package', 'helm/rustfs', '--destination', 'chart'], check=True)
|
||||
path = Path('chart') / f"rustfs-{inputs['version']}.tgz"
|
||||
c.write_json('chart/chart.json', {'version': inputs['version'], 'inputs_sha256': c.digest(inputs),
|
||||
'name': path.name, 'sha256': c.file_digest(path), 'size': path.stat().st_size})
|
||||
PYCHART
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: steps.chart.outputs.reused != 'true'
|
||||
set -eux
|
||||
cp helm/README.md helm/rustfs/
|
||||
helm package ./helm/rustfs \
|
||||
--destination helm/rustfs/ \
|
||||
--version "${{ steps.version.outputs.chart_version }}"
|
||||
|
||||
- name: Upload helm package as artifact
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: helm-package
|
||||
path: chart/*
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
path: |
|
||||
helm/rustfs/*.tgz
|
||||
helm/rustfs/README.md
|
||||
retention-days: 1
|
||||
|
||||
publish-helm-package:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
needs: [ build-helm-package ]
|
||||
if: needs.build-helm-package.result == 'success'
|
||||
|
||||
steps:
|
||||
- name: Checkout helm package repo
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
# persist-credentials-exempt: this checkout's token IS the push credential —
|
||||
# the job git-pushes to rustfs/helm below. Clearing it breaks chart publishing.
|
||||
repository: rustfs/helm
|
||||
token: ${{ secrets.RUSTFS_HELM_PACKAGE }}
|
||||
|
||||
- name: Download helm package
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: helm-package
|
||||
path: ./
|
||||
|
||||
- name: Set up helm
|
||||
uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
|
||||
|
||||
- name: Generate index
|
||||
run: helm repo index . --url https://charts.rustfs.com
|
||||
|
||||
# app_version is derived from the triggering tag name, and this job holds
|
||||
# the cross-repository push token with rustfs/helm already checked out —
|
||||
# the worst place in the repo to paste an attacker-influenced string into
|
||||
# a shell line. Passed through env so bash treats it as data.
|
||||
- name: Push helm package and index file
|
||||
env:
|
||||
GIT_USERNAME: ${{ secrets.USERNAME }}
|
||||
GIT_EMAIL: ${{ secrets.EMAIL_ADDRESS }}
|
||||
APP_VERSION: ${{ needs.build-helm-package.outputs.app_version }}
|
||||
run: |
|
||||
set -eux
|
||||
git config --global user.name "${GIT_USERNAME}"
|
||||
git config --global user.email "${GIT_EMAIL}"
|
||||
git add .
|
||||
git commit -m "Update rustfs helm package with ${APP_VERSION}." || echo "No changes to commit"
|
||||
git push origin main
|
||||
|
||||
@@ -1,415 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Manual packaging of the current origin/main tip. Workflow artifacts only.
|
||||
# Dispatch uses this file from the selected branch. The build fetches
|
||||
# refs/heads/main and checks out that commit, not the dispatch ref or a tag.
|
||||
|
||||
name: Manual main packages
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
architecture:
|
||||
description: "Linux targets: x86 (x86_64 musl+gnu), arm (aarch64 musl+gnu), or all"
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- x86
|
||||
- arm
|
||||
- all
|
||||
default: all
|
||||
version:
|
||||
description: "Binary and package version: MAJOR.MINOR.PATCH, or MAJOR.MINOR.PATCH-(alpha|beta|rc).N. No leading v."
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
CARGO_INCREMENTAL: 0
|
||||
|
||||
jobs:
|
||||
prepare:
|
||||
name: Resolve main tip and version
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
outputs:
|
||||
source_sha: ${{ steps.pin.outputs.source_sha }}
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
toolchain: ${{ steps.inputs.outputs.toolchain }}
|
||||
console: ${{ steps.inputs.outputs.console }}
|
||||
matrix: ${{ steps.matrix.outputs.matrix }}
|
||||
steps:
|
||||
- name: Checkout main
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: main
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Pin the tip of origin/main
|
||||
id: pin
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
|
||||
sha=$(git rev-parse FETCH_HEAD)
|
||||
git checkout --detach "$sha"
|
||||
test "$(git rev-parse HEAD)" = "$sha"
|
||||
printf 'source_sha=%s\n' "$sha" >> "$GITHUB_OUTPUT"
|
||||
printf 'Pinned origin/main %s\n' "$sha"
|
||||
|
||||
- name: Validate package version
|
||||
id: version
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
bash scripts/release/package_manual_artifacts.sh validate \
|
||||
--version "$PACKAGE_VERSION" \
|
||||
--arch x86_64 \
|
||||
--libc gnu | tee "$RUNNER_TEMP/package-version.txt"
|
||||
printf 'version=%s\n' "$PACKAGE_VERSION" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Install stable Rust
|
||||
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
|
||||
with:
|
||||
toolchain: stable
|
||||
|
||||
- name: Resolve console assets and toolchain
|
||||
id: inputs
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
toolchain=$(rustc --version | awk '{print $2}')
|
||||
[[ "$toolchain" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]
|
||||
gh api repos/rustfs/console/releases/latest > console-release.json
|
||||
TOOLCHAIN="$toolchain" python3 - <<'PY'
|
||||
import json, os, sys
|
||||
sys.path.insert(0, "scripts/release")
|
||||
import candidate
|
||||
console = candidate.console_asset(candidate.read_json("console-release.json"))
|
||||
candidate.emit_output("console", json.dumps(console, separators=(",", ":")))
|
||||
candidate.emit_output("toolchain", os.environ["TOOLCHAIN"])
|
||||
print(f"toolchain={os.environ['TOOLCHAIN']}")
|
||||
print(f"console={console['tag']} {console['name']}")
|
||||
PY
|
||||
|
||||
- name: Select Linux targets
|
||||
id: matrix
|
||||
shell: bash
|
||||
env:
|
||||
ARCHITECTURE: ${{ inputs.architecture }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
matrix=$(bash scripts/release/package_manual_artifacts.sh matrix "$ARCHITECTURE")
|
||||
count=$(python3 -c 'import json,sys; print(len(json.loads(sys.argv[1])["include"]))' "$matrix")
|
||||
[[ "$count" -gt 0 ]]
|
||||
printf 'matrix=%s\n' "$matrix" >> "$GITHUB_OUTPUT"
|
||||
printf 'Selected %s target(s) for %s\n' "$count" "$ARCHITECTURE"
|
||||
|
||||
package:
|
||||
name: Package RustFS (${{ matrix.target_id }})
|
||||
needs: prepare
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
# x86_64 uses sm-standard-4, the official release compile pool: the workspace
|
||||
# release build needs that disk (placement rule: disk) and reads build.yml's
|
||||
# cache without writing it. aarch64 uses ubicloud-standard-16-arm so the
|
||||
# packaged binary and console smoke execute natively (placement rule: hardware).
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 210
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
RUSTUP_TOOLCHAIN: ${{ needs.prepare.outputs.toolchain }}
|
||||
RUSTFLAGS: "${{ matrix.rustflags }}"
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
|
||||
steps:
|
||||
- name: Checkout the pinned origin/main commit
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ needs.prepare.outputs.source_sha }}
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
|
||||
- name: Confirm the checkout is that main commit
|
||||
shell: bash
|
||||
env:
|
||||
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
|
||||
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
|
||||
git merge-base --is-ancestor "$SOURCE_SHA" FETCH_HEAD
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: ${{ needs.prepare.outputs.toolchain }}
|
||||
target: ${{ matrix.target }}
|
||||
cache-shared-key: build-${{ matrix.target }}${{ matrix.glibc && format('-glibc-{0}', matrix.glibc) || '' }}${{ matrix.os == 'ubicloud-standard-16-arm' && '-ubicloud-arm' || '' }}
|
||||
# build.yml owns this key. This job only restores it.
|
||||
cache-save-if: 'false'
|
||||
install-cross-tools: ${{ matrix.cross }}
|
||||
install-build-packaging-tools: 'true'
|
||||
install-test-tools: 'false'
|
||||
|
||||
- name: Download static console assets
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CONSOLE_IDENTITY: ${{ needs.prepare.outputs.console }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PYCONSOLE'
|
||||
import json, os, pathlib, subprocess, zipfile
|
||||
value = json.loads(os.environ["CONSOLE_IDENTITY"])
|
||||
subprocess.run(["python3", ".github/actions/setup/download_release_asset.py",
|
||||
"--repository", "rustfs/console", "--tag", value["tag"], "--asset", value["name"],
|
||||
"--sha256", value["sha256"], "--output", "console.zip"], check=True)
|
||||
destination = pathlib.Path("rustfs/static")
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
with zipfile.ZipFile("console.zip") as archive:
|
||||
for member in archive.infolist():
|
||||
path = pathlib.PurePosixPath(member.filename)
|
||||
if path.is_absolute() or ".." in path.parts or "\\" in member.filename:
|
||||
raise SystemExit("unsafe Console archive member")
|
||||
archive.extractall(destination)
|
||||
if not (destination / "index.html").is_file():
|
||||
raise SystemExit("Console archive has no index.html")
|
||||
PYCONSOLE
|
||||
rm -f console.zip
|
||||
|
||||
- name: Build RustFS
|
||||
shell: bash
|
||||
env:
|
||||
RUSTFS_BUILD_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
BUILD_OS: ${{ matrix.build_os }}
|
||||
BUILD_FEATURES: ${{ matrix.features }}
|
||||
BUILD_GLIBC: ${{ matrix.glibc }}
|
||||
BUILD_CROSS: ${{ matrix.cross }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
touch rustfs/build.rs
|
||||
|
||||
feature_args=()
|
||||
if [[ -n "$BUILD_FEATURES" ]]; then
|
||||
feature_args=(--features "$BUILD_FEATURES")
|
||||
fi
|
||||
|
||||
# gnu targets pin glibc 2.31 through cargo-zigbuild (<target>.<glibc>).
|
||||
target_arg="$BUILD_TARGET"
|
||||
if [[ -n "$BUILD_GLIBC" ]]; then
|
||||
target_arg="${target_arg}.${BUILD_GLIBC}"
|
||||
fi
|
||||
|
||||
profile_args=()
|
||||
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
|
||||
profile_args=(--config 'profile.release.strip="none"')
|
||||
fi
|
||||
|
||||
case "$BUILD_CROSS" in
|
||||
true)
|
||||
cargo zigbuild --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
|
||||
;;
|
||||
false)
|
||||
cargo build --release --locked --target "$target_arg" "${feature_args[@]}" "${profile_args[@]}" -p rustfs --bin rustfs
|
||||
;;
|
||||
*)
|
||||
printf 'unexpected cross flag: %q\n' "$BUILD_CROSS" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
bin="target/${BUILD_TARGET}/release/rustfs"
|
||||
version_output=$("$bin" --version)
|
||||
first=${version_output%%$'\n'*}
|
||||
if [[ "$first" != "rustfs ${RUSTFS_BUILD_VERSION}" ]]; then
|
||||
printf 'binary version line is %q, expected %q\n' "$first" "rustfs ${RUSTFS_BUILD_VERSION}" >&2
|
||||
printf '%s\n' "$version_output" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
|
||||
! grep -Fxq "build os : ${BUILD_OS}" <<< "$version_output" || \
|
||||
! grep -Fxq "rust channel : stable-${BUILD_TARGET}" <<< "$version_output"; then
|
||||
printf 'release binary reports an unexpected compile target\n' >&2
|
||||
printf '%s\n' "$version_output" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -n "$BUILD_GLIBC" ]]; then
|
||||
python3 scripts/release/check_glibc.py "$BUILD_GLIBC" "$bin"
|
||||
fi
|
||||
|
||||
- name: Generate the CPU symbol catalog
|
||||
if: endsWith(matrix.target, '-linux-gnu') && matrix.features == 'pyroscope'
|
||||
shell: bash
|
||||
env:
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
cargo metadata --locked --format-version 1 --filter-platform "$BUILD_TARGET" \
|
||||
--features rustfs/pyroscope > "$RUNNER_TEMP/cpu-catalog-metadata.json"
|
||||
features=$(jq -ce '
|
||||
(.packages[] | select(.name == "rustfs") | .id) as $id |
|
||||
.resolve.nodes[] | select(.id == $id) | .features | map(select(. != "default")) | sort
|
||||
' "$RUNNER_TEMP/cpu-catalog-metadata.json")
|
||||
release_dir="target/${BUILD_TARGET}/release"
|
||||
rm -f "$release_dir/rustfs.cpu-symbol-catalog.json" "$release_dir/rustfs.cpu-symbol-catalog.sha256"
|
||||
cargo run --locked -p rustfs-connect-symbol-catalog -- \
|
||||
"$release_dir/rustfs" "$(git rev-parse HEAD)" "$BUILD_TARGET" "$features" \
|
||||
"$release_dir/rustfs.cpu-symbol-catalog.json" > "$release_dir/rustfs.cpu-symbol-catalog.sha256"
|
||||
|
||||
- name: Install DEB and RPM tools
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ruby ruby-dev build-essential rpm fakeroot cpio
|
||||
sudo gem install fpm --version 1.17.0 --no-document
|
||||
|
||||
- name: Package zip, deb, and rpm
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
BUILD_FEATURES: ${{ matrix.features }}
|
||||
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
binary="target/${BUILD_TARGET}/release/rustfs"
|
||||
extra=()
|
||||
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
|
||||
extra=(
|
||||
--zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.json"
|
||||
--zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.sha256"
|
||||
)
|
||||
fi
|
||||
bash scripts/release/package_manual_artifacts.sh package \
|
||||
--version "$PACKAGE_VERSION" \
|
||||
--arch "$ARCH" \
|
||||
--libc "$LIBC" \
|
||||
--binary "$binary" \
|
||||
--output dist \
|
||||
"${extra[@]}"
|
||||
printf 'Packaged %s from origin/main %s\n' "$PACKAGE_VERSION" "$SOURCE_SHA"
|
||||
|
||||
- name: Verify the packaged binary
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
BUILD_TARGET: ${{ matrix.target }}
|
||||
BUILD_OS: ${{ matrix.build_os }}
|
||||
BUILD_FEATURES: ${{ matrix.features }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
stem="rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}"
|
||||
package_file="dist/${stem}.zip"
|
||||
[[ -f "$package_file" && -f "dist/${stem}.deb" && -f "dist/${stem}.rpm" ]]
|
||||
[[ "$(find dist -maxdepth 1 -type f | wc -l)" -eq 3 ]]
|
||||
archive_members=$(unzip -Z1 "$package_file" | sort)
|
||||
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
|
||||
expected_members=$(printf '%s\n' rustfs rustfs.cpu-symbol-catalog.json rustfs.cpu-symbol-catalog.sha256 | sort)
|
||||
else
|
||||
expected_members=$(printf '%s\n' rustfs)
|
||||
fi
|
||||
if [[ "$archive_members" != "$expected_members" ]]; then
|
||||
printf 'Linux package does not contain the expected files\n' >&2
|
||||
printf '%s\n' "$archive_members" >&2
|
||||
exit 1
|
||||
fi
|
||||
binary_dir=$(mktemp -d)
|
||||
trap 'rm -rf "$binary_dir"' EXIT
|
||||
unzip -q "$package_file" rustfs -d "$binary_dir"
|
||||
version_output=$("$binary_dir/rustfs" --version)
|
||||
first=${version_output%%$'\n'*}
|
||||
if [[ "$first" != "rustfs ${PACKAGE_VERSION}" ]]; then
|
||||
printf 'packaged binary version line is %q, expected %q\n' "$first" "rustfs ${PACKAGE_VERSION}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -Fxq "build target : ${BUILD_TARGET}" <<< "$version_output" || \
|
||||
! grep -Fxq "build os : ${BUILD_OS}" <<< "$version_output"; then
|
||||
printf 'packaged binary reports an unexpected compile target\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Verify packaged console
|
||||
if: matrix.target == 'x86_64-unknown-linux-gnu' || matrix.target == 'aarch64-unknown-linux-gnu'
|
||||
shell: bash
|
||||
env:
|
||||
PACKAGE_VERSION: ${{ needs.prepare.outputs.version }}
|
||||
ARCH: ${{ matrix.arch }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
package_file="dist/rustfs-linux-${ARCH}-${LIBC}-v${PACKAGE_VERSION}.zip"
|
||||
|
||||
find_free_port() {
|
||||
python3 - <<'PY'
|
||||
import socket
|
||||
|
||||
with socket.socket() as sock:
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
print(sock.getsockname()[1])
|
||||
PY
|
||||
}
|
||||
|
||||
package_dir=$(mktemp -d)
|
||||
data_dir=$(mktemp -d)
|
||||
log_file="${RUNNER_TEMP}/rustfs-console-smoke.log"
|
||||
server_pid=""
|
||||
trap '
|
||||
if [[ -n "${server_pid:-}" ]]; then
|
||||
kill "$server_pid" 2>/dev/null || true
|
||||
wait "$server_pid" 2>/dev/null || true
|
||||
fi
|
||||
rm -rf "$package_dir" "$data_dir"
|
||||
' EXIT
|
||||
|
||||
unzip -q "$package_file" -d "$package_dir"
|
||||
api_port=$(find_free_port)
|
||||
console_port=$(find_free_port)
|
||||
console_url="http://127.0.0.1:${console_port}/rustfs/console/"
|
||||
|
||||
"$package_dir/rustfs" server \
|
||||
--address "127.0.0.1:${api_port}" \
|
||||
--console-enable \
|
||||
--console-address "127.0.0.1:${console_port}" \
|
||||
--access-key console-smoke \
|
||||
--secret-key console-smoke-secret \
|
||||
"$data_dir" >"$log_file" 2>&1 &
|
||||
server_pid=$!
|
||||
|
||||
for _ in {1..40}; do
|
||||
response=$(curl --silent --output /dev/null --write-out '%{http_code} %{content_type}' "$console_url" || true)
|
||||
if [[ "$response" == 200\ text/html* ]]; then
|
||||
exit 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
|
||||
printf 'Console endpoint did not return 200 text/html: %s\n' "$response" >&2
|
||||
cat "$log_file"
|
||||
exit 1
|
||||
|
||||
- name: Upload packages
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: packages-${{ matrix.target_id }}-v${{ needs.prepare.outputs.version }}
|
||||
path: dist/*
|
||||
if-no-files-found: error
|
||||
compression-level: 0
|
||||
retention-days: 30
|
||||
@@ -112,11 +112,9 @@ jobs:
|
||||
# GitHub-hosted: reliably provides a running Docker daemon + buildx +
|
||||
# python3. See the header note (ci-2) for why the self-hosted sm-standard-4
|
||||
# label was abandoned (no docker.sock on the pod its only run landed on).
|
||||
# sm-standard-2 has no docker.sock either (weekly runs failed at "Enable
|
||||
# buildx" from 2026-09-27 to 2026-11-08), so keep this job on ubuntu-latest.
|
||||
# TODO(ci-8): scheduled-failure alerting (auto-open issue on a red weekly
|
||||
# run) is added by the ci-8 composite action; do not implement it here.
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: sm-standard-2
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
|
||||
@@ -47,8 +47,8 @@ permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: oidc-keycloak-live-${{ github.event_name }}-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name != 'schedule' }}
|
||||
group: oidc-keycloak-live-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
oidc-keycloak-live:
|
||||
|
||||
+825
-38
@@ -1,53 +1,840 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
name: Build candidate packages
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
candidate_manifest:
|
||||
type: string
|
||||
required: true
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Package Workflow - Build DEB/RPM packages
|
||||
#
|
||||
# This workflow builds DEB and RPM packages from pre-built Linux binaries
|
||||
# and uploads them to Cloudflare R2 and the GitHub release.
|
||||
#
|
||||
# Trigger:
|
||||
# - workflow_run: automatically package after "Build and Release" completes
|
||||
# for a release tag (the mac/windows/linux binaries are already uploaded
|
||||
# to the GitHub release before packaging starts)
|
||||
# - workflow_dispatch: manual fallback with a release tag and/or exact build run ID
|
||||
#
|
||||
# Flow:
|
||||
# 1. Resolve and validate the selected Build workflow run and source identity
|
||||
# 2. Download Linux binaries (x86_64-gnu, aarch64-gnu, x86_64-musl,
|
||||
# aarch64-musl) from build artifacts
|
||||
# 3. Build DEB packages for amd64 and arm64, gnu and musl variants
|
||||
# 4. Build RPM packages for x86_64 and aarch64, gnu and musl variants
|
||||
# 5. Upload all packages to Cloudflare R2 and the GitHub release
|
||||
|
||||
name: Package DEB/RPM
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# contents: write is required to upload packages to the GitHub release
|
||||
contents: write
|
||||
actions: read
|
||||
|
||||
on:
|
||||
# Follows the same pattern as docker.yml: run after the release build
|
||||
# workflow completes, so packaging is triggered only by release tags
|
||||
# (e.g. 1.0.0-rc.2, 1.0.0-rc.3), never by development builds.
|
||||
workflow_run:
|
||||
workflows: [ "Build and Release" ]
|
||||
types: [ completed ]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Release tag to package (e.g. 1.0.0-beta.12). Leave empty for latest main build."
|
||||
required: false
|
||||
type: string
|
||||
build_run_id:
|
||||
description: "Build workflow run ID (when combined with tag, both must identify the same release commit)"
|
||||
required: false
|
||||
type: string
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.workflow_run.head_branch || github.event.inputs.tag || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
||||
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
|
||||
jobs:
|
||||
# Resolve which build run to use and extract version info
|
||||
resolve:
|
||||
name: Resolve Build
|
||||
# Auto-trigger only from successful tag builds of "Build and Release".
|
||||
# Tag pushes arrive as event == push with head_branch != main (a
|
||||
# non-main push head_branch is the release tag name). Manual dispatch
|
||||
# stays available as a fallback for backfills and re-runs.
|
||||
if: >-
|
||||
github.event_name == 'workflow_dispatch' ||
|
||||
(github.event.workflow_run.conclusion == 'success' &&
|
||||
github.event.workflow_run.event == 'push' &&
|
||||
github.event.workflow_run.head_branch != 'main')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 10
|
||||
outputs:
|
||||
version: ${{ steps.resolve.outputs.version }}
|
||||
build_type: ${{ steps.resolve.outputs.build_type }}
|
||||
build_run_id: ${{ steps.resolve.outputs.build_run_id }}
|
||||
build_run_number: ${{ steps.resolve.outputs.build_run_number }}
|
||||
head_sha: ${{ steps.resolve.outputs.head_sha }}
|
||||
dev_sequence: ${{ steps.resolve.outputs.dev_sequence }}
|
||||
tag: ${{ steps.resolve.outputs.tag }}
|
||||
steps:
|
||||
- name: Resolve build run
|
||||
id: resolve
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
REPOSITORY: ${{ github.repository }}
|
||||
INPUT_TAG: ${{ github.event.inputs.tag }}
|
||||
INPUT_RUN_ID: ${{ github.event.inputs.build_run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
fail() {
|
||||
echo "❌ $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
TAG=""
|
||||
BUILD_RUN_ID=""
|
||||
case "$EVENT_NAME" in
|
||||
workflow_run)
|
||||
TAG="$HEAD_BRANCH"
|
||||
BUILD_RUN_ID="$WORKFLOW_RUN_ID"
|
||||
;;
|
||||
workflow_dispatch)
|
||||
TAG="$INPUT_TAG"
|
||||
BUILD_RUN_ID="$INPUT_RUN_ID"
|
||||
;;
|
||||
*) fail "unsupported event: $EVENT_NAME" ;;
|
||||
esac
|
||||
|
||||
# Validate and classify tags before using them in API paths or logs.
|
||||
semver_core='(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)'
|
||||
prerelease_id='(alpha|beta|rc)\.(0|[1-9][0-9]*)'
|
||||
if [[ -n "$TAG" ]]; then
|
||||
if [[ "$TAG" =~ ^${semver_core}(-${prerelease_id})?-preview\.(0|[1-9][0-9]*)$ ]]; then
|
||||
BUILD_TYPE=preview
|
||||
elif [[ "$TAG" =~ ^${semver_core}-${prerelease_id}$ ]]; then
|
||||
BUILD_TYPE=prerelease
|
||||
elif [[ "$TAG" =~ ^${semver_core}$ ]]; then
|
||||
BUILD_TYPE=release
|
||||
else
|
||||
fail "tag is not a supported strict package version"
|
||||
fi
|
||||
else
|
||||
BUILD_TYPE=development
|
||||
fi
|
||||
|
||||
if [[ -n "$BUILD_RUN_ID" ]]; then
|
||||
[[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "build run ID must be a positive decimal integer"
|
||||
echo "Using selected build run: $BUILD_RUN_ID"
|
||||
elif [[ -n "$TAG" ]]; then
|
||||
echo "Looking for build run for tag: $TAG"
|
||||
BUILD_RUN_ID=$(gh api --method GET \
|
||||
"repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch="$TAG" -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' 2>/dev/null || true)
|
||||
|
||||
if [[ -z "$BUILD_RUN_ID" ]]; then
|
||||
BUILD_RUN_ID=$(gh api --method GET \
|
||||
"repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f event=push -f status=success -F per_page=100 2>/dev/null |
|
||||
jq -r --arg tag "$TAG" \
|
||||
'[.workflow_runs[] | select(.head_branch == $tag)][0].id // empty' || true)
|
||||
fi
|
||||
[[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "no successful build run found for tag"
|
||||
echo "Found build run: $BUILD_RUN_ID"
|
||||
else
|
||||
echo "No tag specified, looking for latest main build"
|
||||
BUILD_RUN_ID=$(gh api --method GET \
|
||||
"repos/${REPOSITORY}/actions/workflows/build.yml/runs" \
|
||||
-f branch=main -f status=success -F per_page=1 \
|
||||
--jq '.workflow_runs[0].id // empty' 2>/dev/null || true)
|
||||
[[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ ]] || fail "no successful main build found"
|
||||
echo "Latest main build: $BUILD_RUN_ID"
|
||||
fi
|
||||
|
||||
# Fetch once and use the same immutable run metadata for identity,
|
||||
# ordering, workflow provenance, and release-channel validation.
|
||||
RUN_JSON=$(gh api "repos/${REPOSITORY}/actions/runs/${BUILD_RUN_ID}") ||
|
||||
fail "cannot read selected build run"
|
||||
RUN_ID=$(jq -r '.id // empty' <<<"$RUN_JSON")
|
||||
RUN_NUMBER=$(jq -r '.run_number // empty' <<<"$RUN_JSON")
|
||||
RUN_STATUS=$(jq -r '.status // empty' <<<"$RUN_JSON")
|
||||
RUN_CONCLUSION=$(jq -r '.conclusion // empty' <<<"$RUN_JSON")
|
||||
RUN_PATH=$(jq -r '.path // empty' <<<"$RUN_JSON")
|
||||
HEAD_SHA=$(jq -r '.head_sha // empty' <<<"$RUN_JSON")
|
||||
RUN_HEAD_BRANCH=$(jq -r '.head_branch // empty' <<<"$RUN_JSON")
|
||||
|
||||
[[ "$RUN_ID" == "$BUILD_RUN_ID" ]] || fail "run metadata ID mismatch"
|
||||
[[ "$RUN_NUMBER" =~ ^[1-9][0-9]*$ ]] || fail "build run number must be a positive decimal integer"
|
||||
[[ "$RUN_STATUS" == completed && "$RUN_CONCLUSION" == success ]] || fail "selected build run is not successful"
|
||||
[[ "$RUN_PATH" == .github/workflows/build.yml ]] || fail "selected run is not Build and Release"
|
||||
[[ "$HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || fail "selected build run has an invalid head SHA"
|
||||
[[ "$RUN_HEAD_BRANCH" != *$'\n'* && -n "$RUN_HEAD_BRANCH" ]] || fail "selected build run has an invalid head branch"
|
||||
|
||||
if [[ -n "$TAG" ]]; then
|
||||
[[ "$RUN_HEAD_BRANCH" == "$TAG" ]] || fail "tag and build run head branch do not match"
|
||||
|
||||
TAG_REF_JSON=$(gh api "repos/${REPOSITORY}/git/ref/tags/${TAG}") ||
|
||||
fail "cannot resolve release tag ref"
|
||||
TAG_OBJECT_TYPE=$(jq -r '.object.type // empty' <<<"$TAG_REF_JSON")
|
||||
TAG_OBJECT_SHA=$(jq -r '.object.sha // empty' <<<"$TAG_REF_JSON")
|
||||
depth=0
|
||||
while [[ "$TAG_OBJECT_TYPE" == tag && $depth -lt 5 ]]; do
|
||||
TAG_OBJECT_JSON=$(gh api "repos/${REPOSITORY}/git/tags/${TAG_OBJECT_SHA}") ||
|
||||
fail "cannot peel annotated release tag"
|
||||
TAG_OBJECT_TYPE=$(jq -r '.object.type // empty' <<<"$TAG_OBJECT_JSON")
|
||||
TAG_OBJECT_SHA=$(jq -r '.object.sha // empty' <<<"$TAG_OBJECT_JSON")
|
||||
depth=$((depth + 1))
|
||||
done
|
||||
[[ "$TAG_OBJECT_TYPE" == commit && "$TAG_OBJECT_SHA" =~ ^[0-9a-f]{40}$ ]] ||
|
||||
fail "release tag does not resolve to a commit"
|
||||
[[ "$TAG_OBJECT_SHA" == "$HEAD_SHA" ]] || fail "release tag commit and build run head SHA do not match"
|
||||
VERSION="$TAG"
|
||||
DEV_SEQUENCE=""
|
||||
else
|
||||
VERSION="dev-${HEAD_SHA}"
|
||||
DEV_SEQUENCE="$RUN_NUMBER"
|
||||
fi
|
||||
|
||||
{
|
||||
echo "version=$VERSION"
|
||||
echo "build_type=$BUILD_TYPE"
|
||||
echo "build_run_id=$BUILD_RUN_ID"
|
||||
echo "build_run_number=$RUN_NUMBER"
|
||||
echo "head_sha=$HEAD_SHA"
|
||||
echo "dev_sequence=$DEV_SEQUENCE"
|
||||
echo "tag=${TAG}"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "📊 Resolved:"
|
||||
echo " Version: $VERSION"
|
||||
echo " Build type: $BUILD_TYPE"
|
||||
echo " Build run ID: $BUILD_RUN_ID"
|
||||
echo " Build run number: $RUN_NUMBER"
|
||||
|
||||
# Consume the trusted release certification of the resolved build run and
|
||||
# verify the certified input digests before any package is wrapped. A run
|
||||
# without that certification fails closed here; packages built later must
|
||||
# complete their own verification and cannot inherit an unverified
|
||||
# conclusion.
|
||||
release-certification:
|
||||
name: Release Certification
|
||||
needs: resolve
|
||||
if: needs.resolve.result == 'success'
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download and verify release certification
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.RELEASE_VALIDATION_TOKEN || github.token }}
|
||||
BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
|
||||
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
mkdir -p certification
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
# Development artifacts are explicitly uncertified and never touch
|
||||
# official or latest channels; only the staging/dev paths remain.
|
||||
jq -n '{schema_version: 1, certified: false, reason: "development artifacts are explicitly uncertified"}' \
|
||||
> certification/development-uncertified.json
|
||||
exit 0
|
||||
fi
|
||||
if ! gh run download "$BUILD_RUN_ID" -n release-certification -D certification; then
|
||||
echo "Build run ${BUILD_RUN_ID} published no release-certification artifact; failing closed" >&2
|
||||
exit 1
|
||||
fi
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact-sha256 "$(jq -r '.deliverables[0].sha256' certification/candidate.json)" \
|
||||
--artifact-name "$(jq -r '.deliverables[0].name' certification/candidate.json)"
|
||||
|
||||
# Build DEB and RPM packages for each architecture and libc variant
|
||||
package:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
name: Package (${{ matrix.arch }}-${{ matrix.libc }})
|
||||
needs: [ resolve, release-certification ]
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
fail-fast: false
|
||||
max-parallel: 1
|
||||
matrix:
|
||||
target: [linux-x86_64-musl, linux-aarch64-musl, linux-x86_64-gnu, linux-aarch64-gnu]
|
||||
include:
|
||||
- arch: x86_64
|
||||
libc: gnu
|
||||
deb_arch: amd64
|
||||
rpm_arch: x86_64
|
||||
artifact_name: "rustfs-linux-x86_64-gnu"
|
||||
# gnu binaries dynamically link glibc; musl binaries are static
|
||||
# and run without a libc dependency.
|
||||
deb_depends: "libc6 (>= 2.31)"
|
||||
rpm_depends: "glibc >= 2.31"
|
||||
- arch: aarch64
|
||||
libc: gnu
|
||||
deb_arch: arm64
|
||||
rpm_arch: aarch64
|
||||
artifact_name: "rustfs-linux-aarch64-gnu"
|
||||
deb_depends: "libc6 (>= 2.31)"
|
||||
rpm_depends: "glibc >= 2.31"
|
||||
- arch: x86_64
|
||||
libc: musl
|
||||
deb_arch: amd64
|
||||
rpm_arch: x86_64
|
||||
artifact_name: "rustfs-linux-x86_64-musl"
|
||||
deb_depends: ""
|
||||
rpm_depends: ""
|
||||
- arch: aarch64
|
||||
libc: musl
|
||||
deb_arch: arm64
|
||||
rpm_arch: aarch64
|
||||
artifact_name: "rustfs-linux-aarch64-musl"
|
||||
deb_depends: ""
|
||||
rpm_depends: ""
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ fromJson(inputs.candidate_manifest).source_sha }}
|
||||
persist-credentials: false
|
||||
- name: Reuse completed package set
|
||||
id: reuse
|
||||
|
||||
- name: Normalize package metadata
|
||||
id: versions
|
||||
shell: bash
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CANDIDATE_MANIFEST: ${{ inputs.candidate_manifest }}
|
||||
TARGET: ${{ matrix.target }}
|
||||
run: python3 scripts/release/deliver.py reuse --kind packages --target "$TARGET"
|
||||
- name: Install packaging tools
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
SOURCE_VERSION: ${{ needs.resolve.outputs.version }}
|
||||
DEV_SEQUENCE: ${{ needs.resolve.outputs.dev_sequence }}
|
||||
ARCH: ${{ matrix.rpm_arch }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y ruby ruby-dev build-essential rpm fakeroot cpio
|
||||
sudo gem install fpm --version 1.17.0 --no-document
|
||||
- name: Package accepted executable and verify contents
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
set -euo pipefail
|
||||
|
||||
normalized=$(./scripts/release/package_versions.sh \
|
||||
"$BUILD_TYPE" "$SOURCE_VERSION" "$DEV_SEQUENCE" "$ARCH" "$LIBC")
|
||||
printf '%s\n' "$normalized" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download binary artifact from build run
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: ${{ matrix.artifact_name }}*
|
||||
path: ./binary-artifact
|
||||
run-id: ${{ needs.resolve.outputs.build_run_id }}
|
||||
github-token: ${{ github.token }}
|
||||
merge-multiple: true
|
||||
|
||||
- name: Extract binary
|
||||
id: binary
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
ZIP_FILE=$(find ./binary-artifact -name "*.zip" -type f | head -1)
|
||||
if [[ -z "$ZIP_FILE" ]]; then
|
||||
echo "❌ No binary artifact found"
|
||||
find ./binary-artifact -mindepth 1 -maxdepth 1 -print 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Found artifact: $ZIP_FILE"
|
||||
|
||||
mkdir -p ./bin
|
||||
unzip -o "$ZIP_FILE" -d ./bin
|
||||
|
||||
if [[ ! -f ./bin/rustfs ]]; then
|
||||
echo "❌ rustfs binary not found in archive"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
chmod +x ./bin/rustfs
|
||||
stat --printf='%n %s bytes\n' ./bin/rustfs
|
||||
echo "✅ Binary extracted"
|
||||
|
||||
- name: Build DEB package
|
||||
id: deb
|
||||
shell: bash
|
||||
env:
|
||||
DEB_VERSION: ${{ steps.versions.outputs.deb_version }}
|
||||
DEB_ARCH: ${{ matrix.deb_arch }}
|
||||
DEB_FILE: ${{ steps.versions.outputs.deb_file }}
|
||||
DEB_DEPENDS: ${{ matrix.deb_depends }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
PKG_DIR="${DEB_FILE%.deb}"
|
||||
|
||||
echo "Building DEB: ${DEB_FILE}"
|
||||
|
||||
mkdir -p "${PKG_DIR}/DEBIAN"
|
||||
mkdir -p "${PKG_DIR}/usr/bin"
|
||||
mkdir -p "${PKG_DIR}/etc/default"
|
||||
mkdir -p "${PKG_DIR}/lib/systemd/system"
|
||||
mkdir -p "${PKG_DIR}/usr/share/doc/rustfs"
|
||||
|
||||
cp ./bin/rustfs "${PKG_DIR}/usr/bin/"
|
||||
chmod 755 "${PKG_DIR}/usr/bin/rustfs"
|
||||
|
||||
cp deploy/build/rustfs.service "${PKG_DIR}/lib/systemd/system/"
|
||||
|
||||
cat > "${PKG_DIR}/etc/default/rustfs" << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://docs.rustfs.com for more information
|
||||
# RUSTFS_ACCESS_KEY=<changeme>
|
||||
# RUSTFS_SECRET_KEY=<changeme>
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ADDRESS=":9000"
|
||||
# RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
# RUSTFS_CONSOLE_ENABLE=true
|
||||
# RUSTFS_OBS_LOGGER_LEVEL=error
|
||||
# RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
|
||||
ENVEOF
|
||||
|
||||
cat > "${PKG_DIR}/DEBIAN/control" << EOF
|
||||
Package: rustfs
|
||||
Version: ${DEB_VERSION}
|
||||
Section: utils
|
||||
Priority: optional
|
||||
Architecture: ${DEB_ARCH}
|
||||
EOF
|
||||
|
||||
# gnu binaries dynamically link glibc and must declare it; musl
|
||||
# binaries are statically linked and need no libc dependency.
|
||||
if [[ -n "$DEB_DEPENDS" ]]; then
|
||||
echo "Depends: ${DEB_DEPENDS}" >> "${PKG_DIR}/DEBIAN/control"
|
||||
fi
|
||||
|
||||
cat >> "${PKG_DIR}/DEBIAN/control" << EOF
|
||||
Maintainer: RustFS Team <support@rustfs.com>
|
||||
Description: High-performance distributed object storage (${LIBC} build)
|
||||
RustFS is a high-performance distributed object storage software
|
||||
built using Rust. It is compatible with MinIO and S3 API.
|
||||
Homepage: https://rustfs.com
|
||||
EOF
|
||||
|
||||
# Declare /etc/default/rustfs as a conffile so dpkg preserves user
|
||||
# modifications on upgrade instead of silently overwriting them.
|
||||
cat > "${PKG_DIR}/DEBIAN/conffiles" << 'CONFFILES'
|
||||
/etc/default/rustfs
|
||||
CONFFILES
|
||||
|
||||
./scripts/release/package_service_scripts.sh after-install > "${PKG_DIR}/DEBIAN/postinst"
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postinst"
|
||||
|
||||
./scripts/release/package_service_scripts.sh before-remove > "${PKG_DIR}/DEBIAN/prerm"
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/prerm"
|
||||
|
||||
./scripts/release/package_service_scripts.sh after-remove > "${PKG_DIR}/DEBIAN/postrm"
|
||||
chmod 755 "${PKG_DIR}/DEBIAN/postrm"
|
||||
|
||||
cp LICENSE "${PKG_DIR}/usr/share/doc/rustfs/"
|
||||
cp README.md "${PKG_DIR}/usr/share/doc/rustfs/"
|
||||
|
||||
fakeroot dpkg-deb --build "${PKG_DIR}" "$DEB_FILE"
|
||||
|
||||
[[ $(dpkg-deb -f "$DEB_FILE" Package) == rustfs ]]
|
||||
[[ $(dpkg-deb -f "$DEB_FILE" Version) == "$DEB_VERSION" ]]
|
||||
[[ $(dpkg-deb -f "$DEB_FILE" Architecture) == "$DEB_ARCH" ]]
|
||||
dpkg-deb --fsys-tarfile "$DEB_FILE" | tar -tf - | grep -Fx './usr/bin/rustfs' >/dev/null
|
||||
stat --printf='%n %s bytes\n' "$DEB_FILE"
|
||||
echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ DEB built: $DEB_FILE"
|
||||
|
||||
- name: Build RPM package
|
||||
id: rpm
|
||||
shell: bash
|
||||
env:
|
||||
RPM_VERSION: ${{ steps.versions.outputs.rpm_version }}
|
||||
RPM_RELEASE: ${{ steps.versions.outputs.rpm_release }}
|
||||
RPM_ARCH: ${{ matrix.rpm_arch }}
|
||||
RPM_FILE: ${{ steps.versions.outputs.rpm_file }}
|
||||
RPM_DEPENDS: ${{ matrix.rpm_depends }}
|
||||
LIBC: ${{ matrix.libc }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
echo "Building RPM for ${RPM_ARCH} (${LIBC})"
|
||||
|
||||
sudo apt-get update && sudo apt-get install -y ruby ruby-dev build-essential rpm
|
||||
sudo gem install fpm
|
||||
./scripts/test_package_versions.sh --require-package-managers
|
||||
|
||||
# gnu binaries dynamically link glibc and must declare it; musl
|
||||
# binaries are statically linked and need no libc dependency.
|
||||
depends_args=()
|
||||
if [[ -n "$RPM_DEPENDS" ]]; then
|
||||
depends_args+=(--depends "$RPM_DEPENDS")
|
||||
fi
|
||||
|
||||
# Create config file for fpm (DEB build creates it in its package dir structure,
|
||||
# but fpm needs the file to exist before packaging)
|
||||
mkdir -p ./tmp-pkg/etc/default
|
||||
cat > ./tmp-pkg/etc/default/rustfs << 'ENVEOF'
|
||||
# RustFS Environment Configuration
|
||||
# See https://docs.rustfs.com for more information
|
||||
# RUSTFS_ACCESS_KEY=<changeme>
|
||||
# RUSTFS_SECRET_KEY=<changeme>
|
||||
# RUSTFS_VOLUMES=""
|
||||
# RUSTFS_ADDRESS=":9000"
|
||||
# RUSTFS_CONSOLE_ADDRESS=":9001"
|
||||
# RUSTFS_CONSOLE_ENABLE=true
|
||||
# RUSTFS_OBS_LOGGER_LEVEL=error
|
||||
# RUSTFS_OBS_LOG_DIRECTORY="/var/log/rustfs/"
|
||||
ENVEOF
|
||||
|
||||
fpm -s dir -t rpm \
|
||||
--name rustfs \
|
||||
--version "$RPM_VERSION" \
|
||||
--iteration "$RPM_RELEASE" \
|
||||
--architecture "$RPM_ARCH" \
|
||||
--package "$RPM_FILE" \
|
||||
"${depends_args[@]}" \
|
||||
--description "High-performance distributed object storage (${LIBC} build)" \
|
||||
--maintainer "RustFS Team <support@rustfs.com>" \
|
||||
--url "https://rustfs.com" \
|
||||
--license "Apache-2.0" \
|
||||
--before-install <(./scripts/release/package_service_scripts.sh rpm-before-install) \
|
||||
--before-upgrade <(./scripts/release/package_service_scripts.sh rpm-before-upgrade) \
|
||||
--after-install <(./scripts/release/package_service_scripts.sh after-install) \
|
||||
--after-upgrade <(./scripts/release/package_service_scripts.sh after-install) \
|
||||
--before-remove <(./scripts/release/package_service_scripts.sh before-remove) \
|
||||
--after-remove <(./scripts/release/package_service_scripts.sh after-remove) \
|
||||
--rpm-posttrans <(./scripts/release/package_service_scripts.sh rpm-posttrans) \
|
||||
--config-files /etc/default/rustfs \
|
||||
./bin/rustfs=/usr/bin/rustfs \
|
||||
./tmp-pkg/etc/default/rustfs=/etc/default/rustfs \
|
||||
deploy/build/rustfs.service=/lib/systemd/system/rustfs.service \
|
||||
LICENSE=/usr/share/doc/rustfs/LICENSE \
|
||||
README.md=/usr/share/doc/rustfs/README.md
|
||||
|
||||
if [[ ! -f "$RPM_FILE" ]]; then
|
||||
echo "❌ RPM build failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RPM_METADATA=$(rpm -qp --qf '%{NAME}\n%{VERSION}\n%{RELEASE}\n%{ARCH}\n' "$RPM_FILE")
|
||||
EXPECTED_METADATA=$(printf 'rustfs\n%s\n%s\n%s' "$RPM_VERSION" "$RPM_RELEASE" "$RPM_ARCH")
|
||||
[[ "$RPM_METADATA" == "$EXPECTED_METADATA" ]]
|
||||
rpm -qpl "$RPM_FILE" | grep -Fx '/usr/bin/rustfs' >/dev/null
|
||||
stat --printf='%n %s bytes\n' "$RPM_FILE"
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
- name: Download certification evidence
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
# Later-built wrapper verification: the binary inside each package must
|
||||
# match the certified binary digest for this platform before anything is
|
||||
# uploaded, and the package itself stays on staging until promotion.
|
||||
- name: Verify wrapped binary against certification
|
||||
if: needs.resolve.outputs.build_type != 'development'
|
||||
shell: bash
|
||||
env:
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
ARTIFACT_NAME: ${{ matrix.artifact_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
expected="$(jq -r --arg needle "$ARTIFACT_NAME" \
|
||||
'[.deliverables[] | select(.name | contains($needle)) | .binary_sha256][0] // empty' \
|
||||
certification/candidate.json)"
|
||||
if [[ -z "$expected" ]]; then
|
||||
echo "No certified binary digest recorded for ${ARTIFACT_NAME}" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y cpio
|
||||
fi
|
||||
for package_file in "$DEB_FILE" "$RPM_FILE"; do
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
done
|
||||
|
||||
- name: Upload packages to artifacts
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: packages-${{ matrix.arch }}-${{ matrix.libc }}
|
||||
path: |
|
||||
*.deb
|
||||
*.rpm
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload packages to Cloudflare R2
|
||||
if: needs.resolve.outputs.build_type != 'preview' && env.R2_ACCESS_KEY_ID != ''
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "⚠️ R2 credentials missing, skipping upload"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
# Build-phase writes stay on staging for release types: the official
|
||||
# packages/release and packages/latest paths are written only by the
|
||||
# certification-gated promote-packages job, from these same staged
|
||||
# objects. Development packages keep the uncertified dev channel.
|
||||
if [[ "$BUILD_TYPE" == "development" ]]; then
|
||||
R2_PREFIX="artifacts/rustfs/packages/dev"
|
||||
else
|
||||
R2_PREFIX="artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}"
|
||||
fi
|
||||
R2_PATH="s3://${R2_BUCKET}/${R2_PREFIX}/"
|
||||
|
||||
echo "📤 Uploading to $R2_PATH"
|
||||
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "Uploading: $f"
|
||||
aws s3 cp "$f" "$R2_PATH" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
fi
|
||||
done
|
||||
|
||||
echo "✅ Upload complete"
|
||||
|
||||
- name: Upload packages to GitHub Release
|
||||
if: needs.resolve.outputs.tag != ''
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CANDIDATE_MANIFEST: ${{ inputs.candidate_manifest }}
|
||||
TARGET: ${{ matrix.target }}
|
||||
run: python3 scripts/release/deliver.py package --target "$TARGET" --output delivery
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: steps.reuse.outputs.reused != 'true'
|
||||
TAG: ${{ needs.resolve.outputs.tag }}
|
||||
DEB_FILE: ${{ steps.deb.outputs.deb_file }}
|
||||
RPM_FILE: ${{ steps.rpm.outputs.rpm_file }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
# Upload the packages, then refresh the release checksums so the new
|
||||
# assets are covered, matching the binary release flow.
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
echo "📤 Uploading $(basename "$f") to GitHub release ${TAG}..."
|
||||
gh release upload "$TAG" "$f" --clobber
|
||||
fi
|
||||
done
|
||||
|
||||
CHECKSUM_DIR="$(mktemp -d)"
|
||||
gh release download "$TAG" -p 'SHA256SUMS' -p 'SHA512SUMS' \
|
||||
-D "$CHECKSUM_DIR" --clobber
|
||||
|
||||
for spec in "SHA256SUMS:sha256sum" "SHA512SUMS:sha512sum"; do
|
||||
asset="${spec%%:*}"
|
||||
checksum_cmd="${spec##*:}"
|
||||
checksum_file="${CHECKSUM_DIR}/${asset}"
|
||||
|
||||
for f in "$DEB_FILE" "$RPM_FILE"; do
|
||||
if [[ -n "$f" && -f "$f" ]]; then
|
||||
base="$(basename "$f")"
|
||||
# GitHub stores release asset names with '~' normalized to '.'.
|
||||
# Current package file names embed the raw tag and contain no
|
||||
# '~', but older releases may still ship such assets, so the
|
||||
# checksum entries must reference the name as stored on the
|
||||
# release.
|
||||
github_base="${base//\~/.}"
|
||||
# Remove any stale entry (both naming variants), then append
|
||||
grep -Fv -- "$base" "$checksum_file" > "${checksum_file}.tmp" || true
|
||||
grep -Fv -- "$github_base" "${checksum_file}.tmp" > "${checksum_file}.tmp2" || true
|
||||
mv "${checksum_file}.tmp2" "$checksum_file"
|
||||
digest=$("$checksum_cmd" -- "$f" | awk '{print $1}')
|
||||
printf '%s %s\n' "$digest" "$github_base" >> "$checksum_file"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "📤 Updating ${asset} for release ${TAG}..."
|
||||
gh release upload "$TAG" "$checksum_file" --clobber
|
||||
done
|
||||
|
||||
echo "✅ GitHub release assets updated"
|
||||
|
||||
# The only writer of the official packages/release and packages/latest R2
|
||||
# paths. Promotion re-checks each package's wrapped binary against the
|
||||
# passed certification list and copies the staged objects byte-for-byte:
|
||||
# no rebuild-after-verify, no mutable latest resolution downstream.
|
||||
promote-packages:
|
||||
name: Promote Packages
|
||||
needs: [ resolve, package, release-certification ]
|
||||
# Target-channel lock for the official package paths.
|
||||
concurrency:
|
||||
group: release-channel-r2-packages
|
||||
cancel-in-progress: false
|
||||
if: needs.resolve.outputs.tag != '' && (needs.resolve.outputs.build_type == 'release' || needs.resolve.outputs.build_type == 'prerelease')
|
||||
runs-on: ubuntu-latest # gh CLI required; the sm-standard-2 fleet does not ship it
|
||||
timeout-minutes: 30
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
AWS_EC2_METADATA_DISABLED: true
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
name: packages-${{ matrix.target }}
|
||||
path: delivery/*
|
||||
if-no-files-found: error
|
||||
compression-level: 0
|
||||
retention-days: 30
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download packages and certification
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
path: ./packages
|
||||
pattern: packages-*
|
||||
merge-multiple: true
|
||||
|
||||
- name: Download certification evidence
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: release-certification
|
||||
path: ./certification
|
||||
|
||||
- name: Verify and promote staged packages
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -z "$R2_ACCESS_KEY_ID" || -z "$R2_SECRET_ACCESS_KEY" || -z "$R2_ENDPOINT" || -z "$R2_BUCKET" ]]; then
|
||||
echo "❌ R2 credentials missing; refusing to promote to the official channel" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STAGING_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/staging/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/"
|
||||
RELEASE_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/release/"
|
||||
LATEST_PATH="s3://${R2_BUCKET}/artifacts/rustfs/packages/latest/"
|
||||
|
||||
if ! command -v aws >/dev/null 2>&1; then
|
||||
sudo apt-get update && sudo apt-get install -y awscli
|
||||
fi
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID"
|
||||
export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export AWS_DEFAULT_REGION="auto"
|
||||
|
||||
if ! command -v rpm2cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y rpm2cpio
|
||||
fi
|
||||
if ! command -v cpio >/dev/null 2>&1; then
|
||||
sudo apt-get install -y cpio
|
||||
fi
|
||||
|
||||
promoted=0
|
||||
for package_file in packages/*.deb packages/*.rpm; do
|
||||
[[ -f "$package_file" ]] || continue
|
||||
name="$(basename "$package_file")"
|
||||
case "$package_file" in
|
||||
*.deb)
|
||||
dpkg-deb --fsys-tarfile "$package_file" | tar -xOf - ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
*.rpm)
|
||||
rpm2cpio "$package_file" | cpio -i --to-stdout ./usr/bin/rustfs > "$RUNNER_TEMP/wrapped-rustfs"
|
||||
;;
|
||||
esac
|
||||
python3 scripts/release/release_gate.py verify-promotion \
|
||||
--plan .config/release-validation-plan.json \
|
||||
--candidate certification/candidate.json \
|
||||
--summary certification/release-gate-output \
|
||||
--artifact "$RUNNER_TEMP/wrapped-rustfs" \
|
||||
--artifact-role binary
|
||||
|
||||
staged="$(mktemp)"
|
||||
aws s3 cp "${STAGING_PREFIX}${name}" "$staged" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
staged_sha="$(sha256sum "$staged" | awk '{print $1}')"
|
||||
built_sha="$(sha256sum "$package_file" | awk '{print $1}')"
|
||||
if [[ "$staged_sha" != "$built_sha" ]]; then
|
||||
echo "❌ Staged package ${name} does not match the verified bytes" >&2
|
||||
exit 1
|
||||
fi
|
||||
aws s3 cp "$staged" "${RELEASE_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
aws s3 cp "$staged" "${LATEST_PATH}${name}" --endpoint-url "$R2_ENDPOINT" --only-show-errors
|
||||
rm -f "$staged"
|
||||
promoted=$((promoted + 1))
|
||||
echo "✅ Promoted ${name} (${built_sha})"
|
||||
done
|
||||
if [[ "$promoted" -eq 0 ]]; then
|
||||
echo "No staged packages found to promote" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Summary
|
||||
summary:
|
||||
name: Summary
|
||||
needs: [ resolve, package ]
|
||||
# Every completed Build and Release run triggers this workflow; when
|
||||
# resolve declined it (not a release tag), there is nothing to summarise.
|
||||
if: always() && needs.resolve.result != 'skipped'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- name: Print summary
|
||||
shell: bash
|
||||
env:
|
||||
SUMMARY_VERSION: ${{ needs.resolve.outputs.version }}
|
||||
SUMMARY_BUILD_TYPE: ${{ needs.resolve.outputs.build_type }}
|
||||
SUMMARY_BUILD_RUN_ID: ${{ needs.resolve.outputs.build_run_id }}
|
||||
SUMMARY_PACKAGE_STATUS: ${{ needs.package.result }}
|
||||
run: |
|
||||
{
|
||||
echo "## 📦 Package Summary"
|
||||
echo ""
|
||||
echo "| Item | Value |"
|
||||
echo "|------|-------|"
|
||||
echo "| Version | \`${SUMMARY_VERSION}\` |"
|
||||
echo "| Build Type | ${SUMMARY_BUILD_TYPE} |"
|
||||
echo "| Build Run | #${SUMMARY_BUILD_RUN_ID} |"
|
||||
echo "| Package Status | ${SUMMARY_PACKAGE_STATUS} |"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
@@ -1,444 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# s3-tests reference parity lane (rustfs/backlog#2773).
|
||||
#
|
||||
# Runs the ceph/s3-tests implemented list (scripts/s3-tests/implemented_tests.txt)
|
||||
# against every target in .config/reference-parity-targets.toml and fails on
|
||||
# any per-node-id difference between a candidate and the reference:
|
||||
#
|
||||
# - reference: the release pinned in .config/reference-binary.toml, downloaded
|
||||
# and sha256-verified by .github/actions/reference-binary;
|
||||
# - candidate targets: the debug binary built from this checkout exactly as
|
||||
# ci.yml's build-rustfs-debug-binary job builds it for S3 Implemented Tests.
|
||||
#
|
||||
# Every target runs the same harness (scripts/s3-tests/run.sh from this
|
||||
# checkout), test list and server configuration on its own hosted runner. A
|
||||
# target's failures get one rerun against a fresh server; a case that passes it
|
||||
# is flaky, not a difference. scripts/s3-tests/reference_parity.py judges the
|
||||
# result and honours .config/reference-parity-tolerated.txt, whose entries must
|
||||
# each match a real difference.
|
||||
#
|
||||
# The `differential` job adds the 73-operation dual-stack differential
|
||||
# (crates/e2e_test/src/dual_stack_differential, rustfs/backlog#2740) against the
|
||||
# same pinned reference: side A is the released binary (RUSTFS_E2E_REFERENCE_BINARY,
|
||||
# started without RUSTFS_S3_STACK), side B is the candidate built above on its
|
||||
# legacy stack. It reuses the candidate build artifact, never builds the candidate
|
||||
# twice, and prints reference_side, version_a and version_b from the report the test
|
||||
# wrote; it fails when side A was not the reference or both sides report the same
|
||||
# version. Every (op, field, a, b) difference goes to the job summary. Report-only
|
||||
# until the first result has been reviewed.
|
||||
#
|
||||
# Report-only: not part of the Test and Lint aggregate. Scheduled failures are
|
||||
# owned by the scheduled-validation watchdog (.github/scheduled-validations.json).
|
||||
|
||||
name: Reference Parity
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- ".github/workflows/reference-parity.yml"
|
||||
- ".github/actions/reference-binary/**"
|
||||
- ".config/reference-binary.toml"
|
||||
- ".config/reference-parity-targets.toml"
|
||||
- ".config/reference-parity-tolerated.txt"
|
||||
- "scripts/s3-tests/reference_parity.py"
|
||||
- "scripts/junit_diff.py"
|
||||
- "crates/e2e_test/src/dual_stack_differential/**"
|
||||
schedule:
|
||||
- cron: "37 1 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
# Matches ci.yml: these variables are hashed into the rust-cache key, so the
|
||||
# candidate build reads the same ci-dev cache as build-rustfs-debug-binary.
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
jobs:
|
||||
plan:
|
||||
name: Plan parity targets
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
outputs:
|
||||
targets: ${{ steps.targets.outputs.targets }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Validate target list
|
||||
id: targets
|
||||
run: |
|
||||
set -euo pipefail
|
||||
targets="$(python3 scripts/s3-tests/reference_parity.py targets --config .config/reference-parity-targets.toml)"
|
||||
echo "targets=${targets}" >> "$GITHUB_OUTPUT"
|
||||
echo "Parity targets: ${targets}"
|
||||
|
||||
build-candidate:
|
||||
name: Build candidate binary
|
||||
# Critical-path CPU and cache ownership: the same debug build as ci.yml's
|
||||
# build-rustfs-debug-binary, reading the ci-dev cache written on sm-standard-4.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 60
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Build debug binary
|
||||
run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
|
||||
|
||||
- name: Upload candidate binary
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: reference-parity-candidate-binary
|
||||
overwrite: true
|
||||
path: |
|
||||
target/debug/rustfs
|
||||
target/debug/rustfs.e2e.json
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
s3tests:
|
||||
name: s3-tests (${{ matrix.target.name }})
|
||||
needs: [ plan, build-candidate ]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 75
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
target: ${{ fromJSON(needs.plan.outputs.targets) }}
|
||||
env:
|
||||
TARGET_NAME: ${{ matrix.target.name }}
|
||||
TARGET_KIND: ${{ matrix.target.binary }}
|
||||
# Above this many first-run failures the target is systemically broken;
|
||||
# a rerun would only repeat it, so the first run is judged as is.
|
||||
RERUN_CAP: "50"
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Fetch pinned reference binary
|
||||
id: reference
|
||||
if: matrix.target.binary == 'reference'
|
||||
uses: ./.github/actions/reference-binary
|
||||
|
||||
- name: Download candidate binary
|
||||
if: matrix.target.binary == 'candidate'
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: reference-parity-candidate-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Select target binary
|
||||
env:
|
||||
REFERENCE_PATH: ${{ steps.reference.outputs.path }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
case "$TARGET_KIND" in
|
||||
reference) binary="$REFERENCE_PATH" ;;
|
||||
candidate) chmod +x target/debug/rustfs; binary="$PWD/target/debug/rustfs" ;;
|
||||
*) echo "unknown target binary kind: $TARGET_KIND" >&2; exit 1 ;;
|
||||
esac
|
||||
test -x "$binary"
|
||||
echo "TARGET_BINARY=$binary" >> "$GITHUB_ENV"
|
||||
mkdir -p "parity/$TARGET_NAME"
|
||||
{
|
||||
echo "target=$TARGET_NAME"
|
||||
echo "binary=$TARGET_KIND"
|
||||
echo "sha256=$(sha256sum "$binary" | cut -d' ' -f1)"
|
||||
echo "version=$("$binary" --version | tr '\n' ' ')"
|
||||
} | tee "parity/$TARGET_NAME/target.txt"
|
||||
|
||||
- name: Apply target environment
|
||||
env:
|
||||
TARGET_JSON: ${{ toJSON(matrix.target) }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 scripts/s3-tests/reference_parity.py export-env --target-json "$TARGET_JSON" \
|
||||
| tee -a "$GITHUB_ENV" "parity/$TARGET_NAME/target-env.txt"
|
||||
|
||||
- name: Run implemented s3-tests
|
||||
run: |
|
||||
set -uo pipefail
|
||||
report="parity/$TARGET_NAME"
|
||||
harness="artifacts/s3tests-single"
|
||||
|
||||
# Same invocation as ci.yml's S3 Implemented Tests, with a fresh data
|
||||
# root and port per run. $1 optionally replaces the test list.
|
||||
run_suite() {
|
||||
local list="$1" label="$2" root port
|
||||
root="${RUNNER_TEMP}/rustfs-s3tests-${label}"
|
||||
rm -rf "$harness" "$root"
|
||||
mkdir -p "$root"
|
||||
port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
|
||||
env DEPLOY_MODE=binary \
|
||||
RUSTFS_BINARY="$TARGET_BINARY" \
|
||||
TEST_MODE=single \
|
||||
MAXFAIL=0 \
|
||||
XDIST=4 \
|
||||
S3_PORT="$port" \
|
||||
DATA_ROOT="$root" \
|
||||
S3TESTS_CONF="$harness/s3tests.conf" \
|
||||
${list:+IMPLEMENTED_TESTS_FILE="$list"} \
|
||||
./scripts/s3-tests/run.sh
|
||||
}
|
||||
|
||||
# A run is usable only if it produced a JUnit report; test failures
|
||||
# are data for the comparison, a missing report is a lane failure.
|
||||
keep() {
|
||||
local prefix="$1"
|
||||
for file in pytest.log compat-report.md; do
|
||||
if [[ -f "$harness/$file" ]]; then cp "$harness/$file" "$report/${prefix}${file}"; fi
|
||||
done
|
||||
if [[ -f "$harness/rustfs-single/rustfs.log" ]]; then cp "$harness/rustfs-single/rustfs.log" "$report/${prefix}rustfs.log"; fi
|
||||
test -s "$harness/junit.xml" || { echo "::error::$TARGET_NAME produced no JUnit report (${prefix:-first} run)"; return 1; }
|
||||
cp "$harness/junit.xml" "$report/${prefix}junit.xml"
|
||||
}
|
||||
|
||||
run_suite "" first
|
||||
echo "first_run_exit=$?" >> "$report/target.txt"
|
||||
keep "" || exit 1
|
||||
cp "$harness/selected-nodeids.txt" "$report/selected-nodeids.txt" || exit 1
|
||||
|
||||
failures="$(python3 scripts/s3-tests/reference_parity.py failed --report "$report")" || exit 1
|
||||
count="$(printf '%s' "$failures" | grep -c . || true)"
|
||||
echo "first_run_failures=$count" >> "$report/target.txt"
|
||||
if (( count == 0 )); then
|
||||
exit 0
|
||||
fi
|
||||
if (( count > RERUN_CAP )); then
|
||||
echo "::warning::$TARGET_NAME has $count failures (cap $RERUN_CAP); judging the first run without a rerun"
|
||||
echo "rerun=skipped-over-cap" >> "$report/target.txt"
|
||||
exit 0
|
||||
fi
|
||||
printf '%s\n' "$failures" > "$RUNNER_TEMP/rerun-tests.txt"
|
||||
run_suite "$RUNNER_TEMP/rerun-tests.txt" rerun
|
||||
echo "rerun_exit=$?" >> "$report/target.txt"
|
||||
keep rerun- || exit 1
|
||||
|
||||
- name: Upload target report
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: reference-parity-report-${{ matrix.target.name }}
|
||||
overwrite: true
|
||||
path: parity/${{ matrix.target.name }}/
|
||||
if-no-files-found: error
|
||||
retention-days: 14
|
||||
|
||||
differential:
|
||||
name: Dual-stack differential (reference vs candidate)
|
||||
needs: [ build-candidate ]
|
||||
# Compiles the e2e_test harness against the ci-dev cache like the other
|
||||
# sm-standard-4 jobs; the candidate server itself is the downloaded artifact.
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 90
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
NO_PROXY: 127.0.0.1,localhost,::1
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Setup Rust environment
|
||||
uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: stable
|
||||
cache-shared-key: ci-dev
|
||||
cache-save-if: 'false'
|
||||
install-build-packaging-tools: 'false'
|
||||
|
||||
- name: Fetch pinned reference binary
|
||||
id: reference
|
||||
uses: ./.github/actions/reference-binary
|
||||
|
||||
- name: Download candidate binary
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: reference-parity-candidate-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Run dual-stack differential
|
||||
id: differential
|
||||
continue-on-error: true
|
||||
env:
|
||||
REFERENCE_PATH: ${{ steps.reference.outputs.path }}
|
||||
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-differential-logs
|
||||
run: |
|
||||
set -euo pipefail
|
||||
export no_proxy="$NO_PROXY"
|
||||
chmod +x target/debug/rustfs
|
||||
test -x "$REFERENCE_PATH"
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
# Side A: the released binary. Side B: the candidate on its legacy stack.
|
||||
export RUSTFS_E2E_REFERENCE_BINARY="$REFERENCE_PATH"
|
||||
test_name='dual_stack_differential::tests::both_stacks_answer_every_operation_identically'
|
||||
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
|
||||
cargo test --locked -p e2e_test --lib "$test_name" -- --exact --nocapture \
|
||||
2>&1 | tee "$RUNNER_TEMP/differential-test.log"
|
||||
|
||||
- name: Summarize report and verify the reference side
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
python3 - <<'PY'
|
||||
import glob, json, os, sys
|
||||
|
||||
summary = os.environ["GITHUB_STEP_SUMMARY"]
|
||||
reports = sorted(glob.glob("target/dual-stack/self-comparison-*.json"))
|
||||
lines = ["## Dual-stack differential: reference vs candidate", ""]
|
||||
failures = []
|
||||
if len(reports) != 1:
|
||||
failures.append(f"expected exactly one self-comparison report, found {len(reports)}: {reports}")
|
||||
else:
|
||||
report = json.load(open(reports[0]))
|
||||
side = report.get("reference_side")
|
||||
version_a = report.get("version_a")
|
||||
version_b = report.get("version_b")
|
||||
lines += [
|
||||
f"- report: `{reports[0]}`",
|
||||
f"- reference_side: `{side}`",
|
||||
f"- version_a: `{(version_a or '').splitlines()[0] if version_a else version_a}`",
|
||||
f"- version_b: `{(version_b or '').splitlines()[0] if version_b else version_b}`",
|
||||
f"- stack_a: `{report.get('stack_a')}`, stack_b: `{report.get('stack_b')}`",
|
||||
f"- compared_ops: {report.get('compared_ops')}, compared_cases: {report.get('compared_cases')}",
|
||||
"",
|
||||
]
|
||||
print(f"reference_side={side}\nversion_a={version_a}\nversion_b={version_b}")
|
||||
lines += ["```", "version_a:", version_a or "", "", "version_b:", version_b or "", "```", ""]
|
||||
if side != "a":
|
||||
failures.append(f"reference_side is {side!r}, expected 'a'")
|
||||
if not version_a or not version_b:
|
||||
failures.append("version_a or version_b is missing")
|
||||
elif version_a == version_b:
|
||||
failures.append("version_a equals version_b: the reference was replaced by the candidate")
|
||||
diffs = report.get("differences") or []
|
||||
lines.append(f"### Differences: {len(diffs)}")
|
||||
by_op = {}
|
||||
for d in diffs:
|
||||
by_op.setdefault(d["op"], []).append(d)
|
||||
for op in sorted(by_op):
|
||||
lines += ["", f"#### {op}", "", "| case | field | a (reference) | b (candidate) |", "| --- | --- | --- | --- |"]
|
||||
for d in by_op[op]:
|
||||
cell = lambda v: "`" + str(v).replace("|", "\\|").replace("`", "'") + "`"
|
||||
lines.append(f"| {d['case']} | {d['field']} | {cell(d['a'])} | {cell(d['b'])} |")
|
||||
if failures:
|
||||
lines += ["", "### Observation check failed", ""] + [f"- {f}" for f in failures]
|
||||
open(summary, "a").write("\n".join(lines) + "\n")
|
||||
print("\n".join(lines))
|
||||
for f in failures:
|
||||
print(f"::error::{f}")
|
||||
sys.exit(1 if failures else 0)
|
||||
PY
|
||||
|
||||
- name: Upload differential reports
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: reference-differential-report
|
||||
overwrite: true
|
||||
path: |
|
||||
target/dual-stack/*.json
|
||||
${{ runner.temp }}/differential-test.log
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
||||
- name: Fail when the differential found differences
|
||||
if: steps.differential.outcome == 'failure'
|
||||
run: |
|
||||
echo "::error::the dual-stack differential failed; see the job summary for every (op, field, a, b)"
|
||||
exit 1
|
||||
|
||||
compare:
|
||||
name: Compare targets
|
||||
needs: [ plan, s3tests ]
|
||||
if: always() && needs.plan.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Download target reports
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
pattern: reference-parity-report-*
|
||||
path: parity-download
|
||||
|
||||
- name: Compare per node id
|
||||
env:
|
||||
TARGETS: ${{ needs.plan.outputs.targets }}
|
||||
S3TESTS_RESULT: ${{ needs.s3tests.result }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
mkdir -p parity
|
||||
for dir in parity-download/reference-parity-report-*; do
|
||||
if [[ -d "$dir" ]]; then mv "$dir" "parity/${dir#parity-download/reference-parity-report-}"; fi
|
||||
done
|
||||
python3 scripts/s3-tests/reference_parity.py compare \
|
||||
--targets-json "$TARGETS" \
|
||||
--reports parity \
|
||||
--tolerated .config/reference-parity-tolerated.txt \
|
||||
--markdown "$GITHUB_STEP_SUMMARY" \
|
||||
--json parity/diff.json > parity/diff.md
|
||||
status=$?
|
||||
cat parity/diff.md
|
||||
if [[ "$S3TESTS_RESULT" != "success" ]]; then
|
||||
echo "::error::s3-tests target jobs finished with '$S3TESTS_RESULT'"
|
||||
status=1
|
||||
fi
|
||||
exit "$status"
|
||||
|
||||
- name: Upload parity diff
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: reference-parity-diff
|
||||
overwrite: true
|
||||
path: |
|
||||
parity/diff.md
|
||||
parity/diff.json
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
@@ -1,133 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
name: Release candidate (Linux)
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: Final product version (no preview suffix)
|
||||
type: string
|
||||
required: true
|
||||
source_sha:
|
||||
description: Full reviewed source commit
|
||||
type: string
|
||||
required: true
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
concurrency:
|
||||
group: release-candidate-${{ inputs.version }}
|
||||
cancel-in-progress: false
|
||||
jobs:
|
||||
freeze:
|
||||
if: github.repository == 'rustfs/rustfs' && github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
outputs:
|
||||
manifest: ${{ steps.freeze.outputs.manifest }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- name: Freeze or recover original inputs
|
||||
id: freeze
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
VERSION: ${{ inputs.version }}
|
||||
SOURCE_SHA: ${{ inputs.source_sha }}
|
||||
run: python3 scripts/release/pipeline.py prepare --version "$VERSION" --source "$SOURCE_SHA" --directory frozen
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: steps.freeze.outputs.reused != 'true'
|
||||
with:
|
||||
name: release-inputs
|
||||
path: frozen/inputs.json
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
linux:
|
||||
needs: freeze
|
||||
uses: ./.github/workflows/build.yml
|
||||
with:
|
||||
candidate_manifest: ${{ needs.freeze.outputs.manifest }}
|
||||
platforms: linux
|
||||
upgrade:
|
||||
needs: [freeze, linux]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 150
|
||||
env:
|
||||
RUSTUP_TOOLCHAIN: ${{ fromJson(needs.freeze.outputs.manifest).toolchain }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
ref: ${{ fromJson(needs.freeze.outputs.manifest).workflow_sha }}
|
||||
persist-credentials: false
|
||||
- uses: ./.github/actions/setup
|
||||
with:
|
||||
rust-version: ${{ fromJson(needs.freeze.outputs.manifest).toolchain }}
|
||||
cache-shared-key: release-upgrade-harness-hosted
|
||||
cache-save-if: ${{ github.ref == 'refs/heads/main' }}
|
||||
install-build-packaging-tools: "false"
|
||||
install-test-tools: "false"
|
||||
- name: Verify archived server against previous releases
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CANDIDATE_MANIFEST: ${{ needs.freeze.outputs.manifest }}
|
||||
run: |
|
||||
python3 - <<'PYVERIFY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, 'scripts/release')
|
||||
import candidate as c, pipeline, verify
|
||||
inputs = pipeline.manifest_env()
|
||||
run = pipeline.current_run('release-candidate.yml')
|
||||
if any(a['name'] == 'release-upgrade' for a in c.artifact_list(run['id'])):
|
||||
c.download_artifact(run, 'release-upgrade', 'receipts')
|
||||
pipeline.verify_evidence(inputs, Path('receipts'), {run['id']: run})
|
||||
c.emit_output('reused', 'true')
|
||||
else:
|
||||
target = 'linux-x86_64-gnu'
|
||||
c.download_artifact(run, f"candidate-{inputs['candidate_id']}-{target}", 'binary')
|
||||
verify.upgrade(inputs, Path('binary') / f"rustfs-{target}-v{inputs['version']}.zip", Path('receipts'))
|
||||
PYVERIFY
|
||||
id: verify
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: steps.verify.outputs.reused != 'true'
|
||||
with:
|
||||
name: release-upgrade
|
||||
path: receipts/*
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: failure()
|
||||
with:
|
||||
name: release-upgrade-failure-${{ github.run_attempt }}
|
||||
path: receipts/*
|
||||
retention-days: 7
|
||||
seal-linux:
|
||||
needs: [freeze, linux, upgrade]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Collect trusted receipts and seal Linux bytes
|
||||
id: seal
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
python3 scripts/release/pipeline.py linux-seal --directory sealed
|
||||
{
|
||||
echo '## Linux acceptance'
|
||||
echo 'Use this run ID and manifest SHA-256 when starting Release candidate (complete).'
|
||||
sha256sum sealed/linux-manifest.json
|
||||
echo 'Record cluster, performance, integrity and upgrade acceptance in backlog before continuing.'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: steps.seal.outputs.reused != 'true'
|
||||
with:
|
||||
name: release-linux
|
||||
path: sealed/*
|
||||
if-no-files-found: error
|
||||
compression-level: 0
|
||||
retention-days: 30
|
||||
@@ -1,97 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
name: Release candidate (complete)
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
candidate_run_id:
|
||||
description: Successful Linux candidate run ID
|
||||
type: string
|
||||
required: true
|
||||
linux_manifest_sha256:
|
||||
description: Exact accepted Linux manifest SHA-256
|
||||
type: string
|
||||
required: true
|
||||
acceptance_report:
|
||||
description: Backlog report accepting cluster, performance, integrity and upgrade results
|
||||
type: string
|
||||
required: true
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
concurrency:
|
||||
group: release-complete-${{ inputs.candidate_run_id }}
|
||||
cancel-in-progress: false
|
||||
jobs:
|
||||
acceptance:
|
||||
if: github.repository == 'rustfs/rustfs' && github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
environment: release-linux-acceptance
|
||||
outputs:
|
||||
manifest: ${{ steps.accept.outputs.manifest }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- name: Bind acceptance to the exact Linux manifest
|
||||
id: accept
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CANDIDATE_RUN: ${{ inputs.candidate_run_id }}
|
||||
MANIFEST_SHA: ${{ inputs.linux_manifest_sha256 }}
|
||||
REPORT: ${{ inputs.acceptance_report }}
|
||||
run: python3 scripts/release/pipeline.py complete-prepare --run-id "$CANDIDATE_RUN" --digest "$MANIFEST_SHA" --report "$REPORT" --directory linux
|
||||
desktop:
|
||||
needs: acceptance
|
||||
uses: ./.github/workflows/build.yml
|
||||
with:
|
||||
candidate_manifest: ${{ needs.acceptance.outputs.manifest }}
|
||||
platforms: desktop
|
||||
packages:
|
||||
needs: acceptance
|
||||
uses: ./.github/workflows/package.yml
|
||||
with:
|
||||
candidate_manifest: ${{ needs.acceptance.outputs.manifest }}
|
||||
images:
|
||||
needs: acceptance
|
||||
uses: ./.github/workflows/docker.yml
|
||||
with:
|
||||
candidate_manifest: ${{ needs.acceptance.outputs.manifest }}
|
||||
chart:
|
||||
needs: acceptance
|
||||
uses: ./.github/workflows/helm-package.yml
|
||||
with:
|
||||
candidate_manifest: ${{ needs.acceptance.outputs.manifest }}
|
||||
seal:
|
||||
needs: [acceptance, desktop, packages, images, chart]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- name: Verify every deliverable and seal the candidate
|
||||
id: seal
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CANDIDATE_RUN: ${{ inputs.candidate_run_id }}
|
||||
MANIFEST_SHA: ${{ inputs.linux_manifest_sha256 }}
|
||||
REPORT: ${{ inputs.acceptance_report }}
|
||||
run: |
|
||||
python3 scripts/release/pipeline.py complete-seal --run-id "$CANDIDATE_RUN" --digest "$MANIFEST_SHA" --report "$REPORT" --directory sealed
|
||||
{
|
||||
echo '## Final publication approval'
|
||||
echo 'Review this manifest, then explicitly dispatch Publish accepted release with its SHA-256.'
|
||||
sha256sum sealed/release-manifest.json
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
if: steps.seal.outputs.reused != 'true'
|
||||
with:
|
||||
name: release-seal
|
||||
path: sealed/*
|
||||
if-no-files-found: error
|
||||
compression-level: 0
|
||||
retention-days: 30
|
||||
@@ -1,193 +0,0 @@
|
||||
# Copyright 2026 RustFS Team
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
name: Publish accepted release
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
complete_run_id:
|
||||
description: Successful completed candidate run ID
|
||||
type: string
|
||||
required: true
|
||||
manifest_sha256:
|
||||
description: Exact approved final manifest SHA-256 (authorizes publication)
|
||||
type: string
|
||||
required: true
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
concurrency:
|
||||
group: release-publish-${{ inputs.complete_run_id }}
|
||||
cancel-in-progress: false
|
||||
jobs:
|
||||
verify:
|
||||
if: github.repository == 'rustfs/rustfs' && github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
outputs:
|
||||
version: ${{ steps.verify.outputs.version }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- name: Recheck approved bytes and trusted certification
|
||||
id: verify
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
COMPLETE_RUN: ${{ inputs.complete_run_id }}
|
||||
MANIFEST_SHA: ${{ inputs.manifest_sha256 }}
|
||||
run: python3 scripts/release/publish.py verify --run-id "$COMPLETE_RUN" --digest "$MANIFEST_SHA" --directory sealed
|
||||
version:
|
||||
needs: verify
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
concurrency:
|
||||
group: release-version-${{ needs.verify.outputs.version }}
|
||||
cancel-in-progress: false
|
||||
environment: release-publication
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
packages: write
|
||||
security-events: write
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
COMPLETE_RUN: ${{ inputs.complete_run_id }}
|
||||
MANIFEST_SHA: ${{ inputs.manifest_sha256 }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: auto
|
||||
AWS_REQUEST_CHECKSUM_CALCULATION: when_required
|
||||
AWS_RESPONSE_CHECKSUM_VALIDATION: when_required
|
||||
AWS_EC2_METADATA_DISABLED: "true"
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- name: Install promotion tools
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y skopeo
|
||||
aws --version
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
username: rustfs
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_PASSWORD }}
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: quay.io
|
||||
username: ${{ secrets.QUAY_USERNAME }}
|
||||
password: ${{ secrets.QUAY_PASSWORD }}
|
||||
- name: Share registry credentials with the OCI copier
|
||||
run: |
|
||||
install -m 600 "$HOME/.docker/config.json" "$RUNNER_TEMP/registry-auth.json"
|
||||
echo "REGISTRY_AUTH_FILE=$RUNNER_TEMP/registry-auth.json" >> "$GITHUB_ENV"
|
||||
- name: Promote verified version
|
||||
run: python3 scripts/release/publish.py version --run-id "$COMPLETE_RUN" --digest "$MANIFEST_SHA" --directory sealed
|
||||
- name: Publish accepted musl amd64 scan
|
||||
uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4
|
||||
with:
|
||||
sarif_file: sealed/scan-musl-amd64.sarif
|
||||
category: container-image-musl-amd64
|
||||
- name: Publish accepted musl arm64 scan
|
||||
uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4
|
||||
with:
|
||||
sarif_file: sealed/scan-musl-arm64.sarif
|
||||
category: container-image-musl-arm64
|
||||
- name: Publish accepted GNU amd64 scan
|
||||
uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4
|
||||
with:
|
||||
sarif_file: sealed/scan-gnu-amd64.sarif
|
||||
category: container-image-gnu-amd64
|
||||
- name: Publish accepted GNU arm64 scan
|
||||
uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9 # v4
|
||||
with:
|
||||
sarif_file: sealed/scan-gnu-arm64.sarif
|
||||
category: container-image-gnu-arm64
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-version-receipt-${{ github.run_attempt }}
|
||||
path: publication.json
|
||||
if-no-files-found: error
|
||||
retention-days: 90
|
||||
channels:
|
||||
needs: [verify, version]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 90
|
||||
concurrency:
|
||||
group: rustfs-release-channels
|
||||
cancel-in-progress: false
|
||||
permissions:
|
||||
contents: write
|
||||
actions: read
|
||||
packages: write
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
COMPLETE_RUN: ${{ inputs.complete_run_id }}
|
||||
MANIFEST_SHA: ${{ inputs.manifest_sha256 }}
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: auto
|
||||
AWS_REQUEST_CHECKSUM_CALCULATION: when_required
|
||||
AWS_RESPONSE_CHECKSUM_VALIDATION: when_required
|
||||
AWS_EC2_METADATA_DISABLED: "true"
|
||||
R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }}
|
||||
R2_BUCKET: ${{ secrets.R2_BUCKET }}
|
||||
steps:
|
||||
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
persist-credentials: false
|
||||
- name: Install promotion tools
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y skopeo
|
||||
aws --version
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
username: rustfs
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ secrets.GHCR_USERNAME }}
|
||||
password: ${{ secrets.GHCR_PASSWORD }}
|
||||
- uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
|
||||
with:
|
||||
registry: quay.io
|
||||
username: ${{ secrets.QUAY_USERNAME }}
|
||||
password: ${{ secrets.QUAY_PASSWORD }}
|
||||
- name: Share registry credentials with the OCI copier
|
||||
run: |
|
||||
install -m 600 "$HOME/.docker/config.json" "$RUNNER_TEMP/registry-auth.json"
|
||||
echo "REGISTRY_AUTH_FILE=$RUNNER_TEMP/registry-auth.json" >> "$GITHUB_ENV"
|
||||
- uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0
|
||||
- name: Install pinned OSS client
|
||||
run: |
|
||||
curl --fail --location --retry 3 --output "$RUNNER_TEMP/ossutil.zip" https://gosspublic.alicdn.com/ossutil/v2/2.1.1/ossutil-2.1.1-linux-amd64.zip
|
||||
printf '%s %s\n' 60f3a808cbbdfd4b5269b8f967c6a66f564c9dacff52d93a5d21a588976ab5a2 "$RUNNER_TEMP/ossutil.zip" | sha256sum --check -
|
||||
unzip -q "$RUNNER_TEMP/ossutil.zip" -d "$RUNNER_TEMP/ossutil-extract"
|
||||
install -m 755 "$RUNNER_TEMP/ossutil-extract/ossutil-2.1.1-linux-amd64/ossutil" "$RUNNER_TEMP/ossutil"
|
||||
echo "OSSUTIL_BIN=$RUNNER_TEMP/ossutil" >> "$GITHUB_ENV"
|
||||
- name: Promote verified channels
|
||||
env:
|
||||
HELM_TOKEN: ${{ secrets.RUSTFS_HELM_PACKAGE }}
|
||||
OSS_ACCESS_KEY_ID: ${{ secrets.ALICLOUDOSS_KEY_ID }}
|
||||
OSS_ACCESS_KEY_SECRET: ${{ secrets.ALICLOUDOSS_KEY_SECRET }}
|
||||
OSS_REGION: cn-beijing
|
||||
OSS_ENDPOINT: https://oss-cn-beijing.aliyuncs.com
|
||||
run: python3 scripts/release/publish.py channels --run-id "$COMPLETE_RUN" --digest "$MANIFEST_SHA" --directory sealed
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
with:
|
||||
name: release-channels-receipt-${{ github.run_attempt }}
|
||||
path: publication.json
|
||||
if-no-files-found: error
|
||||
retention-days: 90
|
||||
@@ -26,13 +26,7 @@ permissions:
|
||||
jobs:
|
||||
alert-on-incomplete-run:
|
||||
name: Alert on incomplete scheduled run
|
||||
# The registry still owns responsibility and cadence selection. Successful
|
||||
# and PR runs can never produce an incomplete scheduled-run alert.
|
||||
if: >-
|
||||
github.repository == 'rustfs/rustfs' &&
|
||||
github.event.workflow_run.conclusion != 'success' &&
|
||||
(github.event.workflow_run.event == 'schedule' ||
|
||||
github.event.workflow_run.event == 'workflow_run')
|
||||
if: github.repository == 'rustfs/rustfs'
|
||||
# Stay on GitHub-hosted runners: this monitor must keep running when the
|
||||
# self-hosted sm-standard-* pool is down, which is exactly when the
|
||||
# scheduled validations it watches stall.
|
||||
|
||||
@@ -7,10 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Notifications and audit
|
||||
|
||||
- Notify and audit targets that keep the default `queue_dir` now work on native Windows: the defaults `/opt/rustfs/events` and `/opt/rustfs/audit` resolve per node, to `%ProgramData%\RustFS\events` and `%ProgramData%\RustFS\audit` on Windows, instead of failing validation as non-absolute paths. The new `RUSTFS_TARGET_STORE_DIR` (Helm `config.rustfs.target_store_dir`) relocates both default queues on any platform. Linux and macOS behavior is unchanged when it is unset, persisted configuration is unchanged, and pending entries are not migrated when the root changes. See `docs/operations/target-queue-storage.md`.
|
||||
|
||||
### Deployment
|
||||
|
||||
- Existing volumes created by older root-running container images must be writable by the non-root runtime identity before upgrading. The default container UID/GID is `10001:10001`; stop the affected instances, verify host mount paths, and for ordinary bind mounts run `sudo chown -R 10001:10001 /actual/data /actual/logs` on each RustFS data/log mount. Account for ACLs, NFS, Kubernetes, and rootless UID mappings. Image-build ownership changes do not migrate mounted data. See the Docker Quick Start in [README.md](README.md).
|
||||
@@ -18,13 +14,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Security
|
||||
|
||||
- **Key-scoped KMS policy statements now hold for every spelling of an AWS key**: a statement such as `Deny kms:Decrypt` on `arn:aws:kms:::key/<key-id>` was matched against the key identifier exactly as the request spelled it, so naming the same key by its AWS ARN (`arn:aws:kms:<region>:<account>:key/<key-id>`) was evaluated as a different key and the `Deny` did not apply. With `RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true` and the AWS KMS backend, an identity allowed on all keys but denied one could still write under that key, and read objects stored under its ARN spelling; the per-key admin endpoints were affected the same way. A key ARN is now evaluated as its key id, on the SSE-KMS data path and the admin endpoints alike. A request that names a key by alias, which only the backend can resolve, now fails closed against key-scoped statements: every key-scoped `Deny` applies and only an `Allow` on every key covers it. The KMS deletion impact report recognizes the default key and bucket default-encryption references across the id and ARN spellings as well. Deployments without key-scoped KMS statements, and the other KMS backends, whose key ids have a single spelling, are unaffected.
|
||||
|
||||
- **Header-signed SigV4 requests honour only signed headers** (GHSA-xm99-m3gq-83g8): a request authenticated with a SigV4 `Authorization` header that carries an `x-amz-*` request header not listed in its `SignedHeaders` is now rejected with `403 AccessDenied` ("There were headers present in the request which were not signed"), matching AWS S3 and the presigned rule from GHSA-g8w9-qw9q-fghr. Previously anyone holding one header-signed `PutObject` request could add an unsigned `x-amz-copy-source` and turn it into a `CopyObject` that ran with the signer's permissions, copying any object the signer could read into the target. An `Authorization` header whose algorithm token is not `AWS4-HMAC-SHA256` is now rejected instead of being verified as SigV4. The request-envelope headers `x-amz-content-sha256`, `x-amz-decoded-content-length`, `x-amz-trailer` and `x-amz-checksum-algorithm` (the same set the upstream `s3s` fix exempts) and `x-amz-cf-id` (CloudFront) remain tolerated unsigned; AWS SDKs and RustFS's own signers already sign every other `x-amz-*` header. SigV2, JWT and anonymous requests are unchanged.
|
||||
|
||||
### Replication
|
||||
|
||||
- Site replication now replicates SSE-KMS objects when the receiving site runs with `RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true`. The receiving site authorizes each replica write against the KMS key it resolves to, and the managed `site-replicator-0` account held no `kms` grant, so every SSE-KMS object stayed `FAILED` with `AccessDenied` while SSE-S3 and SSE-C objects replicated. The account's embedded policy now grants `kms:GenerateDataKey` on every key; existing accounts are refreshed by the reconcile tick and the failed objects catch up afterwards. It is deliberately not granted `kms:Decrypt`, so reads proxied to a peer for an SSE-KMS object are still refused under enforcement. Bucket replication through a remote target needs the same grant on the identity whose credentials the target uses. Sites that do not enable enforcement are unaffected.
|
||||
- Object Lock replication PUTs now carry a required integrity header, fixing target rejection introduced by the plain-payload default ([#7097](https://github.com/rustfs/rustfs/pull/7097)). This changes the default outbound request for locked objects but adds no persisted format.
|
||||
- Multipart source objects stay on the multipart transport even when their checksum record is a whole-object checksum, so objects above the single-PUT limit remain replicable ([#7047](https://github.com/rustfs/rustfs/pull/7047)).
|
||||
- Targets that mint their own version IDs now use a per-target version ledger for tag, retention, legal-hold, and permanent-delete mutations; ambiguous pre-ledger matches fail with backoff instead of guessing ([#7368](https://github.com/rustfs/rustfs/pull/7368)). This adds dual-prefixed internal metadata keys that older readers ignore.
|
||||
@@ -40,11 +33,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
- **Presigned URLs honour only signed headers** (GHSA-g8w9-qw9q-fghr): a SigV4 presigned request that carries an `x-amz-*` request header not listed in `X-Amz-SignedHeaders` is now rejected with `403 AccessDenied` ("There were headers present in the request which were not signed"), matching AWS S3. Previously the holder of a presigned `PutObject` URL could add unsigned `x-amz-tagging`, `x-amz-storage-class`, `x-amz-website-redirect-location`, ACL, metadata, Object Lock or SSE headers and have them applied. Presigners that intend a property must set it before signing so the SDK lists the header in `SignedHeaders`; `x-amz-cf-id` (CloudFront) remains tolerated unsigned. Header-signed SigV4 and SigV2 requests are unchanged.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **KMS keys could not be described by ARN through the path route**: `GET /rustfs/admin/v3/kms/keys/{key_id}` read the path segment verbatim, so a key id that has to be percent-encoded to fit in one segment — an AWS key ARN, with its `:` and `/` — was looked up, authorized and audited under its encoded spelling and answered `404`. The segment is now percent-decoded first; a segment that is not valid UTF-8 after decoding is refused with `400 InvalidArgument`. The `describe-key?keyId=` form was unaffected.
|
||||
|
||||
- **SSE-KMS objects written through the AWS KMS backend could not be read**: `GetObject`, `CopyObject` and every other read of an `aws:kms` object failed with `400 InvalidRequest` ("SSE-S3 requires RUSTFS_SSE_S3_MASTER_KEY to be set ...") on a deployment using `RUSTFS_KMS_BACKEND=aws`, although the write and `HeadObject` succeeded. The AWS backend stored the `GenerateDataKey` blob as it came from AWS, the read path took those unrecognized bytes for a locally wrapped key, and AWS was never asked to decrypt. The backend now stores the blob inside the same data-key envelope the other backends use, and a blob stored without one is routed to AWS when the AWS backend is active. Objects written by 1.0.4 and 1.0.5 are intact and become readable after the upgrade, with no rewrite. Setting `RUSTFS_SSE_S3_MASTER_KEY` was never a workaround. An object written by this release is not readable by an older one; the same was already true of every object the older release wrote itself.
|
||||
- **Immediate KMS key deletion no longer reports success without destroying the key** (behavior change): with `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, a `DELETE /rustfs/admin/v3/kms/keys/delete` request with `force_immediate` against an `Enabled` or `Disabled` key on the Vault KV2 or Vault Transit backend returned success but only moved the key to `PendingDeletion`; the key material stayed in Vault and objects encrypted under it remained readable, while the Local backend destroyed the key in that one request. Key destruction is now the same two explicit steps on every backend: schedule the deletion, then either let the waiting window elapse or send `force_immediate` to destroy the pending key early. `force_immediate` against a key that is not `PendingDeletion` is refused with `400` and changes nothing, so a success response always means the key is gone and no single request can destroy a key in use. Automation that destroyed Local keys with one `force_immediate` request must schedule the deletion first. On Vault Transit, a read under a key that no longer exists in Vault now fails with `KMS.NotFoundException` instead of `503 ServiceUnavailable`. No persisted format changes. See `docs/operations/kms-key-destruction.md`.
|
||||
- **`mc admin info` and `mc admin replicate resync` work again**: since the cluster snapshot view (#3803), `GET /v3/info` and `GET /v3/storageinfo` returned the madmin `InfoMessage` / `StorageInfo` nested under an `info` key next to `admin_discovery` (and, for `/info`, `bitrotSelftest`). madmin decodes those bodies directly, so `mc admin info` printed empty values and `mc admin replicate resync start/status` rejected every alias with "alias provided is not part of cluster replication" because the local `deploymentID` read as empty. Requests on the MinIO-compatible `/minio/admin/v3` prefix now receive the bare madmin body again; `/rustfs/admin/v3`, which rc and the console use, keeps the envelope unchanged.
|
||||
- **Replication status writebacks no longer invalidate scanner cycles**: every replication status writeback (`persist_replication_state_if_current` in `Update` mode, including the heals the scanner queues during its own walk) marked the bucket dirty on the namespace path, advanced the dirty-usage generation and woke the scanner, so a site-replication resync or any steadily replicated bucket ended every cycle `superseded`, the authoritative usage baseline was never certified, and the console reported usage freshness as stale for as long as replication ran (#8375). Those writebacks are now status class: they do not touch the dirty-usage generation, the activity snapshot, the dirty-usage wake or the durable dirty journal, and only the process-local producer coverage is recorded. They need no rescan queue, because a deployment with an active replication rule already walks every bucket on every cycle and that walk refreshes `replication_stats`. Tier-expiration cleanup, which removes data, stays on the namespace path. No persisted format, protocol, admin API or configuration changes. Design: `docs/architecture/scanner-per-bucket-convergence.md`.
|
||||
- **A bucket written during a scanner walk no longer supersedes the whole cycle**: the post-walk cluster activity proof required the entire activity snapshot and the dirty-usage generation to be unchanged, so a single-node deployment with one bucket written every few seconds never reached a `Complete` cycle, `usage_snapshot_converged` never became true and the authoritative usage baseline was never certified (#8494). Dirty usage is now certified per bucket: a structurally complete walk that the guardrail did not veto is `Complete`, buckets whose dirty generation still matches the start snapshot are cleared, and buckets written during the walk stay dirty and are published as the pending set, so `usage_snapshot_converged` means "no bucket pending" and the next cycle is scoped to the pending buckets. Such a cycle writes the authoritative usage object; the observed object is now selected by an explicit observational mark that vetoed cycles carry. The guardrail itself vetoes only on the fields that change which namespace the walk described (peer set, process instance, protocol version, topology, maintenance generation, data movement, movement generation, publication blocked) and reports the field and host in the scanner log and in the new `usage_freshness` fields `guardrail_veto_total`, `last_guardrail_veto_unix_secs` and `last_guardrail_veto_reason`. In a distributed deployment a remote peer whose dirty usage moved during the walk still supersedes the cycle until scoped acknowledgements can name its buckets. `.usage.v2.json` gains the additive, default-when-absent fields `usage_snapshot_pending_buckets`, `usage_snapshot_pending_bucket_count` and `usage_snapshot_observational`; older readers ignore them and treat the snapshot as an authoritative but unconverged publication. Quota is unchanged: an uncertified bucket keeps its last usage value as the floor and live writes do not advance it. No protocol or configuration changes. Design: `docs/architecture/scanner-per-bucket-convergence.md`.
|
||||
|
||||
Generated
+55
-185
@@ -322,9 +322,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "archive-trait"
|
||||
version = "0.0.16"
|
||||
version = "0.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b5040ca935c09888e7dfb04383b4d62fa3571e25bc66b6f345b591e97958524"
|
||||
checksum = "999bf88cc483664cab77700ca2fdfb0415a73412abe284ef5bc2319d00938e00"
|
||||
dependencies = [
|
||||
"cap-std",
|
||||
"thiserror 2.0.21",
|
||||
@@ -811,9 +811,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "async-nats"
|
||||
version = "0.52.0"
|
||||
version = "0.51.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "015f0a7a3b0b29701b78c76a4d4a122624c97c71fa30363381664eb729d619c7"
|
||||
checksum = "a4f92759f418774b78c36ef07885849150ce950c67d9bcdeff8a9c253f57d9a1"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"base64 0.22.1",
|
||||
@@ -3906,9 +3906,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9"
|
||||
version = "0.5.8"
|
||||
version = "0.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c83757d61dc75b3e9d3579a3004c7fe0c42aff2f779ca9eeec7f0c8e1ae3fc10"
|
||||
checksum = "2f3298a54fd82af243c9f122595762f5b0e48bd3d1ea8fe3f3eb5e95c130e881"
|
||||
dependencies = [
|
||||
"bon",
|
||||
"dial9-core",
|
||||
@@ -3990,9 +3990,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "dial9-tokio-telemetry"
|
||||
version = "0.5.8"
|
||||
version = "0.5.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f0eb05627c12c57e40510e63f9a3d5f628ab4b32fec9ccdb0ae98544c13f15e"
|
||||
checksum = "0428cba60fbcf37ab82666f3a7ef3b6d0671e90dfb6a352376813b471319f061"
|
||||
dependencies = [
|
||||
"bon",
|
||||
"bytes",
|
||||
@@ -4525,9 +4525,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.11"
|
||||
version = "1.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4"
|
||||
checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
@@ -4594,7 +4594,7 @@ checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a"
|
||||
dependencies = [
|
||||
"io-lifetimes 2.0.4",
|
||||
"rustix",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4698,12 +4698,6 @@ version = "0.3.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
|
||||
|
||||
[[package]]
|
||||
name = "futures-timer"
|
||||
version = "3.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
|
||||
|
||||
[[package]]
|
||||
name = "futures-util"
|
||||
version = "0.3.34"
|
||||
@@ -5391,9 +5385,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath"
|
||||
version = "0.31.1"
|
||||
version = "0.30.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4039758c8bca06a34c475d17aaafc17ac1073ce2cecd8dba8c82e69edb300bbe"
|
||||
checksum = "d9632c1fd213e63c1b88d8f3e59543da4999b17f859cc1e1d67cf8f2207e8935"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -5426,15 +5420,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-drain"
|
||||
version = "0.31.1"
|
||||
version = "0.30.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbba0a3badfb9ebf95f22dc6cdb4afcd0ff07b91a85fa3196fd1060576ea3065"
|
||||
checksum = "738324f75aec847dea984f6806402f4f1bdfab437b4700fc8c3ebcad2dff6424"
|
||||
|
||||
[[package]]
|
||||
name = "hotpath-macros"
|
||||
version = "0.31.1"
|
||||
version = "0.30.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7d980014875338771b00ac44f1c9e2ea1a10650aa68ce54226233bc767714361"
|
||||
checksum = "5d92fdfec803932e5764ea93c201cf7c6c2d8cff1871da44f64f5094c9c3ef8a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -6418,9 +6412,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "lzma-rust2"
|
||||
version = "0.25.0"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef1d73c59a56771343712b22ea5615092c612c16ba291c0979e3105dfeec4890"
|
||||
checksum = "490750560fa7851fbbf5d3ab1a8750f09350ed361ed8aed85b1e1e40587812ee"
|
||||
dependencies = [
|
||||
"sha2 0.11.1",
|
||||
]
|
||||
@@ -7920,34 +7914,10 @@ version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared 0.14.0",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"phf_shared 0.14.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.12.1"
|
||||
@@ -10029,119 +9999,10 @@ dependencies = [
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway"
|
||||
version = "0.62.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"futures-timer",
|
||||
"futures-util",
|
||||
"http 1.5.0",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"httparse",
|
||||
"hyper",
|
||||
"pin-project-lite",
|
||||
"rustfs-gateway-core",
|
||||
"rustfs-gateway-http",
|
||||
"rustfs-gateway-macros",
|
||||
"rustfs-gateway-sig",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-gateway-xml",
|
||||
"sha2 0.11.1",
|
||||
"tokio",
|
||||
"tower",
|
||||
"tracing",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-adapter"
|
||||
version = "1.0.5"
|
||||
dependencies = [
|
||||
"http 1.5.0",
|
||||
"rustfs-credentials",
|
||||
"rustfs-gateway",
|
||||
"rustfs-gateway-http",
|
||||
"rustfs-iam",
|
||||
"rustfs-policy",
|
||||
"serde_json",
|
||||
"time",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-core"
|
||||
version = "0.60.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"http 1.5.0",
|
||||
"md-5 0.11.0",
|
||||
"percent-encoding",
|
||||
"rustfs-gateway-http",
|
||||
"rustfs-gateway-sig",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-types",
|
||||
"rustfs-gateway-xml",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-http"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"httparse",
|
||||
"memchr",
|
||||
"phf 0.14.0",
|
||||
"rustfs-gateway-stream",
|
||||
"rustfs-gateway-types",
|
||||
"sha2 0.11.1",
|
||||
"smallvec",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-macros"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 3.0.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-sig"
|
||||
version = "0.15.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
"rustfs-gateway-http",
|
||||
"sha1 0.11.0",
|
||||
"sha2 0.11.1",
|
||||
"smallvec",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-stream"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
@@ -10151,8 +10012,8 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-gateway-types"
|
||||
version = "0.40.0+aws.2026-10-02"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
|
||||
version = "0.32.0+aws.2026-10-02"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
@@ -10175,7 +10036,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-xml"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=7915d06950f88611d883f6ab3d95efc90c65dec0#7915d06950f88611d883f6ab3d95efc90c65dec0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=21283e874c2381bab611e9347acbc5b92e9e2b3c#21283e874c2381bab611e9347acbc5b92e9e2b3c"
|
||||
dependencies = [
|
||||
"quick-xml 0.41.0",
|
||||
]
|
||||
@@ -10527,18 +10388,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc"
|
||||
version = "0.7.4"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc7914232a854336c62058c24a8362c553ac8e7adba2e2395ce9097db24d1510"
|
||||
checksum = "216ecfc9341b5d6e2506d47ce57c9232ae1b8020bb10264ea05286fb234286e7"
|
||||
dependencies = [
|
||||
"rustfs-mimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-mimalloc-sys"
|
||||
version = "0.7.4"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8073624242781448512682f1931624ee67108b6b7873b22fb29c1a1df689ac0c"
|
||||
checksum = "fb16ad68f96883407a75cdb88252b43dc2ea592428f0fc54ff7885756de4b814"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
@@ -11912,9 +11773,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "serde_spanned"
|
||||
version = "1.1.3"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c187e0a369471c550af028cba56d4f4fa2a489ad28151ae95a4d340a2a8b782"
|
||||
checksum = "4e7523beb55eece201a2356bee0bbca0d1ab466c14c07703b2e0ee6d42cb0c2c"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
@@ -12244,9 +12105,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.16.5"
|
||||
version = "1.16.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46"
|
||||
checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -12441,9 +12302,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "starshard"
|
||||
version = "2.4.1"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0883700b50fd0b62120ebc4c07230e02f0cf1ead474c3b3b44eaa13e0a999ea"
|
||||
checksum = "fc45de5060fab0de00d7a42230877543ae00c37caf6f49975e69851089b023ac"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hashbrown 0.17.1",
|
||||
@@ -12692,9 +12553,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tar-codec"
|
||||
version = "0.0.16"
|
||||
version = "0.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6875ba507bbec6022500f46477159a03f07ebdffdb4ea043254974531df1dac2"
|
||||
checksum = "b9dad615e50d181afea4bafea23dacb8e33ed52dde06503813570634b932d467"
|
||||
dependencies = [
|
||||
"archive-trait",
|
||||
"tar-framing",
|
||||
@@ -12704,9 +12565,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tar-framing"
|
||||
version = "0.0.16"
|
||||
version = "0.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "49727b1a300514960d425d0e741a7da3e693170f3db39de3180fdbbc07e542c5"
|
||||
checksum = "563e8181b47c9cdcce569eae3ad431eec71543d6269dd60c7bd2350f70bcd233"
|
||||
dependencies = [
|
||||
"thiserror 2.0.21",
|
||||
"tokio",
|
||||
@@ -12742,10 +12603,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.3.4",
|
||||
"getrandom 0.4.3",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13080,9 +12941,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "1.2.1+spec-1.1.0"
|
||||
version = "1.2.0+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d5755c51863cdac711f1956baf8eb9588e060755f7d3c4c979e70ea1e1f32cd7"
|
||||
checksum = "c5ffcdf8f50436dbbfd675c1fe1b8e0854f42b7defea8584bdebe1aee900e561"
|
||||
dependencies = [
|
||||
"indexmap 2.14.2",
|
||||
"serde_core",
|
||||
@@ -13095,9 +12956,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "1.1.3+spec-1.1.0"
|
||||
version = "1.1.2+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49"
|
||||
checksum = "2b86d767906c6c42421dcba507eb9d203e779497710a47782a224bb871653053"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
@@ -13990,6 +13851,15 @@ dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.59.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
|
||||
dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.60.2"
|
||||
@@ -14162,7 +14032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"windows-sys 0.52.0",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14412,9 +14282,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.5.0"
|
||||
version = "9.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0700e6c427aff64acdbf02e85fbfd946c2ee12319d1af013092f3a2178914ff8"
|
||||
checksum = "eaeb7f16d2605b49aef586996ded29e65642fab7259a3077e537351df6d80426"
|
||||
dependencies = [
|
||||
"aes 0.9.4",
|
||||
"bzip2",
|
||||
|
||||
+6
-14
@@ -26,7 +26,6 @@ members = [
|
||||
"crates/ecstore", # Erasure coding storage implementation
|
||||
"crates/e2e_test", # End-to-end test suite
|
||||
"crates/filemeta", # File metadata management
|
||||
"crates/gateway-adapter", # RustFS implementations of the gateway extension traits
|
||||
"crates/heal", # Erasure set and object healing
|
||||
"crates/heal-contracts", # Heal request/response channel contracts
|
||||
"crates/iam", # Identity and Access Management
|
||||
@@ -94,7 +93,6 @@ redundant_clone = "warn"
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.5" }
|
||||
rustfs-gateway-adapter = { path = "crates/gateway-adapter", version = "1.0.5" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.5" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.5" }
|
||||
rustfs-scanner-metrics = { path = "crates/scanner-metrics", version = "1.0.5" }
|
||||
@@ -149,14 +147,8 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.40.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.62.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", default-features = false }
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -164,7 +156,7 @@ async_zip = { default-features = false, version = "0.0.19" }
|
||||
mysql_async = { default-features = false, version = "0.37.1" }
|
||||
async-compression = { version = "0.4.52" }
|
||||
async-trait = "0.1.92"
|
||||
async-nats = { version = "0.52.0", default-features = false }
|
||||
async-nats = { version = "0.51.0", default-features = false }
|
||||
axum = "0.8.9"
|
||||
futures = "0.3.34"
|
||||
futures-core = "0.3.34"
|
||||
@@ -264,8 +256,8 @@ anyhow = "1.0.104"
|
||||
arc-swap = "1.9.3"
|
||||
rustfs-tokio-tar = { version = "0.7.1" }
|
||||
# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
|
||||
tar-codec = "0.0.16"
|
||||
tar-framing = "0.0.16"
|
||||
tar-codec = "0.0.15"
|
||||
tar-framing = "0.0.15"
|
||||
atoi = "3.1.1"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.13.0" }
|
||||
@@ -409,7 +401,7 @@ dav-server = "0.11.0"
|
||||
# Performance Analysis and Memory Profiling
|
||||
rustfs-mimalloc = { version = "0.7.3" }
|
||||
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
|
||||
hotpath = { version = "0.31.1", default-features = false }
|
||||
hotpath = { version = "0.30.3", default-features = false }
|
||||
# Snapshot testing for output format regression detection
|
||||
insta = { version = "1.49" }
|
||||
|
||||
|
||||
+2
-11
@@ -12,9 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
ARG BINARY_SOURCE=download
|
||||
|
||||
FROM alpine:3.24.2 AS download
|
||||
FROM alpine:3.24.2 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
@@ -76,14 +74,6 @@ COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /build/rc
|
||||
|
||||
|
||||
# Candidate packaging supplies the accepted executables from verified archives.
|
||||
# The default download stage remains available for local image builds.
|
||||
FROM scratch AS supplied
|
||||
ARG TARGETARCH
|
||||
COPY .release-inputs/${TARGETARCH}/ /build/
|
||||
|
||||
FROM ${BINARY_SOURCE} AS build
|
||||
|
||||
FROM alpine:3.24.2
|
||||
|
||||
ARG RELEASE=latest
|
||||
@@ -113,6 +103,7 @@ RUN apk upgrade --no-cache && \
|
||||
tzdata \
|
||||
&& test "$(TZ=Asia/Kolkata date +%z)" = "+0530"
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
COPY --from=build /build/rc /usr/bin/rc
|
||||
COPY entrypoint.sh /etc/rustfs/entrypoint.sh
|
||||
|
||||
+1
-11
@@ -12,9 +12,7 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
ARG BINARY_SOURCE=download
|
||||
|
||||
FROM ubuntu:26.04 AS download
|
||||
FROM ubuntu:26.04 AS build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE=latest
|
||||
@@ -82,14 +80,6 @@ RUN set -eux; \
|
||||
COPY scripts/download_rc.sh /usr/local/bin/download_rc.sh
|
||||
RUN sh /usr/local/bin/download_rc.sh "$TARGETARCH" gnu "$RC_RELEASE" /build/rc
|
||||
|
||||
# Candidate packaging supplies the accepted executables from verified archives.
|
||||
# The default download stage remains available for local image builds.
|
||||
FROM scratch AS supplied
|
||||
ARG TARGETARCH
|
||||
COPY .release-inputs/${TARGETARCH}/ /build/
|
||||
|
||||
FROM ${BINARY_SOURCE} AS build
|
||||
|
||||
FROM ubuntu:26.04
|
||||
|
||||
ARG RELEASE=latest
|
||||
|
||||
+4
-22
@@ -24,10 +24,9 @@
|
||||
#
|
||||
# Supports cross-compilation for amd64 and arm64 via TARGETPLATFORM.
|
||||
|
||||
# Do not declare TARGETPLATFORM, BUILDPLATFORM or TARGETARCH here: a global
|
||||
# declaration without a value shadows BuildKit's automatic values, so the
|
||||
# stage-level re-declarations below see empty strings. They are only
|
||||
# declared inside the stage that uses them.
|
||||
ARG TARGETPLATFORM
|
||||
ARG BUILDPLATFORM
|
||||
ARG TARGETARCH
|
||||
ARG RUSTFS_BUILD_FEATURES=""
|
||||
ARG RC_RELEASE=latest
|
||||
|
||||
@@ -170,24 +169,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
;; \
|
||||
esac
|
||||
|
||||
# TARGETARCH is only provided by BuildKit. Fall back to TARGETPLATFORM (for
|
||||
# example set by docker-compose) and then to the build host, like the cargo
|
||||
# step above, so every path that builds this file passes a real architecture.
|
||||
RUN set -eu; \
|
||||
rc_arch="${TARGETARCH:-}"; \
|
||||
if [ -z "${rc_arch}" ]; then \
|
||||
case "${TARGETPLATFORM:-}" in \
|
||||
linux/amd64) rc_arch=amd64 ;; \
|
||||
linux/arm64) rc_arch=arm64 ;; \
|
||||
*) \
|
||||
case "$(uname -m)" in \
|
||||
x86_64) rc_arch=amd64 ;; \
|
||||
aarch64|arm64) rc_arch=arm64 ;; \
|
||||
*) echo "TARGETARCH is empty and TARGETPLATFORM='${TARGETPLATFORM:-}' is not linux/amd64 or linux/arm64; cannot choose an rc architecture on $(uname -m)" >&2; exit 1 ;; \
|
||||
esac ;; \
|
||||
esac; \
|
||||
fi; \
|
||||
sh /usr/src/rustfs/scripts/download_rc.sh "${rc_arch}" musl "${RC_RELEASE}" /usr/local/bin/rc
|
||||
RUN sh /usr/src/rustfs/scripts/download_rc.sh "$TARGETARCH" musl "$RC_RELEASE" /usr/local/bin/rc
|
||||
|
||||
# -----------------------------
|
||||
# Development stage (keeps toolchain)
|
||||
|
||||
@@ -145,7 +145,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.5
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.4
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
@@ -199,7 +199,7 @@ docker run -d --name rustfs -p 9000:9000 \
|
||||
Notes:
|
||||
- `RUSTFS_NOTIFY_ENABLE=true` enables the global notify module switch.
|
||||
- For ARN `arn:rustfs:sqs::primary:webhook`, use instance-scoped env vars with `_PRIMARY`.
|
||||
- If queue dir is omitted, the official image maps `/opt/rustfs/events` into the persistent `/data` volume so it is writable by the runtime user and pending events survive container recreation. Override `queue_dir` when using another deployment layout, or set `RUSTFS_TARGET_STORE_DIR` to move every default queue; native Windows defaults to `%ProgramData%\RustFS`. See [Target queue storage](docs/operations/target-queue-storage.md).
|
||||
- If queue dir is omitted, the official image maps `/opt/rustfs/events` into the persistent `/data` volume so it is writable by the runtime user and pending events survive container recreation. Override `queue_dir` when using another deployment layout.
|
||||
- `RUSTFS_NOTIFY_WEBHOOK_SKIP_TLS_VERIFY_PRIMARY` defaults to `false`; enabling it skips webhook TLS certificate verification, allows MITM attacks, and emits a startup warning. Prefer `RUSTFS_NOTIFY_WEBHOOK_CLIENT_CA_PRIMARY` for private CAs.
|
||||
- Since `1.0.0-beta.11`, webhook endpoints on private or container networks
|
||||
(`Docker Compose service names`, `host.docker.internal`, RFC 1918 addresses) are
|
||||
@@ -224,7 +224,7 @@ For developers who want to build RustFS Docker images from source with multi-arc
|
||||
./docker-buildx.sh --push
|
||||
|
||||
# Build specific version
|
||||
./docker-buildx.sh --release 1.0.5 --push
|
||||
./docker-buildx.sh --release 1.0.4 --push
|
||||
|
||||
# Build for custom registry
|
||||
./docker-buildx.sh --registry your-registry.com --namespace yourname --push
|
||||
@@ -242,7 +242,7 @@ You can also use Make targets for convenience:
|
||||
```bash
|
||||
make docker-buildx # Build locally
|
||||
make docker-buildx-push # Build and push
|
||||
make docker-buildx-version VERSION=1.0.5 # Build specific version
|
||||
make docker-buildx-version VERSION=1.0.4 # Build specific version
|
||||
make help-docker # Show all Docker-related commands
|
||||
```
|
||||
|
||||
|
||||
+3
-3
@@ -138,7 +138,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.5
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.4
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
@@ -176,7 +176,7 @@ docker compose -f docker-compose-simple.yml up -d
|
||||
./docker-buildx.sh --push
|
||||
|
||||
# 构建指定版本
|
||||
./docker-buildx.sh --release 1.0.5 --push
|
||||
./docker-buildx.sh --release 1.0.4 --push
|
||||
|
||||
# 构建并推送到自定义仓库
|
||||
./docker-buildx.sh --registry your-registry.com --namespace yourname --push
|
||||
@@ -193,7 +193,7 @@ docker compose -f docker-compose-simple.yml up -d
|
||||
```bash
|
||||
make docker-buildx # 本地构建
|
||||
make docker-buildx-push # 构建并推送
|
||||
make docker-buildx-version VERSION=1.0.5 # 构建指定版本
|
||||
make docker-buildx-version VERSION=1.0.4 # 构建指定版本
|
||||
make help-docker # 显示所有 Docker 相关命令
|
||||
```
|
||||
|
||||
|
||||
@@ -902,104 +902,4 @@ mod tests {
|
||||
assert_eq!(raw.len(), 8);
|
||||
assert_eq!(&raw[..], reference.digest().to_be_bytes().as_slice());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checksums_streaming_chunks_and_header_encoding_match_oneshot_across_unaligned_boundaries() {
|
||||
use crate::http::HttpChecksum;
|
||||
|
||||
let all = [
|
||||
ChecksumAlgorithm::Crc32,
|
||||
ChecksumAlgorithm::Crc32c,
|
||||
ChecksumAlgorithm::Crc64Nvme,
|
||||
ChecksumAlgorithm::Sha1,
|
||||
ChecksumAlgorithm::Sha256,
|
||||
ChecksumAlgorithm::Sha512,
|
||||
ChecksumAlgorithm::Xxhash3,
|
||||
ChecksumAlgorithm::Xxhash64,
|
||||
ChecksumAlgorithm::Xxhash128,
|
||||
];
|
||||
|
||||
for &payload_len in &[0usize, 1, 31, 33, 1024, 4096 + 37, 64 * 1024 + 19] {
|
||||
let payload: Vec<u8> = (0..payload_len).map(|i| ((i * 19 + 5) % 253) as u8).collect();
|
||||
for algorithm in all {
|
||||
let mut one_shot = algorithm.into_impl();
|
||||
one_shot.update(&payload);
|
||||
let expected_raw = one_shot.finalize();
|
||||
assert_eq!(expected_raw.len(), algorithm.raw_len());
|
||||
|
||||
let mut one_shot_hdr = algorithm.into_impl();
|
||||
one_shot_hdr.update(&payload);
|
||||
let expected_headers = one_shot_hdr.headers();
|
||||
|
||||
for &chunk_size in &[1usize, 31, 1021, 4096] {
|
||||
if payload_len > 4096 && chunk_size == 1 {
|
||||
continue;
|
||||
}
|
||||
let mut streaming: Box<dyn HttpChecksum> = algorithm.into_impl();
|
||||
for chunk in payload.chunks(chunk_size) {
|
||||
streaming.update(chunk);
|
||||
}
|
||||
let actual_headers = streaming.headers();
|
||||
assert_eq!(
|
||||
actual_headers.get(algorithm.http_header_name()),
|
||||
expected_headers.get(algorithm.http_header_name()),
|
||||
"{algorithm:?} streaming mismatch at payload_len={payload_len}, chunk_size={chunk_size}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "utility throughput microbenchmark; run via cargo test -p rustfs-checksums --lib --release -- --ignored --nocapture"]
|
||||
fn microbench_checksums_streaming_chunks_and_header_encoding() {
|
||||
use crate::http::HttpChecksum;
|
||||
use std::hint::black_box;
|
||||
use std::time::Instant;
|
||||
|
||||
let all = [
|
||||
ChecksumAlgorithm::Crc32,
|
||||
ChecksumAlgorithm::Crc32c,
|
||||
ChecksumAlgorithm::Crc64Nvme,
|
||||
ChecksumAlgorithm::Sha1,
|
||||
ChecksumAlgorithm::Sha256,
|
||||
ChecksumAlgorithm::Sha512,
|
||||
ChecksumAlgorithm::Xxhash3,
|
||||
ChecksumAlgorithm::Xxhash64,
|
||||
ChecksumAlgorithm::Xxhash128,
|
||||
];
|
||||
|
||||
let payload: Vec<u8> = (0..(256 * 1024)).map(|i| ((i * 19 + 5) % 253) as u8).collect();
|
||||
for &chunk_size in &[4 * 1024usize, 64 * 1024, 256 * 1024] {
|
||||
for algorithm in all {
|
||||
let mut one_shot = algorithm.into_impl();
|
||||
one_shot.update(&payload);
|
||||
let expected_headers = one_shot.headers();
|
||||
|
||||
let iters = 8usize;
|
||||
let t0 = Instant::now();
|
||||
let mut last_headers = http::HeaderMap::new();
|
||||
for _ in 0..iters {
|
||||
let mut streaming: Box<dyn HttpChecksum> = algorithm.into_impl();
|
||||
for chunk in payload.chunks(chunk_size) {
|
||||
streaming.update(black_box(chunk));
|
||||
}
|
||||
last_headers = streaming.headers();
|
||||
black_box(&last_headers);
|
||||
}
|
||||
let elapsed = t0.elapsed();
|
||||
assert_eq!(
|
||||
last_headers.get(algorithm.http_header_name()),
|
||||
expected_headers.get(algorithm.http_header_name()),
|
||||
"{algorithm:?} streaming chunk_size={chunk_size} header mismatch"
|
||||
);
|
||||
let total_mib = (payload.len() * iters) as f64 / (1024.0 * 1024.0);
|
||||
let mibs = total_mib / elapsed.as_secs_f64().max(1e-9);
|
||||
println!(
|
||||
"[microbench][checksums] algo={algorithm:?} chunk_size={chunk_size}B throughput={mibs:.1} MiB/s (size={}B)",
|
||||
algorithm.raw_len()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -276,13 +276,6 @@ Drive timeout profile preset:
|
||||
- values must be positive; `0` or an invalid value falls back to the default, and values above `60` are clamped to `60`.
|
||||
- the setting is read by the aggregating node only; it does not change the internode RPC wire contract. Any retry shares one round deadline rather than receiving a fresh timeout.
|
||||
|
||||
## Target queue directory
|
||||
|
||||
- `RUSTFS_TARGET_STORE_DIR`
|
||||
- absolute root for notify and audit target queues whose `queue_dir` is the default `/opt/rustfs/events` or `/opt/rustfs/audit`; they open `<root>/events` and `<root>/audit` on this node.
|
||||
- unset keeps those paths on Linux and macOS; native Windows uses `%ProgramData%\RustFS\events` and `%ProgramData%\RustFS\audit`.
|
||||
- explicit `queue_dir` values are never relocated, and the persisted target configuration is unchanged. Changing the root does not migrate pending entries. See `docs/operations/target-queue-storage.md`.
|
||||
|
||||
## Startup filesystem boundary policy
|
||||
|
||||
- `RUSTFS_UNSUPPORTED_FS_POLICY` controls startup behavior when RustFS detects local endpoint filesystems that are outside the supported production boundary.
|
||||
|
||||
@@ -144,12 +144,6 @@ pub const POSTGRES_QUEUE_LIMIT: &str = "queue_limit";
|
||||
/// Applies to both notify and audit target queue stores.
|
||||
pub const ENV_TARGET_STORE_COMPRESS: &str = "RUSTFS_TARGET_STORE_COMPRESS";
|
||||
|
||||
/// Environment variable that relocates the root of the default notify and audit queue directories.
|
||||
///
|
||||
/// It applies only to targets whose `queue_dir` is the logical default (`EVENT_DEFAULT_DIR` or
|
||||
/// `AUDIT_DEFAULT_DIR`); their queues move to `<root>/events` and `<root>/audit` on this node.
|
||||
pub const ENV_TARGET_STORE_DIR: &str = "RUSTFS_TARGET_STORE_DIR";
|
||||
|
||||
/// Queue-store compression is enabled by default to reduce disk footprint.
|
||||
pub const DEFAULT_TARGET_STORE_COMPRESS: bool = true;
|
||||
|
||||
|
||||
@@ -91,12 +91,7 @@ pub const ENV_OBS_LOG_MATCH_MODE: &str = "RUSTFS_OBS_LOG_MATCH_MODE";
|
||||
|
||||
/// Default values for log cleanup
|
||||
pub const DEFAULT_OBS_LOG_MAX_TOTAL_SIZE_BYTES: u64 = 2 * 1024 * 1024 * 1024; // 2 GiB
|
||||
pub const DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: u64 = 0; // No per-file trim by the cleaner
|
||||
/// Size at which the active log file is rolled when
|
||||
/// `RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES` is unset. Without a size roll
|
||||
/// the active file is bounded only by the rotation period, and the cleaner
|
||||
/// never touches the active file. An explicit `0` keeps size rolling off.
|
||||
pub const DEFAULT_OBS_LOG_ACTIVE_FILE_ROLL_SIZE_BYTES: u64 = 256 * 1024 * 1024; // 256 MiB
|
||||
pub const DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES: u64 = 0; // No single file limit
|
||||
pub const DEFAULT_OBS_LOG_COMPRESS_OLD_FILES: bool = true;
|
||||
pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL: u32 = 6;
|
||||
pub const DEFAULT_OBS_LOG_COMPRESSION_ALGORITHM_GZIP: &str = "gzip";
|
||||
|
||||
@@ -29,7 +29,6 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern:
|
||||
| **distributed 4×4** | [`src/distributed/`](src/distributed) | Nightly `e2e-distributed` lane: S3, object lock/WORM, versioning, bucket/site replication, quota, expand/decommission/rebalance, concurrency, chaos, 4-node upgrade of historical data and IAM AK/SK. Map: [`docs/testing/distributed-e2e.md`](../../docs/testing/distributed-e2e.md) |
|
||||
| **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup |
|
||||
| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory |
|
||||
| **dual-stack differential** | `dual_stack_differential` | Two single-node servers answer one SDK script for all 73 S3 operations and every differing `(op, field, a, b)` is reported. Side A is the binary under test on `RUSTFS_S3_STACK=legacy`; side B runs `RUSTFS_E2E_STACK_B`. Set `RUSTFS_E2E_REFERENCE_BINARY` to a released `rustfs` binary to make side A that binary instead (started without `RUSTFS_S3_STACK`, which older releases do not know). A missing, non-file or non-executable path fails the run; it never skips or falls back to the same-binary mode. The report in `<target>/dual-stack/<run>.json` records `reference_side` and each side's `--version` output |
|
||||
|
||||
The external-tool `storage_metric_ownership_test` validates the OTLP/Collector/Prometheus path, including a rolling upgrade and node failures. See the [storage metrics guide](../../docs/operations/storage-metrics.md) for its required binaries and focused command.
|
||||
|
||||
|
||||
@@ -774,18 +774,6 @@ impl RustFSTestEnvironment {
|
||||
.await
|
||||
}
|
||||
|
||||
/// Like [`Self::start_rustfs_server_from_binary`], but leaves other running RustFS processes
|
||||
/// alone, so that two servers of different binaries can be alive at once.
|
||||
pub async fn start_rustfs_server_from_binary_without_cleanup(
|
||||
&mut self,
|
||||
binary_path: &Path,
|
||||
extra_args: Vec<&str>,
|
||||
extra_env: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, false)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Start RustFS server with basic configuration
|
||||
pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.start_rustfs_server_inner(extra_args, &[], true).await
|
||||
|
||||
@@ -1285,9 +1285,7 @@ async fn list_parts_disk_loss_after_metadata_read(
|
||||
let error = result.expect_err("one missing directory and three unavailable participants cannot prove an empty upload");
|
||||
let response = error.raw_response().expect("ListParts must return an S3 service response");
|
||||
assert_eq!(response.status().as_u16(), 503);
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("ServiceUnavailable"));
|
||||
assert_eq!(error.as_service_error().and_then(|err| err.meta().code()), Some("SlowDownRead"));
|
||||
assert!(response.headers().get("x-rustfs-error-cause").is_none());
|
||||
assert!(response.headers().get("retry-after").is_none());
|
||||
assert!(
|
||||
|
||||
@@ -493,12 +493,7 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
|
||||
assert!(legacy.parts().is_empty());
|
||||
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
|
||||
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(
|
||||
current.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable")
|
||||
);
|
||||
assert_eq!(current.as_service_error().and_then(|error| error.meta().code()), Some("SlowDownRead"));
|
||||
dist.cluster.start_node_from_binary(3, previous).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
|
||||
for (path, before) in original_files {
|
||||
|
||||
@@ -232,12 +232,6 @@ pub(crate) struct Report {
|
||||
pub run: String,
|
||||
pub stack_a: String,
|
||||
pub stack_b: String,
|
||||
/// Which side is the pinned release binary: `"a"`, or `None` when both sides are the binary
|
||||
/// under test.
|
||||
pub reference_side: Option<&'static str>,
|
||||
/// `--version` output of the binary each side actually ran.
|
||||
pub version_a: String,
|
||||
pub version_b: String,
|
||||
pub region_a: Option<String>,
|
||||
pub region_b: Option<String>,
|
||||
pub compared_ops: usize,
|
||||
@@ -486,13 +480,10 @@ mod tests {
|
||||
fn the_report_lists_each_difference_as_op_field_a_b() {
|
||||
let directory = std::env::temp_dir().join(format!("dual-stack-report-{}", uuid::Uuid::new_v4()));
|
||||
let report = Report {
|
||||
schema: 2,
|
||||
schema: 1,
|
||||
run: "unit".into(),
|
||||
stack_a: "legacy".into(),
|
||||
stack_b: "legacy".into(),
|
||||
reference_side: Some("a"),
|
||||
version_a: "rustfs 1.0.4".into(),
|
||||
version_b: "rustfs 1.1.0".into(),
|
||||
region_a: None,
|
||||
region_b: Some("eu-west-1".into()),
|
||||
compared_ops: 1,
|
||||
@@ -507,9 +498,6 @@ mod tests {
|
||||
let _ = std::fs::remove_dir_all(&directory);
|
||||
assert_eq!(path.file_name().and_then(|name| name.to_str()), Some("unit.json"));
|
||||
assert_eq!(json["compared_ops"], 1);
|
||||
assert_eq!(json["reference_side"], "a");
|
||||
assert_eq!(json["version_a"], "rustfs 1.0.4");
|
||||
assert_eq!(json["version_b"], "rustfs 1.1.0");
|
||||
assert_eq!(
|
||||
json["differences"],
|
||||
serde_json::json!([{
|
||||
|
||||
@@ -14,10 +14,7 @@
|
||||
|
||||
//! Dual-stack differential: one `rustfs` binary, two processes, one SDK script (rustfs/backlog#2740).
|
||||
//!
|
||||
//! Stack A runs `RUSTFS_S3_STACK=legacy` of the binary under test, unless `RUSTFS_E2E_REFERENCE_BINARY`
|
||||
//! names a released binary: then A is that binary, started without `RUSTFS_S3_STACK` (older
|
||||
//! releases do not know it), and a bad path fails the run instead of falling back (see
|
||||
//! `reference`). The report names which side is the reference and the `--version` each side ran. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! Stack A always runs `RUSTFS_S3_STACK=legacy`. Stack B runs the value of `RUSTFS_E2E_STACK_B`
|
||||
//! (default `legacy` until the gateway stack is assembled), and `RUSTFS_E2E_STACK_B_REGION`, when
|
||||
//! set, starts B in another region so that a difference can be injected by hand. Both servers
|
||||
//! answer the script in `scenarios`, which covers every operation of `impl S3 for FS`; `compare`
|
||||
@@ -26,20 +23,16 @@
|
||||
//! is process-global, so two stacks in one process would read each other's buckets.
|
||||
|
||||
mod compare;
|
||||
mod reference;
|
||||
mod scenarios;
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path, workspace_root};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, workspace_root};
|
||||
use compare::{Difference, Observation, Report, compare, compared_cases, compared_ops, render};
|
||||
use reference::{SideA, binary_version};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::path::PathBuf;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const STACK_A: &str = "legacy";
|
||||
/// Label of side A in the report when it is the released reference binary.
|
||||
const STACK_A_REFERENCE: &str = "reference";
|
||||
const ENV_STACK_B: &str = "RUSTFS_E2E_STACK_B";
|
||||
const ENV_STACK_B_REGION: &str = "RUSTFS_E2E_STACK_B_REGION";
|
||||
/// Becomes `gateway` once rustfs/backlog#2734 T2.5 assembles the gateway stack.
|
||||
@@ -59,31 +52,22 @@ const CONTROL_REGION_B: &str = "eu-west-1";
|
||||
const MIN_NORMAL_SUCCESSES: usize = 68;
|
||||
|
||||
struct PairConfig {
|
||||
side_a: SideA,
|
||||
stack_b: String,
|
||||
region_a: Option<String>,
|
||||
region_b: Option<String>,
|
||||
}
|
||||
|
||||
impl PairConfig {
|
||||
fn from_env() -> Result<Self, BoxError> {
|
||||
Ok(Self {
|
||||
side_a: SideA::from_env()?,
|
||||
fn from_env() -> Self {
|
||||
Self {
|
||||
stack_b: std::env::var(ENV_STACK_B).unwrap_or_else(|_| DEFAULT_STACK_B.to_owned()),
|
||||
region_a: None,
|
||||
region_b: std::env::var(ENV_STACK_B_REGION).ok(),
|
||||
})
|
||||
}
|
||||
|
||||
fn stack_a_label(&self) -> &'static str {
|
||||
match self.side_a {
|
||||
SideA::SameBinary => STACK_A,
|
||||
SideA::Reference(_) => STACK_A_REFERENCE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Starts one server of the binary under test on `stack`, overriding its region only when asked.
|
||||
/// Starts one server on `stack`, overriding its region only when asked.
|
||||
async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = vec![("RUSTFS_S3_STACK", stack)];
|
||||
@@ -94,28 +78,12 @@ async fn start_stack(stack: &str, region: Option<&str>) -> Result<RustFSTestEnvi
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts one server of a released binary. `RUSTFS_S3_STACK` is not passed: older releases do not
|
||||
/// know it and have exactly one stack.
|
||||
async fn start_reference(binary: &Path, region: Option<&str>) -> Result<RustFSTestEnvironment, BoxError> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let mut vars = Vec::new();
|
||||
if let Some(region) = region {
|
||||
vars.push(("RUSTFS_REGION", region));
|
||||
}
|
||||
env.start_rustfs_server_from_binary_without_cleanup(binary, vec![], &vars)
|
||||
.await?;
|
||||
Ok(env)
|
||||
}
|
||||
|
||||
/// Starts stack A and stack B side by side, each with its own port and data directory.
|
||||
async fn spawn_pair(config: &PairConfig) -> Result<(RustFSTestEnvironment, RustFSTestEnvironment), BoxError> {
|
||||
let side_a = async {
|
||||
match &config.side_a {
|
||||
SideA::SameBinary => start_stack(STACK_A, config.region_a.as_deref()).await,
|
||||
SideA::Reference(binary) => start_reference(binary, config.region_a.as_deref()).await,
|
||||
}
|
||||
};
|
||||
tokio::try_join!(side_a, start_stack(&config.stack_b, config.region_b.as_deref()))
|
||||
tokio::try_join!(
|
||||
start_stack(STACK_A, config.region_a.as_deref()),
|
||||
start_stack(&config.stack_b, config.region_b.as_deref())
|
||||
)
|
||||
}
|
||||
|
||||
fn report_directory() -> PathBuf {
|
||||
@@ -136,34 +104,16 @@ struct Run {
|
||||
|
||||
/// Runs the script against both stacks concurrently and writes the report.
|
||||
async fn run_differential(label: &str, config: &PairConfig) -> Run {
|
||||
let binary_under_test = rustfs_binary_path();
|
||||
let (version_a, version_b) = match &config.side_a {
|
||||
SideA::SameBinary => {
|
||||
let version = binary_version(&binary_under_test)
|
||||
.await
|
||||
.expect("capture the binary under test version");
|
||||
(version.clone(), version)
|
||||
}
|
||||
SideA::Reference(binary) => (
|
||||
binary_version(binary).await.expect("capture the reference binary version"),
|
||||
binary_version(&binary_under_test)
|
||||
.await
|
||||
.expect("capture the binary under test version"),
|
||||
),
|
||||
};
|
||||
let (server_a, server_b) = spawn_pair(config).await.expect("start stack A and stack B");
|
||||
let (a, b) = tokio::join!(Box::pin(scenarios::run(&server_a)), Box::pin(scenarios::run(&server_b)));
|
||||
let millis = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map_or(0, |elapsed| elapsed.as_millis());
|
||||
let report = Report {
|
||||
schema: 2,
|
||||
schema: 1,
|
||||
run: format!("{label}-{millis}-{}", std::process::id()),
|
||||
stack_a: config.stack_a_label().to_owned(),
|
||||
stack_a: STACK_A.to_owned(),
|
||||
stack_b: config.stack_b.clone(),
|
||||
reference_side: matches!(config.side_a, SideA::Reference(_)).then_some("a"),
|
||||
version_a,
|
||||
version_b,
|
||||
region_a: config.region_a.clone(),
|
||||
region_b: config.region_b.clone(),
|
||||
compared_ops: compared_ops(&a, &b).len(),
|
||||
@@ -181,7 +131,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn both_stacks_answer_every_operation_identically() {
|
||||
init_logging();
|
||||
let config = PairConfig::from_env().expect("RUSTFS_E2E_REFERENCE_BINARY must be valid when set");
|
||||
let config = PairConfig::from_env();
|
||||
let run = run_differential("self-comparison", &config).await;
|
||||
let report = &run.report;
|
||||
|
||||
@@ -196,11 +146,9 @@ mod tests {
|
||||
assert_eq!(report.compared_ops, scenarios::OPERATIONS.len(), "report: {}", run.path.display());
|
||||
assert!(
|
||||
report.differences.is_empty(),
|
||||
"stack {} ({}) and stack {} ({}) differ on {} of {} cases (report: {}):\n{}",
|
||||
"stack {} and stack {} differ on {} of {} cases (report: {}):\n{}",
|
||||
report.stack_a,
|
||||
report.version_a,
|
||||
report.stack_b,
|
||||
report.version_b,
|
||||
report.differences.len(),
|
||||
report.compared_cases,
|
||||
run.path.display(),
|
||||
@@ -212,11 +160,7 @@ mod tests {
|
||||
.iter()
|
||||
.filter(|observation| observation.case == "negative" && !(400..500).contains(&observation.status))
|
||||
.collect();
|
||||
assert!(
|
||||
not_4xx.is_empty(),
|
||||
"negative cases must fail with a 4xx on side A ({}): {not_4xx:#?}",
|
||||
report.version_a
|
||||
);
|
||||
assert!(not_4xx.is_empty(), "negative cases must fail with a 4xx on {STACK_A}: {not_4xx:#?}");
|
||||
let successes = run
|
||||
.a
|
||||
.iter()
|
||||
@@ -224,8 +168,7 @@ mod tests {
|
||||
.count();
|
||||
assert!(
|
||||
successes >= MIN_NORMAL_SUCCESSES,
|
||||
"only {successes} normal cases succeeded on side A ({}): {:#?}",
|
||||
report.version_a,
|
||||
"only {successes} normal cases succeeded on {STACK_A}: {:#?}",
|
||||
run.a
|
||||
);
|
||||
}
|
||||
@@ -234,7 +177,6 @@ mod tests {
|
||||
async fn differential_detects_an_injected_difference() {
|
||||
init_logging();
|
||||
let config = PairConfig {
|
||||
side_a: SideA::SameBinary,
|
||||
stack_b: STACK_A.to_owned(),
|
||||
region_a: Some(CONTROL_REGION_A.to_owned()),
|
||||
region_b: Some(CONTROL_REGION_B.to_owned()),
|
||||
|
||||
@@ -1,163 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Which binary side A of the differential runs, and what it reports about itself.
|
||||
//!
|
||||
//! `RUSTFS_E2E_REFERENCE_BINARY` names a released `rustfs` binary to use as the reference. Unset,
|
||||
//! side A is the binary under test on `RUSTFS_S3_STACK=legacy`. Set, it must resolve to an
|
||||
//! executable file or the run fails: a reference that was asked for and silently replaced by the
|
||||
//! binary under test would report an agreement with itself as agreement with the release.
|
||||
|
||||
use std::ffi::OsString;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
pub(super) const ENV_REFERENCE_BINARY: &str = "RUSTFS_E2E_REFERENCE_BINARY";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(super) enum SideA {
|
||||
/// Both sides are the binary under test; A runs `RUSTFS_S3_STACK=legacy`.
|
||||
SameBinary,
|
||||
/// A is this released binary, started without `RUSTFS_S3_STACK`.
|
||||
Reference(PathBuf),
|
||||
}
|
||||
|
||||
impl SideA {
|
||||
pub(super) fn from_env() -> Result<Self, BoxError> {
|
||||
Self::resolve(std::env::var_os(ENV_REFERENCE_BINARY))
|
||||
}
|
||||
|
||||
/// `None` is the only value that selects same-binary mode; every other value must validate.
|
||||
pub(super) fn resolve(value: Option<OsString>) -> Result<Self, BoxError> {
|
||||
let Some(value) = value else {
|
||||
return Ok(Self::SameBinary);
|
||||
};
|
||||
let path = PathBuf::from(value);
|
||||
validate_executable(&path)?;
|
||||
Ok(Self::Reference(path))
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_executable(path: &Path) -> Result<(), BoxError> {
|
||||
if path.as_os_str().is_empty() {
|
||||
return Err(format!("{ENV_REFERENCE_BINARY} is set but empty").into());
|
||||
}
|
||||
let metadata = std::fs::metadata(path)
|
||||
.map_err(|error| format!("{ENV_REFERENCE_BINARY} points to {}, which cannot be read: {error}", path.display()))?;
|
||||
if !metadata.is_file() {
|
||||
return Err(format!("{ENV_REFERENCE_BINARY} points to {}, which is not a file", path.display()).into());
|
||||
}
|
||||
if !is_executable(&metadata) {
|
||||
return Err(format!("{ENV_REFERENCE_BINARY} points to {}, which is not executable", path.display()).into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn is_executable(metadata: &std::fs::Metadata) -> bool {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
metadata.permissions().mode() & 0o111 != 0
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn is_executable(_metadata: &std::fs::Metadata) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// What `<binary> --version` printed, trimmed. Fails when the binary does not run, exits
|
||||
/// non-zero, or prints nothing, because an empty version would report nothing as observed.
|
||||
pub(super) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
|
||||
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
|
||||
if !output.status.success() {
|
||||
return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
|
||||
}
|
||||
let version = String::from_utf8_lossy(&output.stdout).trim().to_owned();
|
||||
if version.is_empty() {
|
||||
return Err(format!("{} --version printed nothing", binary.display()).into());
|
||||
}
|
||||
Ok(version)
|
||||
}
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
fn script(dir: &Path, name: &str, body: &str, mode: u32) -> PathBuf {
|
||||
let path = dir.join(name);
|
||||
std::fs::write(&path, body).expect("write the stand-in");
|
||||
std::fs::set_permissions(&path, std::fs::Permissions::from_mode(mode)).expect("chmod");
|
||||
path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_path_is_an_error_not_a_fallback() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let error = SideA::resolve(Some(dir.path().join("absent").into_os_string())).expect_err("must fail closed");
|
||||
assert!(error.to_string().contains("cannot be read"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_executable_file_is_an_error() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = script(dir.path(), "rustfs", "#!/bin/sh\necho v\n", 0o644);
|
||||
let error = SideA::resolve(Some(path.into_os_string())).expect_err("must fail closed");
|
||||
assert!(error.to_string().contains("not executable"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn directory_is_an_error() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let error = SideA::resolve(Some(dir.path().as_os_str().to_owned())).expect_err("must fail closed");
|
||||
assert!(error.to_string().contains("not a file"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_value_is_an_error() {
|
||||
let error = SideA::resolve(Some(OsString::new())).expect_err("must fail closed");
|
||||
assert!(error.to_string().contains("empty"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unset_selects_same_binary_mode() {
|
||||
assert_eq!(SideA::resolve(None).expect("unset is valid"), SideA::SameBinary);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn executable_file_selects_reference_mode() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = script(dir.path(), "rustfs", "#!/bin/sh\necho v\n", 0o755);
|
||||
assert_eq!(
|
||||
SideA::resolve(Some(path.clone().into_os_string())).expect("valid"),
|
||||
SideA::Reference(path)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn version_output_is_captured_from_the_binary() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let path = script(dir.path(), "rustfs", "#!/bin/sh\necho 'rustfs 1.0.4 (abc123)'\n", 0o755);
|
||||
assert_eq!(binary_version(&path).await.expect("version"), "rustfs 1.0.4 (abc123)");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failing_or_silent_version_is_an_error() {
|
||||
let dir = tempfile::tempdir().expect("tempdir");
|
||||
let failing = script(dir.path(), "failing", "#!/bin/sh\nexit 3\n", 0o755);
|
||||
assert!(binary_version(&failing).await.is_err());
|
||||
let silent = script(dir.path(), "silent", "#!/bin/sh\nexit 0\n", 0o755);
|
||||
assert!(binary_version(&silent).await.is_err());
|
||||
}
|
||||
}
|
||||
@@ -586,18 +586,7 @@ async fn test_reads_under_a_deleted_kms_key_report_key_not_found() -> Result<(),
|
||||
.await?;
|
||||
assert_eq!(put.ssekms_key_id(), Some(doomed_key_id.as_str()));
|
||||
|
||||
// Destruction is two requests: schedule the deletion, then destroy the
|
||||
// pending key without waiting out its window.
|
||||
info!("🗑️ scheduling {doomed_key_id} for deletion, then destroying it");
|
||||
kms_admin_request(
|
||||
&base.url,
|
||||
http::Method::DELETE,
|
||||
"/rustfs/admin/v3/kms/keys/delete",
|
||||
Some(&serde_json::json!({ "key_id": doomed_key_id }).to_string()),
|
||||
&base.access_key,
|
||||
&base.secret_key,
|
||||
)
|
||||
.await?;
|
||||
info!("🗑️ deleting {doomed_key_id} immediately");
|
||||
kms_admin_request(
|
||||
&base.url,
|
||||
http::Method::DELETE,
|
||||
|
||||
@@ -21,15 +21,12 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, admin_create_user, init_logging};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, Delete, ObjectIdentifier, VersioningConfiguration};
|
||||
use std::error::Error;
|
||||
use tracing::info;
|
||||
|
||||
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
|
||||
|
||||
async fn create_bucket(client: &Client, bucket: &str) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
Ok(())
|
||||
@@ -93,7 +90,7 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Duplicate and repeated slashes after a leading slash collapse MinIO-style across single and bulk operations.
|
||||
/// Duplicate and repeated slashes after a leading slash collapse MinIO-style.
|
||||
#[tokio::test]
|
||||
async fn test_put_object_with_duplicate_slashes_normalized() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -147,258 +144,8 @@ mod tests {
|
||||
"GET after DELETE with raw key must return HTTP 404, got {error:?}"
|
||||
);
|
||||
|
||||
assert_bulk_delete_normalizes_keys(&client).await?;
|
||||
assert_bulk_delete_authorizes_normalized_keys(&env, &client).await?;
|
||||
assert_bulk_delete_preserves_version_identity(&client).await?;
|
||||
|
||||
env.stop_server();
|
||||
info!("Test completed successfully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_bulk_delete_normalizes_keys(client: &Client) -> TestResult {
|
||||
let bucket = "test-bulk-leading-slash";
|
||||
create_bucket(client, bucket).await?;
|
||||
let cases = [
|
||||
("/foo/bar", "foo/bar"),
|
||||
("//keyname", "keyname"),
|
||||
("/dir///sub//file", "dir/sub/file"),
|
||||
("/folder///", "folder/"),
|
||||
("/literal%2F+名", "literal%2F+名"),
|
||||
("plain", "plain"),
|
||||
];
|
||||
let invalid = ["///", "/dir/../keep", "dir//keep"];
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("keep")
|
||||
.body(ByteStream::from_static(b"keep"))
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("dir/keep")
|
||||
.body(ByteStream::from_static(b"keep"))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
for quiet in [false, true] {
|
||||
for (raw, _) in cases {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(raw)
|
||||
.body(ByteStream::from_static(b""))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
let listing = client.list_objects_v2().bucket(bucket).send().await?;
|
||||
let stored: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
|
||||
let mut expected: Vec<_> = cases
|
||||
.iter()
|
||||
.map(|(_, normalized)| *normalized)
|
||||
.chain(["dir/keep", "keep"])
|
||||
.collect();
|
||||
expected.sort_unstable();
|
||||
assert_eq!(stored, expected, "PUT must store the exact normalized targets before bulk deletion");
|
||||
let objects = cases
|
||||
.iter()
|
||||
.map(|(raw, _)| *raw)
|
||||
.chain(invalid)
|
||||
.map(|key| ObjectIdentifier::builder().key(key).build())
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let delete = Delete::builder().set_objects(Some(objects)).quiet(quiet).build()?;
|
||||
// A second identical batch also pins idempotent deletion of absent normalized keys.
|
||||
for _ in 0..2 {
|
||||
let output = client.delete_objects().bucket(bucket).delete(delete.clone()).send().await?;
|
||||
let errors: Vec<_> = output.errors().iter().map(|error| (error.key(), error.code())).collect();
|
||||
assert_eq!(
|
||||
errors,
|
||||
invalid.map(|key| (Some(key), Some("InvalidArgument"))),
|
||||
"unexpected per-key errors: {output:?}"
|
||||
);
|
||||
let deleted: Vec<_> = output.deleted().iter().filter_map(|object| object.key()).collect();
|
||||
let expected = if quiet {
|
||||
vec![]
|
||||
} else {
|
||||
cases.iter().map(|(raw, _)| *raw).collect()
|
||||
};
|
||||
assert_eq!(deleted, expected, "responses must retain request keys, including directory slashes");
|
||||
}
|
||||
let listing = client.list_objects_v2().bucket(bucket).send().await?;
|
||||
let remaining: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(
|
||||
remaining,
|
||||
["dir/keep", "keep"],
|
||||
"bulk delete must remove normalized objects without resolving dot segments"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_bulk_delete_authorizes_normalized_keys(env: &RustFSTestEnvironment, admin: &Client) -> TestResult {
|
||||
let bucket = "test-bulk-slash-policy";
|
||||
let user = "bulk-slash-user";
|
||||
admin_create_user(env, user, "bulkSlashTestSecret123!").await?;
|
||||
create_bucket(admin, bucket).await?;
|
||||
for key in ["allowed/item", "protected/item"] {
|
||||
admin
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from_static(b"keep"))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{"Effect": "Allow", "Principal": {"AWS": [user]}, "Action": ["s3:DeleteObject"],
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}/*")]},
|
||||
{"Effect": "Deny", "Principal": {"AWS": [user]}, "Action": ["s3:DeleteObject"],
|
||||
"Resource": [format!("arn:aws:s3:::{bucket}/protected/*")]}
|
||||
]
|
||||
});
|
||||
admin
|
||||
.put_bucket_policy()
|
||||
.bucket(bucket)
|
||||
.policy(policy.to_string())
|
||||
.send()
|
||||
.await?;
|
||||
let client = env.create_s3_client_with_credentials(user, "bulkSlashTestSecret123!");
|
||||
let output = client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key("/allowed///item").build()?)
|
||||
.objects(ObjectIdentifier::builder().key("/protected///item").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(output.deleted().len(), 1, "allowed normalized key must be deleted: {output:?}");
|
||||
assert_eq!(output.deleted()[0].key(), Some("/allowed///item"));
|
||||
assert_eq!(output.errors().len(), 1, "normalized resource denial must apply: {output:?}");
|
||||
assert_eq!(output.errors()[0].key(), Some("/protected///item"));
|
||||
assert_eq!(output.errors()[0].code(), Some("AccessDenied"));
|
||||
let listing = admin.list_objects_v2().bucket(bucket).send().await?;
|
||||
let remaining: Vec<_> = listing.contents().iter().filter_map(|object| object.key()).collect();
|
||||
assert_eq!(remaining, ["protected/item"], "only the authorized object may be removed");
|
||||
let protected = admin.get_object().bucket(bucket).key("protected/item").send().await?;
|
||||
assert_eq!(protected.body.collect().await?.into_bytes().as_ref(), b"keep");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_bulk_delete_preserves_version_identity(client: &Client) -> TestResult {
|
||||
let bucket = "test-bulk-slash-versions";
|
||||
let raw = "/history///file";
|
||||
let normalized = "history/file";
|
||||
create_bucket(client, bucket).await?;
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
VersioningConfiguration::builder()
|
||||
.status(BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
let first = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(raw)
|
||||
.body(ByteStream::from_static(b"old"))
|
||||
.send()
|
||||
.await?;
|
||||
let first_id = first.version_id().ok_or("first PUT omitted version ID")?;
|
||||
let latest = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(raw)
|
||||
.body(ByteStream::from_static(b"latest"))
|
||||
.send()
|
||||
.await?;
|
||||
let latest_id = latest.version_id().ok_or("latest PUT omitted version ID")?;
|
||||
let invalid = client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key("///").build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(invalid.deleted().is_empty(), "slash-only keys must not delete a versioned object");
|
||||
assert_eq!(
|
||||
invalid.errors().len(),
|
||||
1,
|
||||
"slash-only keys must remain invalid in versioned buckets: {invalid:?}"
|
||||
);
|
||||
assert_eq!(invalid.errors()[0].key(), Some("///"));
|
||||
assert_eq!(invalid.errors()[0].code(), Some("InvalidArgument"));
|
||||
let deleted = client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key(raw).version_id(first_id).build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(deleted.errors().is_empty(), "historical delete failed: {deleted:?}");
|
||||
assert_eq!(deleted.deleted().len(), 1);
|
||||
assert_eq!(deleted.deleted()[0].key(), Some(raw));
|
||||
assert_eq!(deleted.deleted()[0].version_id(), Some(first_id));
|
||||
let listing = client.list_object_versions().bucket(bucket).send().await?;
|
||||
assert_eq!(listing.versions().len(), 1, "only the latest data version should remain: {listing:?}");
|
||||
assert_eq!(listing.versions()[0].version_id(), Some(latest_id));
|
||||
|
||||
let deleted = client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key(raw).build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(deleted.errors().is_empty(), "delete marker creation failed: {deleted:?}");
|
||||
assert_eq!(deleted.deleted().len(), 1);
|
||||
assert_eq!(deleted.deleted()[0].key(), Some(raw));
|
||||
assert_eq!(deleted.deleted()[0].delete_marker(), Some(true));
|
||||
let marker = deleted.deleted()[0]
|
||||
.delete_marker_version_id()
|
||||
.ok_or("missing delete marker ID")?;
|
||||
let listing = client.list_object_versions().bucket(bucket).send().await?;
|
||||
assert_eq!(listing.delete_markers().len(), 1);
|
||||
assert_eq!(listing.delete_markers()[0].key(), Some(normalized));
|
||||
assert_eq!(listing.delete_markers()[0].version_id(), Some(marker));
|
||||
|
||||
let deleted = client
|
||||
.delete_objects()
|
||||
.bucket(bucket)
|
||||
.delete(
|
||||
Delete::builder()
|
||||
.objects(ObjectIdentifier::builder().key(raw).version_id(marker).build()?)
|
||||
.build()?,
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
assert!(deleted.errors().is_empty(), "explicit marker removal failed: {deleted:?}");
|
||||
assert_eq!(deleted.deleted().len(), 1);
|
||||
assert_eq!(deleted.deleted()[0].key(), Some(raw));
|
||||
assert_eq!(deleted.deleted()[0].version_id(), Some(marker));
|
||||
let listing = client.list_object_versions().bucket(bucket).send().await?;
|
||||
assert!(listing.delete_markers().is_empty());
|
||||
let restored = client.get_object().bucket(bucket).key(normalized).send().await?;
|
||||
assert_eq!(restored.version_id(), Some(latest_id));
|
||||
assert_eq!(restored.body.collect().await?.into_bytes().as_ref(), b"latest");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
use crate::bucket::metadata::BUCKET_METADATA_FILE;
|
||||
use crate::bucket::replication::ReplicationMigrationBridge;
|
||||
use crate::disk::{BUCKET_META_PREFIX, MIGRATING_META_BUCKET, RUSTFS_META_BUCKET};
|
||||
use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found, to_object_err};
|
||||
use crate::error::{Error, Result, is_err_strict_not_found, is_err_strict_volume_not_found};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketInfo, BucketOperations, BucketOptions},
|
||||
@@ -334,14 +334,6 @@ async fn migration_target_exists<S: EcstoreObjectOperations>(store: &S, path: &s
|
||||
}
|
||||
}
|
||||
|
||||
/// Names the target record in a failed migration write, as the existence
|
||||
/// probe's error already does. The commit path returns bare disk errors, and on
|
||||
/// Windows a blocked record is first seen here: reading through a regular-file
|
||||
/// ancestor reports the record missing instead of refusing access to it.
|
||||
fn migration_write_error(err: Error, path: &str) -> Error {
|
||||
to_object_err(err, vec![RUSTFS_META_BUCKET, path])
|
||||
}
|
||||
|
||||
async fn migrate_one_if_missing<S>(
|
||||
store: Arc<S>,
|
||||
opts: &ObjectOptions,
|
||||
@@ -377,10 +369,7 @@ where
|
||||
.unwrap_or(data);
|
||||
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
store
|
||||
.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts)
|
||||
.await
|
||||
.map_err(|err| migration_write_error(err, path))?;
|
||||
store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, opts).await?;
|
||||
info!("Migrated {label}");
|
||||
Ok(())
|
||||
}
|
||||
@@ -492,10 +481,7 @@ where
|
||||
Err(_) => return Err(MigrationMetadataError::Incompatible(path.to_owned()).into()),
|
||||
};
|
||||
let mut put_data = PutObjReader::from_vec(data);
|
||||
store
|
||||
.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts)
|
||||
.await
|
||||
.map_err(|err| migration_write_error(err, path))?;
|
||||
store.put_object(RUSTFS_META_BUCKET, path, &mut put_data, &opts).await?;
|
||||
info!("Migrated IAM config: {path}");
|
||||
total_migrated += 1;
|
||||
}
|
||||
@@ -545,27 +531,6 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_write_errors_name_the_target_record_without_fragmenting_groups() {
|
||||
use super::{Error, migration_write_error};
|
||||
|
||||
let first = migration_write_error(Error::FileAccessDenied, "buckets/first/.metadata.bin");
|
||||
let second = migration_write_error(Error::FileAccessDenied, "config/iam/format.json");
|
||||
assert!(matches!(first, Error::PrefixAccessDenied(..)), "unexpected error: {first:?}");
|
||||
assert!(
|
||||
first.to_string().contains(".rustfs.sys/buckets/first/.metadata.bin"),
|
||||
"the startup error must name the blocked record: {first}"
|
||||
);
|
||||
assert_eq!(first, second, "record paths must not fragment error grouping");
|
||||
|
||||
let unrelated = migration_write_error(Error::other("disk offline"), "buckets/first/.metadata.bin");
|
||||
assert_eq!(
|
||||
unrelated,
|
||||
Error::other("disk offline"),
|
||||
"errors without a record form pass through unchanged"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn migration_errors_group_by_cause_and_retain_typed_record_context() {
|
||||
use super::{Error, MigrationMetadataError};
|
||||
|
||||
@@ -41,12 +41,8 @@ use std::{
|
||||
future::Future,
|
||||
io::ErrorKind,
|
||||
pin::Pin,
|
||||
sync::{
|
||||
Arc, LazyLock, Mutex,
|
||||
atomic::{AtomicU8, Ordering},
|
||||
},
|
||||
sync::{LazyLock, Mutex},
|
||||
task::{Context, Poll},
|
||||
time::Duration,
|
||||
};
|
||||
use tonic::{service::interceptor::InterceptedService, transport::Channel};
|
||||
use tower::Service;
|
||||
@@ -74,7 +70,7 @@ pub(crate) async fn scanner_control_time_out_client(
|
||||
let channel = get_channel_for_class(addr, ChannelClass::Control)
|
||||
.await
|
||||
.map_err(|err| crate::error::Error::other(err.to_string()))?;
|
||||
let channel = ReplayScopeChannel::new(BatchLockDeadlineChannel::new(channel), interceptor.replay_scope_audience());
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
let limit = rustfs_protos::scoped_dirty_usage::SCOPED_DIRTY_USAGE_MAX_REQUEST_BYTES as usize;
|
||||
Ok(ScannerControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(limit)
|
||||
@@ -88,7 +84,7 @@ pub async fn heal_control_time_out_client(
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = get_channel_for_class(addr, ChannelClass::Control).await?;
|
||||
let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
|
||||
let channel = ReplayScopeChannel::new(BatchLockDeadlineChannel::new(channel), interceptor.replay_scope_audience());
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -101,7 +97,7 @@ pub async fn tier_mutation_control_time_out_client(
|
||||
let interceptor = interceptor.with_rpc_audience(addr)?;
|
||||
let channel = get_channel_for_class(addr, ChannelClass::Control).await?;
|
||||
let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
|
||||
let channel = ReplayScopeChannel::new(BatchLockDeadlineChannel::new(channel), interceptor.replay_scope_audience());
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -122,7 +118,7 @@ pub async fn node_service_time_out_client_for_class(
|
||||
let channel = get_channel_for_class(addr, class).await?;
|
||||
|
||||
let max_message_size = rustfs_protos::internode_rpc_max_message_size();
|
||||
let channel = ReplayScopeChannel::new(BatchLockDeadlineChannel::new(channel), interceptor.replay_scope_audience());
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
Ok(NodeServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
.max_encoding_message_size(max_message_size))
|
||||
@@ -137,7 +133,7 @@ pub(crate) async fn node_service_delete_recovery_client(
|
||||
let channel = rustfs_protos::create_channel_with_timeout(addr, timeout)
|
||||
.await
|
||||
.map_err(|error| DiskError::RemoteClientUnavailable(error.to_string()))?;
|
||||
let channel = ReplayScopeChannel::new(BatchLockDeadlineChannel::new(channel), interceptor.replay_scope_audience());
|
||||
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
|
||||
let max_message_size = rustfs_protos::internode_rpc_max_message_size();
|
||||
Ok(NodeServiceClient::with_interceptor(channel, interceptor)
|
||||
.max_decoding_message_size(max_message_size)
|
||||
@@ -253,95 +249,7 @@ pub struct ReplayScopeChannel<S> {
|
||||
}
|
||||
|
||||
/// The channel type used by internode clients after v2 authentication and replay-scope handling.
|
||||
pub type AuthenticatedChannel = ReplayScopeChannel<BatchLockDeadlineChannel<Channel>>;
|
||||
|
||||
/// Gate lock-batch payloads after the transport's internal readiness queues.
|
||||
/// Other RPC bodies pass through without an additional wrapper.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct BatchLockDeadlineChannel<S> {
|
||||
inner: S,
|
||||
}
|
||||
|
||||
impl<S> BatchLockDeadlineChannel<S> {
|
||||
fn new(inner: S) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Service<HttpRequest<tonic::body::Body>> for BatchLockDeadlineChannel<S>
|
||||
where
|
||||
S: Service<HttpRequest<tonic::body::Body>>,
|
||||
{
|
||||
type Response = S::Response;
|
||||
type Error = S::Error;
|
||||
type Future = S::Future;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, mut request: HttpRequest<tonic::body::Body>) -> Self::Future {
|
||||
if let Some(deadline) = request.extensions().get::<BatchLockRpcDeadline>().cloned() {
|
||||
let body = std::mem::replace(request.body_mut(), tonic::body::Body::empty());
|
||||
*request.body_mut() = tonic::body::Body::new(BatchLockDeadlineBody {
|
||||
inner: body,
|
||||
deadline,
|
||||
dispatched: false,
|
||||
failed: false,
|
||||
});
|
||||
}
|
||||
self.inner.call(request)
|
||||
}
|
||||
}
|
||||
|
||||
struct BatchLockDeadlineBody {
|
||||
inner: tonic::body::Body,
|
||||
deadline: BatchLockRpcDeadline,
|
||||
dispatched: bool,
|
||||
failed: bool,
|
||||
}
|
||||
|
||||
impl http_body::Body for BatchLockDeadlineBody {
|
||||
type Data = bytes::Bytes;
|
||||
type Error = tonic::Status;
|
||||
|
||||
fn poll_frame(
|
||||
mut self: Pin<&mut Self>,
|
||||
cx: &mut Context<'_>,
|
||||
) -> Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
|
||||
if self.failed {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
if !self.dispatched
|
||||
&& let Err(error) = self.deadline.remaining_before_dispatch()
|
||||
{
|
||||
self.failed = true;
|
||||
return Poll::Ready(Some(Err(error)));
|
||||
}
|
||||
let frame = std::task::ready!(Pin::new(&mut self.inner).poll_frame(cx));
|
||||
if let Some(Ok(frame)) = &frame
|
||||
&& frame.is_data()
|
||||
&& !self.dispatched
|
||||
{
|
||||
// Codec work can consume the deadline in this poll. Arbitrate only
|
||||
// after encoding, immediately before any payload leaves the body.
|
||||
if let Err(error) = self.deadline.mark_dispatched() {
|
||||
self.failed = true;
|
||||
return Poll::Ready(Some(Err(error)));
|
||||
}
|
||||
self.dispatched = true;
|
||||
}
|
||||
Poll::Ready(frame)
|
||||
}
|
||||
|
||||
fn is_end_stream(&self) -> bool {
|
||||
self.failed || self.inner.is_end_stream()
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> http_body::SizeHint {
|
||||
self.inner.size_hint()
|
||||
}
|
||||
}
|
||||
pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum PeerReplayCapability {
|
||||
@@ -518,9 +426,7 @@ where
|
||||
)
|
||||
}) {
|
||||
Ok(headers) => request.headers_mut().extend(headers),
|
||||
Err(error) => {
|
||||
debug!(error = %error, "could not attach replay-scoped RPC signature")
|
||||
}
|
||||
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -582,82 +488,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// A batch may leave local readiness queues only while its original budget is live.
|
||||
/// Cancellation and submission arbitrate once so a cancelled caller cannot send later.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct BatchLockRpcDeadline {
|
||||
deadline: tokio::time::Instant,
|
||||
max_acquire_wait: Duration,
|
||||
dispatch: Arc<AtomicU8>,
|
||||
}
|
||||
|
||||
impl BatchLockRpcDeadline {
|
||||
const PENDING: u8 = 0;
|
||||
const DISPATCHED: u8 = 1;
|
||||
const CANCELLED: u8 = 2;
|
||||
|
||||
pub(crate) fn new(deadline: tokio::time::Instant, max_acquire_wait: Duration) -> Self {
|
||||
Self {
|
||||
deadline,
|
||||
max_acquire_wait,
|
||||
dispatch: Arc::new(AtomicU8::new(Self::PENDING)),
|
||||
}
|
||||
}
|
||||
|
||||
/// True proves that this batch cannot send payload or enter the server lock handler.
|
||||
/// Transport headers may already have left an internal channel queue.
|
||||
pub(crate) fn cancel_before_dispatch(&self) -> bool {
|
||||
match self
|
||||
.dispatch
|
||||
.compare_exchange(Self::PENDING, Self::CANCELLED, Ordering::AcqRel, Ordering::Acquire)
|
||||
{
|
||||
Ok(_) => true,
|
||||
Err(state) => state == Self::CANCELLED,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn was_dispatched(&self) -> bool {
|
||||
self.dispatch.load(Ordering::Acquire) == Self::DISPATCHED
|
||||
}
|
||||
|
||||
fn remaining_before_dispatch(&self) -> Result<Duration, tonic::Status> {
|
||||
let remaining = self.deadline.saturating_duration_since(tokio::time::Instant::now());
|
||||
if remaining <= self.max_acquire_wait || self.dispatch.load(Ordering::Acquire) == Self::CANCELLED {
|
||||
return Err(tonic::Status::deadline_exceeded("Lock batch budget exhausted before payload dispatch"));
|
||||
}
|
||||
Ok(remaining)
|
||||
}
|
||||
|
||||
fn update_timeout(&self, request: &mut tonic::Request<()>) -> Result<(), tonic::Status> {
|
||||
request.set_timeout(self.remaining_before_dispatch()?);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mark_dispatched(&self) -> Result<(), tonic::Status> {
|
||||
self.remaining_before_dispatch()?;
|
||||
self.dispatch
|
||||
.compare_exchange(Self::PENDING, Self::DISPATCHED, Ordering::AcqRel, Ordering::Acquire)
|
||||
.map(|_| ())
|
||||
.map_err(|_| tonic::Status::deadline_exceeded("Lock batch cancelled before dispatch"))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) async fn dispatch_payload_for_test(&self) -> Result<(), tonic::Status> {
|
||||
use http_body_util::BodyExt as _;
|
||||
let mut body = BatchLockDeadlineBody {
|
||||
inner: tonic::body::Body::new(http_body_util::Full::new(bytes::Bytes::from_static(b"payload"))),
|
||||
deadline: self.clone(),
|
||||
dispatched: false,
|
||||
failed: false,
|
||||
};
|
||||
match body.frame().await {
|
||||
Some(Ok(frame)) if frame.is_data() => Ok(()),
|
||||
Some(Err(error)) => Err(error),
|
||||
_ => Err(tonic::Status::internal("test lock payload did not produce a data frame")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TonicSignatureInterceptor {
|
||||
audience: Option<String>,
|
||||
body_digest_strict: bool,
|
||||
@@ -752,21 +582,11 @@ impl TonicInterceptor {
|
||||
}
|
||||
|
||||
impl tonic::service::Interceptor for TonicInterceptor {
|
||||
fn call(&mut self, mut req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
|
||||
let batch_deadline = req.extensions().get::<BatchLockRpcDeadline>().cloned();
|
||||
if let Some(deadline) = &batch_deadline {
|
||||
deadline.update_timeout(&mut req)?;
|
||||
}
|
||||
let mut req = match self {
|
||||
fn call(&mut self, req: tonic::Request<()>) -> Result<tonic::Request<()>, tonic::Status> {
|
||||
match self {
|
||||
TonicInterceptor::Signature(interceptor) => interceptor.call(req),
|
||||
TonicInterceptor::NoOp(interceptor) => interceptor.call(req),
|
||||
}?;
|
||||
if let Some(deadline) = batch_deadline {
|
||||
// Signing consumes budget, but Channel readiness only reserves a
|
||||
// queue slot. The payload body arbitrates actual dispatch later.
|
||||
deadline.update_timeout(&mut req)?;
|
||||
}
|
||||
Ok(req)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -962,136 +782,6 @@ mod tests {
|
||||
test_request_for("Ping")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_dispatch_and_cancellation_have_one_winner() {
|
||||
for _ in 0..100 {
|
||||
let deadline = BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_secs(1), Duration::ZERO);
|
||||
let other = deadline.clone();
|
||||
let barrier = Arc::new(std::sync::Barrier::new(2));
|
||||
let other_barrier = barrier.clone();
|
||||
let submit = std::thread::spawn(move || {
|
||||
other_barrier.wait();
|
||||
other.mark_dispatched().is_ok()
|
||||
});
|
||||
barrier.wait();
|
||||
let cancelled = deadline.cancel_before_dispatch();
|
||||
let submitted = submit.join().expect("submission thread");
|
||||
assert_ne!(cancelled, submitted);
|
||||
assert_eq!(deadline.was_dispatched(), submitted);
|
||||
assert!(deadline.mark_dispatched().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_interceptor_rejects_exhausted_wait_reserve_before_submission() {
|
||||
let deadline =
|
||||
BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_millis(20), Duration::from_millis(30));
|
||||
let mut request = tonic::Request::new(());
|
||||
request.extensions_mut().insert(deadline.clone());
|
||||
let status = TonicInterceptor::NoOp(NoOpInterceptor)
|
||||
.call(request)
|
||||
.expect_err("insufficient wait reserve");
|
||||
assert_eq!(status.code(), tonic::Code::DeadlineExceeded);
|
||||
assert!(!deadline.was_dispatched());
|
||||
assert!(deadline.cancel_before_dispatch());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn batch_interceptor_rejects_cancelled_ready_request_and_accepts_live_budget() {
|
||||
let deadline = BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_secs(2), Duration::from_millis(1));
|
||||
let mut request = tonic::Request::new(());
|
||||
request.extensions_mut().insert(deadline.clone());
|
||||
assert!(deadline.cancel_before_dispatch());
|
||||
assert!(TonicInterceptor::NoOp(NoOpInterceptor).call(request).is_err());
|
||||
assert!(!deadline.was_dispatched());
|
||||
|
||||
let deadline = BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_secs(2), Duration::from_millis(1));
|
||||
let mut request = tonic::Request::new(());
|
||||
request.extensions_mut().insert(deadline.clone());
|
||||
let request = TonicInterceptor::NoOp(NoOpInterceptor).call(request).expect("live budget");
|
||||
assert!(!deadline.was_dispatched(), "interceptor completion does not bypass the transport queue");
|
||||
assert!(request.metadata().contains_key("grpc-timeout"));
|
||||
assert!(deadline.cancel_before_dispatch());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn batch_body_rejects_payload_after_channel_queue_deadline() {
|
||||
let deadline = BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_millis(20), Duration::ZERO);
|
||||
tokio::time::advance(Duration::from_millis(21)).await;
|
||||
let error = deadline
|
||||
.dispatch_payload_for_test()
|
||||
.await
|
||||
.expect_err("queued payload expired");
|
||||
assert_eq!(error.code(), tonic::Code::DeadlineExceeded);
|
||||
assert!(!deadline.was_dispatched());
|
||||
assert!(deadline.cancel_before_dispatch());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn batch_body_rejects_payload_when_encoding_crosses_deadline_in_one_poll() {
|
||||
use http_body_util::BodyExt as _;
|
||||
struct SlowEncodingBody {
|
||||
deadline: tokio::time::Instant,
|
||||
}
|
||||
impl http_body::Body for SlowEncodingBody {
|
||||
type Data = bytes::Bytes;
|
||||
type Error = tonic::Status;
|
||||
|
||||
fn poll_frame(
|
||||
self: Pin<&mut Self>,
|
||||
_cx: &mut Context<'_>,
|
||||
) -> Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
|
||||
// Model codec work that begins before the deadline but blocks
|
||||
// this single poll until the original budget has expired.
|
||||
tokio::time::resume();
|
||||
std::thread::sleep(
|
||||
self.deadline.saturating_duration_since(tokio::time::Instant::now()) + Duration::from_millis(1),
|
||||
);
|
||||
Poll::Ready(Some(Ok(http_body::Frame::data(bytes::Bytes::from_static(b"encoded payload")))))
|
||||
}
|
||||
}
|
||||
let expires = tokio::time::Instant::now() + Duration::from_millis(20);
|
||||
let deadline = BatchLockRpcDeadline::new(expires, Duration::ZERO);
|
||||
let mut body = BatchLockDeadlineBody {
|
||||
inner: tonic::body::Body::new(SlowEncodingBody { deadline: expires }),
|
||||
deadline: deadline.clone(),
|
||||
dispatched: false,
|
||||
failed: false,
|
||||
};
|
||||
let error = body
|
||||
.frame()
|
||||
.await
|
||||
.expect("first frame result")
|
||||
.expect_err("late encoded payload");
|
||||
assert_eq!(error.code(), tonic::Code::DeadlineExceeded);
|
||||
assert!(!deadline.was_dispatched());
|
||||
assert!(deadline.cancel_before_dispatch());
|
||||
assert!(body.frame().await.is_none(), "failed body must not expose a later frame");
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn batch_body_keeps_cleanup_responsibility_after_first_payload() {
|
||||
use http_body_util::BodyExt as _;
|
||||
let deadline = BatchLockRpcDeadline::new(tokio::time::Instant::now() + Duration::from_millis(20), Duration::ZERO);
|
||||
let frames = futures::stream::iter([
|
||||
Ok::<_, tonic::Status>(http_body::Frame::data(bytes::Bytes::from_static(b"first"))),
|
||||
Ok(http_body::Frame::data(bytes::Bytes::from_static(b"last"))),
|
||||
]);
|
||||
let mut body = BatchLockDeadlineBody {
|
||||
inner: tonic::body::Body::new(http_body_util::StreamBody::new(frames)),
|
||||
deadline: deadline.clone(),
|
||||
dispatched: false,
|
||||
failed: false,
|
||||
};
|
||||
assert!(body.frame().await.expect("first frame").expect("live payload").is_data());
|
||||
assert!(deadline.was_dispatched());
|
||||
assert!(!deadline.cancel_before_dispatch());
|
||||
tokio::time::advance(Duration::from_millis(21)).await;
|
||||
assert!(body.frame().await.expect("last frame").expect("late drain").is_data());
|
||||
assert!(body.frame().await.is_none());
|
||||
assert!(!deadline.cancel_before_dispatch(), "a sent payload requires late cleanup ownership");
|
||||
}
|
||||
|
||||
fn test_request_for(method: &'static str) -> tonic::Request<()> {
|
||||
let mut request = tonic::Request::new(());
|
||||
request
|
||||
|
||||
@@ -169,72 +169,6 @@ fn decode_delete_recovery_batch_errors(codes: Vec<Option<u32>>, expected_len: us
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn log_delete_recovery_batch_result(
|
||||
endpoint: &Endpoint,
|
||||
expected_len: usize,
|
||||
started: Instant,
|
||||
result: &Result<Vec<Option<DiskError>>>,
|
||||
) {
|
||||
let elapsed_ms = started.elapsed().as_millis();
|
||||
let mut disk_error_counts = std::collections::BTreeMap::<u32, usize>::new();
|
||||
let (failure_scope, rpc_status) = match result {
|
||||
Ok(errors) => {
|
||||
for error in errors.iter().flatten() {
|
||||
*disk_error_counts.entry(error.to_u32()).or_default() += 1;
|
||||
}
|
||||
("object_group", None)
|
||||
}
|
||||
Err(error) => {
|
||||
disk_error_counts.insert(error.to_u32(), expected_len);
|
||||
let rpc_status = match error {
|
||||
DiskError::Io(source) => crate::cluster::rpc::client::embedded_tonic_status(source).map(tonic::Status::code),
|
||||
_ => None,
|
||||
};
|
||||
("rpc_or_reply", rpc_status)
|
||||
}
|
||||
};
|
||||
let failed_group_count = disk_error_counts.values().sum::<usize>();
|
||||
let result = if result.is_err() || failed_group_count > 0 {
|
||||
"failed"
|
||||
} else {
|
||||
"ok"
|
||||
};
|
||||
if result == "failed" || elapsed_ms >= 1_000 {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
result,
|
||||
operation = "apply_versions",
|
||||
failure_scope,
|
||||
pool_index = endpoint.pool_idx,
|
||||
set_index = endpoint.set_idx,
|
||||
disk_index = endpoint.disk_idx,
|
||||
attempted_group_count = expected_len,
|
||||
failed_group_count,
|
||||
disk_error_counts = ?disk_error_counts,
|
||||
rpc_status = ?rpc_status,
|
||||
elapsed_ms,
|
||||
"Recoverable batch delete RPC completed"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
result,
|
||||
operation = "apply_versions",
|
||||
pool_index = endpoint.pool_idx,
|
||||
set_index = endpoint.set_idx,
|
||||
disk_index = endpoint.disk_idx,
|
||||
attempted_group_count = expected_len,
|
||||
failed_group_count,
|
||||
elapsed_ms,
|
||||
"Recoverable batch delete RPC completed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_delete_recovery_inspection(
|
||||
volume: &str,
|
||||
object: &str,
|
||||
@@ -997,16 +931,8 @@ pub struct RemoteDisk {
|
||||
#[cfg(test)]
|
||||
recovery_monitor_teardown_hook: Arc<tokio::sync::Mutex<Option<Arc<RecoveryMonitorTeardownHook>>>>,
|
||||
data_transport: Arc<dyn InternodeDataTransport>,
|
||||
/// Bound the per-operation failure warnings of this peer. An unreachable
|
||||
/// peer fails every in-flight shard operation; the counters keep the exact
|
||||
/// numbers per operation. Per handle, so one peer cannot hide another's.
|
||||
timeout_logs: rustfs_utils::LogThrottle,
|
||||
network_error_logs: rustfs_utils::LogThrottle,
|
||||
}
|
||||
|
||||
/// Shortest gap between two failure warnings of one kind from one remote disk.
|
||||
const REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS: u64 = 5_000;
|
||||
|
||||
struct RecoveryMonitorLease {
|
||||
active: Arc<AtomicBool>,
|
||||
}
|
||||
@@ -1438,8 +1364,6 @@ impl RemoteDisk {
|
||||
#[cfg(test)]
|
||||
recovery_monitor_teardown_hook: Arc::new(tokio::sync::Mutex::new(None)),
|
||||
data_transport,
|
||||
timeout_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
|
||||
network_error_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
|
||||
};
|
||||
record_drive_runtime_state(ep, RuntimeDriveHealthState::Online);
|
||||
|
||||
@@ -2166,20 +2090,17 @@ impl RemoteDisk {
|
||||
"op" => op.to_string()
|
||||
)
|
||||
.increment(1);
|
||||
if let Some(suppressed) = self.timeout_logs.claim() {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = timeout_duration.as_millis(),
|
||||
suppressed,
|
||||
state = "timeout",
|
||||
"Remote disk operation timed out"
|
||||
);
|
||||
}
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = timeout_duration.as_millis(),
|
||||
state = "timeout",
|
||||
"Remote disk operation timed out"
|
||||
);
|
||||
}
|
||||
|
||||
async fn handle_network_like_error<T>(
|
||||
@@ -2199,20 +2120,17 @@ impl RemoteDisk {
|
||||
"op" => op.to_string()
|
||||
)
|
||||
.increment(1);
|
||||
if let Some(suppressed) = self.network_error_logs.claim() {
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = timeout_duration.as_millis(),
|
||||
suppressed,
|
||||
state = "network_like_error",
|
||||
"Remote disk operation returned network-like error"
|
||||
);
|
||||
}
|
||||
warn!(
|
||||
event = EVENT_REMOTE_DISK_RPC,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REMOTE_DISK,
|
||||
endpoint = %self.endpoint,
|
||||
addr = %self.addr,
|
||||
op,
|
||||
timeout_ms = timeout_duration.as_millis(),
|
||||
state = "network_like_error",
|
||||
"Remote disk operation returned network-like error"
|
||||
);
|
||||
if failure_health_action == FailureHealthAction::MarkFailure {
|
||||
self.mark_faulty_and_evict("operation_network_error").await;
|
||||
}
|
||||
@@ -4455,7 +4373,6 @@ impl DiskAPI for RemoteDisk {
|
||||
opts: DeleteOptions,
|
||||
) -> Vec<Option<DiskError>> {
|
||||
let expected_len = versions.len();
|
||||
let started = Instant::now();
|
||||
let result = match self
|
||||
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::ApplyVersions { versions, opts })
|
||||
.await
|
||||
@@ -4464,7 +4381,6 @@ impl DiskAPI for RemoteDisk {
|
||||
Ok(_) => Err(Error::other("unexpected recoverable batch delete reply")),
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
log_delete_recovery_batch_result(&self.endpoint, expected_len, started, &result);
|
||||
match result {
|
||||
Ok(errors) => errors,
|
||||
Err(error) => vec![Some(error); expected_len],
|
||||
@@ -5757,62 +5673,6 @@ mod tests {
|
||||
assert!(decode_delete_recovery_batch_errors(vec![Some(u32::MAX)], 1).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recoverable_batch_delete_diagnostics_preserve_context_and_redact_error_details() {
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = Registry::default().with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.with_writer(logs.clone())
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.json()
|
||||
.flatten_event(true)
|
||||
.with_span_list(true),
|
||||
);
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
|
||||
let endpoint = Endpoint {
|
||||
url: url::Url::parse("http://127.0.0.1:9000/private-disk-path").expect("valid diagnostic endpoint"),
|
||||
is_local: false,
|
||||
pool_idx: 1,
|
||||
set_idx: 2,
|
||||
disk_idx: 3,
|
||||
};
|
||||
let rpc_failure = Err(DiskError::from(Status::deadline_exceeded("private-error-detail")));
|
||||
let group_failures = Ok(vec![None, Some(DiskError::FileNotFound), Some(DiskError::FileNotFound)]);
|
||||
tracing::info_span!("request-span", request_id = "req-batch-delete-diagnostics").in_scope(|| {
|
||||
log_delete_recovery_batch_result(&endpoint, 1000, Instant::now(), &rpc_failure);
|
||||
log_delete_recovery_batch_result(&endpoint, 3, Instant::now(), &group_failures);
|
||||
});
|
||||
let entries = logs.lines();
|
||||
assert_eq!(entries.len(), 2, "one summary per disk batch, never per object");
|
||||
assert_eq!(entries[0]["failure_scope"], "rpc_or_reply");
|
||||
assert_eq!(entries[0]["failed_group_count"], 1000);
|
||||
assert_eq!(entries[0]["rpc_status"], "Some(DeadlineExceeded)");
|
||||
assert_eq!(entries[1]["failure_scope"], "object_group");
|
||||
assert_eq!(entries[1]["failed_group_count"], 2);
|
||||
for entry in entries {
|
||||
assert_eq!(entry["pool_index"], 1);
|
||||
assert_eq!(entry["set_index"], 2);
|
||||
assert_eq!(entry["disk_index"], 3);
|
||||
assert!(
|
||||
entry["spans"]
|
||||
.as_array()
|
||||
.expect("request spans recorded")
|
||||
.iter()
|
||||
.any(|span| { span["request_id"] == "req-batch-delete-diagnostics" })
|
||||
);
|
||||
let text = entry.to_string();
|
||||
assert!(!text.contains("private-error-detail"));
|
||||
assert!(!text.contains("private-disk-path"));
|
||||
}
|
||||
assert!(rpc_failure.is_err(), "diagnostics borrow and preserve the original failure");
|
||||
assert_eq!(
|
||||
group_failures,
|
||||
Ok(vec![None, Some(DiskError::FileNotFound), Some(DiskError::FileNotFound)])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_recovery_inspection_rejects_wrong_transaction_object_and_protocol() {
|
||||
let transaction = Uuid::new_v4();
|
||||
@@ -11828,66 +11688,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// A peer that fails every operation logs one warning per interval, each
|
||||
/// operation still gets its own error, and another peer's first warning is
|
||||
/// not held back by the noisy one.
|
||||
#[tokio::test]
|
||||
async fn rpc_failure_warnings_are_bounded_per_remote_disk() {
|
||||
async fn remote_disk(port: u16) -> RemoteDisk {
|
||||
let url = url::Url::parse(&format!("http://127.0.0.1:{port}/data")).expect("endpoint url");
|
||||
let endpoint = Endpoint {
|
||||
url,
|
||||
is_local: false,
|
||||
pool_idx: 0,
|
||||
set_idx: 0,
|
||||
disk_idx: 0,
|
||||
};
|
||||
RemoteDisk::new(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
Arc::new(TcpHttpInternodeDataTransport),
|
||||
)
|
||||
.await
|
||||
.expect("remote disk handle")
|
||||
}
|
||||
let noisy = remote_disk(59981).await;
|
||||
let quiet = remote_disk(59982).await;
|
||||
|
||||
for _ in 0..50 {
|
||||
let error = noisy
|
||||
.execute_with_timeout_for_op_and_health_action(
|
||||
"read_all",
|
||||
|| async {
|
||||
Err::<(), Error>(DiskError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::ConnectionRefused,
|
||||
"connection refused",
|
||||
)))
|
||||
},
|
||||
Duration::from_secs(1),
|
||||
FailureHealthAction::IgnoreFailure,
|
||||
)
|
||||
.await
|
||||
.expect_err("every failing operation still fails");
|
||||
assert!(matches!(error, DiskError::Io(io) if io.kind() == std::io::ErrorKind::ConnectionRefused));
|
||||
noisy.record_timeout("read_all", Duration::from_secs(1));
|
||||
}
|
||||
|
||||
assert!(
|
||||
noisy.network_error_logs.claim().is_none(),
|
||||
"the burst used this peer's network-error window"
|
||||
);
|
||||
assert!(noisy.timeout_logs.claim().is_none(), "the burst used this peer's timeout window");
|
||||
assert_eq!(
|
||||
quiet.network_error_logs.claim(),
|
||||
Some(0),
|
||||
"another peer's first warning is not suppressed"
|
||||
);
|
||||
assert_eq!(quiet.timeout_logs.claim(), Some(0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_execute_with_timeout_keeps_remote_disk_online_for_business_error() {
|
||||
let addr = "http://127.0.0.1:59994".to_string();
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4325,28 +4325,20 @@ fn take_presynced_tmp_file(file: &Path) -> bool {
|
||||
PRESYNCED_TMP_FILES.remove(file).is_some()
|
||||
}
|
||||
|
||||
fn open_lazy_local_file_std(
|
||||
file_path: &Path,
|
||||
volume_dir: &Path,
|
||||
_reclaim_on_shutdown: bool,
|
||||
parent_exists: bool,
|
||||
) -> std::io::Result<std::fs::File> {
|
||||
let mut open_opts = std::fs::OpenOptions::new();
|
||||
open_opts.write(true).create(true).truncate(true);
|
||||
let file = match parent_exists.then(|| open_opts.open(file_path)) {
|
||||
Some(Ok(file)) => file,
|
||||
Some(Err(err)) if err.kind() != std::io::ErrorKind::NotFound => return Err(to_file_error(err)),
|
||||
_ => {
|
||||
if let Some(parent) = file_path.parent()
|
||||
&& (volume_dir.as_os_str().is_empty() || !volume_dir.starts_with(parent))
|
||||
&& let Err(err) = super::fs::create_dir_all_std(parent)
|
||||
&& err.kind() != std::io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(to_file_error(err));
|
||||
}
|
||||
open_opts.open(file_path).map_err(to_file_error)?
|
||||
}
|
||||
};
|
||||
fn open_lazy_local_file_std(file_path: &Path, volume_dir: &Path, _reclaim_on_shutdown: bool) -> std::io::Result<std::fs::File> {
|
||||
if let Some(parent) = file_path.parent()
|
||||
&& (volume_dir.as_os_str().is_empty() || !volume_dir.starts_with(parent))
|
||||
&& let Err(err) = super::fs::create_dir_all_std(parent)
|
||||
&& err.kind() != std::io::ErrorKind::AlreadyExists
|
||||
{
|
||||
return Err(to_file_error(err));
|
||||
}
|
||||
let file = std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.open(file_path)
|
||||
.map_err(to_file_error)?;
|
||||
#[cfg(target_os = "macos")]
|
||||
if _reclaim_on_shutdown {
|
||||
let _ = set_std_fd_nocache(&file);
|
||||
@@ -4361,7 +4353,6 @@ struct LazyLocalFileWriter {
|
||||
reclaim_on_shutdown: bool,
|
||||
sync_on_close: bool,
|
||||
mark_dir_presynced: bool,
|
||||
parent_exists: bool,
|
||||
bytes_written: usize,
|
||||
state: LazyLocalFileWriterState,
|
||||
}
|
||||
@@ -4382,7 +4373,6 @@ impl LazyLocalFileWriter {
|
||||
reclaim_on_shutdown: bool,
|
||||
sync_on_close: bool,
|
||||
mark_dir_presynced: bool,
|
||||
parent_exists: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
file_path,
|
||||
@@ -4391,7 +4381,6 @@ impl LazyLocalFileWriter {
|
||||
reclaim_on_shutdown,
|
||||
sync_on_close,
|
||||
mark_dir_presynced,
|
||||
parent_exists,
|
||||
bytes_written: 0,
|
||||
state: LazyLocalFileWriterState::Unopened,
|
||||
}
|
||||
@@ -4404,13 +4393,12 @@ impl LazyLocalFileWriter {
|
||||
let reclaim_on_shutdown = self.reclaim_on_shutdown;
|
||||
let sync_on_close = self.sync_on_close;
|
||||
let mark_dir_presynced = self.mark_dir_presynced;
|
||||
let parent_exists = self.parent_exists;
|
||||
match std::mem::replace(&mut self.state, LazyLocalFileWriterState::Closed) {
|
||||
LazyLocalFileWriterState::Unopened => {
|
||||
let file_path = self.file_path.clone();
|
||||
let volume_dir = self.volume_dir.clone();
|
||||
self.state = LazyLocalFileWriterState::Writing(tokio::task::spawn_blocking(move || {
|
||||
let mut file = open_lazy_local_file_std(&file_path, &volume_dir, reclaim_on_shutdown, parent_exists)?;
|
||||
let mut file = open_lazy_local_file_std(&file_path, &volume_dir, reclaim_on_shutdown)?;
|
||||
std::io::Write::write_all(&mut file, &payload).map_err(to_file_error)?;
|
||||
if close_after {
|
||||
if sync_on_close {
|
||||
@@ -4552,9 +4540,8 @@ impl AsyncWrite for LazyLocalFileWriter {
|
||||
let file_path = self.file_path.clone();
|
||||
let volume_dir = self.volume_dir.clone();
|
||||
let reclaim_on_shutdown = self.reclaim_on_shutdown;
|
||||
let parent_exists = self.parent_exists;
|
||||
self.state = LazyLocalFileWriterState::Closing(tokio::task::spawn_blocking(move || {
|
||||
open_lazy_local_file_std(&file_path, &volume_dir, reclaim_on_shutdown, parent_exists).map(drop)
|
||||
open_lazy_local_file_std(&file_path, &volume_dir, reclaim_on_shutdown).map(drop)
|
||||
}));
|
||||
}
|
||||
LazyLocalFileWriterState::Open(file) => {
|
||||
@@ -5606,14 +5593,11 @@ impl LocalIoBackend for StdBackend {
|
||||
let file_path = local_disk_object_path(self.io_root(), volume, path)?;
|
||||
check_path_length(file_path.to_string_lossy().as_ref())?;
|
||||
let mut mark_dir_presynced = false;
|
||||
let mut parent_exists = false;
|
||||
if let Some(parent) = file_path.parent() {
|
||||
check_path_length(parent.to_string_lossy().as_ref())?;
|
||||
parent_exists = parent == volume_dir;
|
||||
if volume == RUSTFS_META_TMP_BUCKET {
|
||||
unmark_presynced_tmp_file(&file_path);
|
||||
let parent_was_presynced = unmark_presynced_tmp_data_dir(parent);
|
||||
parent_exists |= parent_was_presynced;
|
||||
let file_name = file_path.file_name().and_then(|n| n.to_str()).unwrap_or_default();
|
||||
mark_dir_presynced = sync_on_close
|
||||
&& (file_name == "part.1"
|
||||
@@ -5667,7 +5651,6 @@ impl LocalIoBackend for StdBackend {
|
||||
reclaim_on_shutdown,
|
||||
sync_on_close,
|
||||
mark_dir_presynced,
|
||||
parent_exists,
|
||||
)))
|
||||
}
|
||||
WriteMode::Append => {
|
||||
@@ -11541,14 +11524,11 @@ impl LocalDisk {
|
||||
) -> Result<()> {
|
||||
multipart_staging::reject_reserved_anchor(path)?;
|
||||
let fuse_with_admission = admission.is_some() && !crash_inject::is_armed(CrashPoint::MetaWriteAfterTmpBeforeRename, path);
|
||||
// Held until the staging root is published, by the filesystem work
|
||||
// itself, so a concurrent listing cannot prune the marker in between.
|
||||
let mut index_pin = None;
|
||||
let prepared_index = if volume == RUSTFS_META_MULTIPART_BUCKET {
|
||||
if fuse_with_admission {
|
||||
self.multipart_index.prepare_record_head(path, &buf).await
|
||||
} else {
|
||||
index_pin = self.multipart_index.record_head(path, &buf).await?;
|
||||
self.multipart_index.record_head(path, &buf).await?;
|
||||
None
|
||||
}
|
||||
} else {
|
||||
@@ -11574,10 +11554,8 @@ impl LocalDisk {
|
||||
let root = self.publication_root.clone();
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&file_path, namespace_owner).await;
|
||||
os::run_blocking_namespace_operation_with_sync(lease, do_sync, move || {
|
||||
let _index_pin = prepared_index.map(|record| record.run_std()).transpose()?;
|
||||
#[cfg(test)]
|
||||
if let Some(pin) = &_index_pin {
|
||||
pin.pause_for_test();
|
||||
if let Some(record) = prepared_index {
|
||||
record.run_std()?;
|
||||
}
|
||||
super::fs::create_dir_all_std(&tmp_volume_dir)?;
|
||||
#[cfg(test)]
|
||||
@@ -11621,16 +11599,12 @@ impl LocalDisk {
|
||||
return Err(DiskError::Unexpected);
|
||||
}
|
||||
|
||||
let publication_owner: Option<Arc<dyn Send + Sync>> = match index_pin {
|
||||
Some(pin) => Some(Arc::new((namespace_owner.clone(), pin))),
|
||||
None => namespace_owner.clone(),
|
||||
};
|
||||
os::rename_all_with_precondition_and_parent_sync(
|
||||
&tmp_file_path,
|
||||
&file_path,
|
||||
&volume_dir,
|
||||
&self.publication_root,
|
||||
publication_owner,
|
||||
namespace_owner.clone(),
|
||||
admission.as_ref().map(|admission| admission.precondition()),
|
||||
admission.is_some() && sync && durability.syncs_commit_metadata(),
|
||||
)
|
||||
|
||||
@@ -286,15 +286,12 @@ pub(super) fn fsync_inline_commit_std(
|
||||
|
||||
if destination_was_absent {
|
||||
let mut ancestor = dst.parent().and_then(Path::parent);
|
||||
let mut ancestor_step = 0usize;
|
||||
while let Some(ancestor_dir) = ancestor {
|
||||
if !ancestor_dir.starts_with(dst_volume_dir) {
|
||||
break;
|
||||
}
|
||||
let fsync_started = rustfs_io_metrics::put_stage_timer();
|
||||
if let Err(err) =
|
||||
os::fsync_commit_ancestor_directory_limited_std(ancestor_dir, dst_volume_dir, ancestor_step, commit_guard)
|
||||
{
|
||||
if let Err(err) = os::fsync_dir_limited_std(ancestor_dir) {
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
|
||||
fsync_started,
|
||||
@@ -313,7 +310,6 @@ pub(super) fn fsync_inline_commit_std(
|
||||
break;
|
||||
}
|
||||
ancestor = ancestor_dir.parent();
|
||||
ancestor_step += 1;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -521,12 +517,8 @@ impl LocalDisk {
|
||||
state.preflight_rejection = Some(LocalRenamePreflightRejection(()));
|
||||
})?;
|
||||
crate::hp_guard!("LocalDisk::rename_data");
|
||||
if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET
|
||||
&& let Some(pin) = self.multipart_index.record_info(dst_path, &fi).await?
|
||||
{
|
||||
// The owner chain reaches every lease this rename publishes under,
|
||||
// so the index marker stays pinned until that work has returned.
|
||||
state.namespace_owner = Some(Arc::new((state.namespace_owner.take(), pin)));
|
||||
if dst_volume == super::super::RUSTFS_META_MULTIPART_BUCKET {
|
||||
self.multipart_index.record_info(dst_path, &fi).await?;
|
||||
}
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
if fi.is_healing() {
|
||||
|
||||
@@ -41,53 +41,19 @@ struct DiscoverySession {
|
||||
touched: Instant,
|
||||
}
|
||||
|
||||
/// Names an object-scoped scan skips without a stat (plain children of the
|
||||
/// key's directory, and entries of a UUID child that cannot be delete evidence)
|
||||
/// cost one budget entry per this many names: enough to bound a hostile prefix
|
||||
/// without making a real folder prefix undeletable.
|
||||
/// Plain (non-transaction) children of an object-scoped discovery root cost one
|
||||
/// budget entry per this many names: enough to bound a hostile prefix without
|
||||
/// making a real folder prefix undeletable.
|
||||
const PLAIN_CHILDREN_PER_BUDGET_ENTRY: u32 = 64;
|
||||
|
||||
/// Entries one step of the resumable (background) discovery may visit. The
|
||||
/// foreground bounded protocol has its own, larger per-call allowance.
|
||||
const DISCOVERY_STEP_ENTRIES: u32 = 4096;
|
||||
|
||||
/// Wall-clock allowance of one bounded discovery call: a third of the drive
|
||||
/// timeout that wraps the call (including its wait for a lane worker), capped at
|
||||
/// ten seconds, so an incomplete walk returns before the call can time out.
|
||||
fn bounded_discovery_time_limit() -> std::time::Duration {
|
||||
(crate::disk::disk_store::get_max_timeout_duration() / 3).min(std::time::Duration::from_secs(10))
|
||||
}
|
||||
|
||||
/// Whether an entry inside a UUID child of an object-scoped root can name one
|
||||
/// of the key's delete transactions: a journal file of a rollback directory or
|
||||
/// a data-directory marker. Every other name is inert to discovery.
|
||||
fn may_be_delete_evidence(name: &str) -> bool {
|
||||
[PREPARED, STORAGE_FORMAT_FILE_BACKUP, DELETE_QUORUM_COMMITTED_FILE].contains(&name)
|
||||
|| [
|
||||
DELETE_DATA_DIR_MARKER_PREFIX,
|
||||
RESERVED_DELETE_DATA_DIR_MARKER_PREFIX,
|
||||
RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX,
|
||||
]
|
||||
.iter()
|
||||
.any(|prefix| delete_transaction_marker_id(name, prefix).is_some())
|
||||
}
|
||||
|
||||
impl DiscoverySession {
|
||||
/// Consume one bounded step, returning ownership only when traversal remains.
|
||||
fn advance(self, limit: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>)> {
|
||||
self.advance_bounded(limit, DISCOVERY_STEP_ENTRIES, &|| false)
|
||||
self.advance_bounded(limit, super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)
|
||||
.map(|(transactions, next, _)| (transactions, next))
|
||||
}
|
||||
|
||||
/// Walk until `limit` transactions, `max_entries` charged entries, the end
|
||||
/// of the scope, or `should_stop` (checked before every entry); any early
|
||||
/// stop returns the remaining traversal as continuation state.
|
||||
fn advance_bounded(
|
||||
mut self,
|
||||
limit: u32,
|
||||
max_entries: u32,
|
||||
should_stop: &dyn Fn() -> bool,
|
||||
) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
|
||||
fn advance_bounded(mut self, limit: u32, max_entries: u32) -> std::io::Result<(Vec<DeleteTransactionId>, Option<Self>, u32)> {
|
||||
if self.frames.is_empty() {
|
||||
match std::fs::read_dir(&self.root) {
|
||||
Ok(entries) => self.frames.push((String::new(), entries)),
|
||||
@@ -100,11 +66,10 @@ impl DiscoverySession {
|
||||
let mut transactions = BTreeSet::new();
|
||||
let mut seen_candidates = HashSet::new();
|
||||
let mut visited = 0u32;
|
||||
let mut skipped_by_name = 0u32;
|
||||
let mut plain_children = 0u32;
|
||||
while !self.frames.is_empty()
|
||||
&& transactions.len() < usize::try_from(limit).map_err(|_| std::io::Error::from(ErrorKind::InvalidInput))?
|
||||
&& visited < max_entries
|
||||
&& !should_stop()
|
||||
{
|
||||
let (parent, entries) = self
|
||||
.frames
|
||||
@@ -125,22 +90,13 @@ impl DiscoverySession {
|
||||
return Err(std::io::Error::from(ErrorKind::InvalidData));
|
||||
}
|
||||
// An object-scoped scan only ever descends into transaction (UUID)
|
||||
// directories at the root, and inside them only journal files and
|
||||
// data-directory markers can be evidence. Every other name (plain
|
||||
// child keys of a prefix directory, and the `xl.meta`, data
|
||||
// directories or children of a UUID-named child key) is skipped by
|
||||
// name without a stat and charged at one entry per readdir batch, so
|
||||
// the work stays bounded while deleting a missing key that names a
|
||||
// folder prefix with many children, UUID-named or not, can complete.
|
||||
if self.object.is_some()
|
||||
&& if parent.is_empty() {
|
||||
!canonical_uuid(&name).is_some_and(|id| !id.is_nil())
|
||||
} else {
|
||||
!may_be_delete_evidence(&name)
|
||||
}
|
||||
{
|
||||
skipped_by_name += 1;
|
||||
if skipped_by_name.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
|
||||
// directories at the root. Plain child keys of a prefix directory are
|
||||
// skipped by name without a stat; they are charged at one entry per
|
||||
// readdir batch so the work stays bounded while deleting a missing key
|
||||
// that names a prefix with thousands of children can still complete.
|
||||
if self.object.is_some() && parent.is_empty() && !canonical_uuid(&name).is_some_and(|id| !id.is_nil()) {
|
||||
plain_children += 1;
|
||||
if plain_children.is_multiple_of(PLAIN_CHILDREN_PER_BUDGET_ENTRY) {
|
||||
visited += 1;
|
||||
}
|
||||
continue;
|
||||
@@ -1172,8 +1128,6 @@ struct PreparedDeleteReceiptEntry {
|
||||
journal_raw: Vec<u8>,
|
||||
journal: DeleteRecoveryJournal,
|
||||
sole_transaction: bool,
|
||||
had_old_data_dir: bool,
|
||||
rollback_dir_fd: Option<Arc<std::fs::File>>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -1183,8 +1137,6 @@ struct PublishedDeleteReceiptEntry {
|
||||
file_path: PathBuf,
|
||||
staging_root: PathBuf,
|
||||
journal_raw: Vec<u8>,
|
||||
had_old_data_dir: bool,
|
||||
rollback_dir_fd: Option<Arc<std::fs::File>>,
|
||||
}
|
||||
|
||||
const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
|
||||
@@ -1435,16 +1387,8 @@ impl LocalDisk {
|
||||
self.validate_delete_discovery_owner(&session)?;
|
||||
// The blocking owner drops every directory handle even if the RPC waiter disappears.
|
||||
// A partial inventory is never evidence of absence and never enters DISCOVERY_SESSIONS.
|
||||
// A large folder prefix may take a while to walk, so the probe rides the
|
||||
// durable-mutation lane like bounded inspection, never the one-worker
|
||||
// foreground lane that every other mutation on the drive queues behind.
|
||||
// It also stops, incomplete, when its waiter goes away or well before the
|
||||
// drive timeout, so a slow walk answers SlowDown instead of marking the
|
||||
// drive failed or holding a lane worker after the request is gone.
|
||||
let deadline = Instant::now() + bounded_discovery_time_limit();
|
||||
os::run_cancellable_blocking_namespace_operation_with_sync(lease, true, move |is_cancelled| {
|
||||
let should_stop = || is_cancelled() || Instant::now() >= deadline;
|
||||
let (transactions, next, visited) = session.advance_bounded(256, max_entries, &should_stop)?;
|
||||
os::run_blocking_namespace_operation(lease, move || {
|
||||
let (transactions, next, visited) = session.advance_bounded(256, max_entries)?;
|
||||
Ok(super::super::BoundedDeleteRecoveryDiscovery {
|
||||
transactions: if next.is_none() { transactions } else { Vec::new() },
|
||||
visited,
|
||||
@@ -2551,9 +2495,8 @@ impl LocalDisk {
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
type PreparedDeleteTuple =
|
||||
(Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool, bool, Option<bool>);
|
||||
let (old_dir, target, journal_raw, journal, sole_transaction, destination_missing, old_data_dir_state) =
|
||||
type PreparedDeleteTuple = (Option<Uuid>, Option<Vec<u8>>, Vec<u8>, DeleteRecoveryJournal, bool, bool, bool);
|
||||
let (old_dir, target, journal_raw, journal, sole_transaction, destination_missing, old_data_dir_present) =
|
||||
match (|| -> Result<Option<PreparedDeleteTuple>> {
|
||||
let mut meta = FileMeta::load(&buf)?;
|
||||
let mut deleted_headers: Vec<_> = meta
|
||||
@@ -2570,12 +2513,13 @@ impl LocalDisk {
|
||||
if old_dir.is_some_and(|dir| dir == transaction || dir.is_nil()) {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
let mut old_path_to_verify = None;
|
||||
if let Some(uuid) = old_dir {
|
||||
meta.data.remove(vec![fi_clone.version_id.unwrap_or_default(), uuid])?;
|
||||
let old_path = path_join(&[file_path.as_path(), Path::new(uuid.to_string().as_str())]);
|
||||
check_path_length(old_path.to_string_lossy().as_ref())?;
|
||||
old_path_to_verify = Some(old_path);
|
||||
if old_path.join(STORAGE_FORMAT_FILE).exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
let surviving: HashSet<_> = meta
|
||||
.versions
|
||||
@@ -2657,12 +2601,6 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(old_path) = old_path_to_verify
|
||||
&& (!scanned_file_path || old_data_dir_exists)
|
||||
&& old_path.join(STORAGE_FORMAT_FILE).exists()
|
||||
{
|
||||
return Ok(None);
|
||||
}
|
||||
sole_transaction &= predecessors.is_empty();
|
||||
let mut sparse = fi_clone;
|
||||
sparse.data = None;
|
||||
@@ -2688,21 +2626,15 @@ impl LocalDisk {
|
||||
journal,
|
||||
sole_transaction,
|
||||
scanned_file_path && !destination_exists,
|
||||
scanned_file_path.then_some(old_data_dir_exists),
|
||||
scanned_file_path && old_data_dir_exists,
|
||||
)))
|
||||
})() {
|
||||
Ok(Some(tuple)) => tuple,
|
||||
Ok(None) => return Ok(Ok(None)),
|
||||
Err(err) => return Ok(Err(err)),
|
||||
};
|
||||
let staged = match tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root) {
|
||||
Ok(dir) => dir,
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root)?
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
std::fs::create_dir_all(&staging_root)?;
|
||||
let staged = tempfile::Builder::new().prefix("delete-prep-").tempdir_in(&staging_root)?;
|
||||
let staged_dir = staged.path().to_path_buf();
|
||||
let backup_path = staged_dir.join(STORAGE_FORMAT_FILE_BACKUP);
|
||||
if std::fs::hard_link(&xl_path, &backup_path).is_err() {
|
||||
@@ -2726,12 +2658,8 @@ impl LocalDisk {
|
||||
for file in written_files {
|
||||
file.sync_data()?;
|
||||
}
|
||||
#[cfg(unix)]
|
||||
let rollback_dir_fd = sync.then(|| std::fs::File::open(&staged_dir).ok().map(Arc::new)).flatten();
|
||||
#[cfg(not(unix))]
|
||||
let rollback_dir_fd: Option<Arc<std::fs::File>> = None;
|
||||
if sync {
|
||||
os::fsync_directory_handle_std(&staged_dir, rollback_dir_fd.as_deref())?;
|
||||
os::fsync_dir_std(&staged_dir)?;
|
||||
}
|
||||
if !destination_missing {
|
||||
match std::fs::symlink_metadata(&destination) {
|
||||
@@ -2754,12 +2682,8 @@ impl LocalDisk {
|
||||
}
|
||||
let old_data_path = old_dir
|
||||
.map(|uuid| file_path.join(uuid.to_string()))
|
||||
.filter(|p| old_data_dir_state.unwrap_or_else(|| p.is_dir()));
|
||||
#[cfg(unix)]
|
||||
let file_path_dir = sync.then(|| std::fs::File::open(&file_path).ok()).flatten();
|
||||
#[cfg(not(unix))]
|
||||
let file_path_dir: Option<std::fs::File> = None;
|
||||
if sync && let Err(err) = os::fsync_directory_handle_std(&file_path, file_path_dir.as_ref()) {
|
||||
.filter(|p| old_data_dir_present || p.is_dir());
|
||||
if sync && let Err(err) = os::fsync_dir_std(&file_path) {
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
@@ -2768,15 +2692,11 @@ impl LocalDisk {
|
||||
let mut reserved_data_path = None;
|
||||
if let Some(old_path) = old_data_path {
|
||||
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
#[cfg(unix)]
|
||||
let old_dir_file = sync.then(|| std::fs::File::open(&old_path).ok()).flatten();
|
||||
#[cfg(not(unix))]
|
||||
let old_dir_file: Option<std::fs::File> = None;
|
||||
let reserve_res = (|| -> std::io::Result<()> {
|
||||
let marker = std::fs::File::create(&reserved_marker)?;
|
||||
if sync {
|
||||
marker.sync_all()?;
|
||||
os::fsync_directory_handle_std(&old_path, old_dir_file.as_ref())?;
|
||||
os::fsync_dir_std(&old_path)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
@@ -2785,7 +2705,7 @@ impl LocalDisk {
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
reserved_data_path = Some((old_path, reserved_marker, old_dir_file));
|
||||
reserved_data_path = Some((old_path, reserved_marker));
|
||||
}
|
||||
if let Some(ref target_bytes) = target {
|
||||
// A provisional head must stay outside the published recovery transaction.
|
||||
@@ -2817,7 +2737,7 @@ impl LocalDisk {
|
||||
})()
|
||||
};
|
||||
if let Err(err) = write_res {
|
||||
if let Some((_, reserved_marker, _)) = &reserved_data_path {
|
||||
if let Some((_, reserved_marker)) = &reserved_data_path {
|
||||
let _ = std::fs::remove_file(reserved_marker);
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
@@ -2826,20 +2746,19 @@ impl LocalDisk {
|
||||
} else if let Err(err) = std::fs::remove_file(&xl_path)
|
||||
&& err.kind() != ErrorKind::NotFound
|
||||
{
|
||||
if let Some((_, reserved_marker, _)) = &reserved_data_path {
|
||||
if let Some((_, reserved_marker)) = &reserved_data_path {
|
||||
let _ = std::fs::remove_file(reserved_marker);
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&destination);
|
||||
return Err(err);
|
||||
}
|
||||
if sync {
|
||||
os::fsync_directory_handle_std(&file_path, file_path_dir.as_ref())?;
|
||||
os::fsync_dir_std(&file_path)?;
|
||||
}
|
||||
let had_old_data_dir = reserved_data_path.is_some();
|
||||
if let Some((old_path, reserved_marker, old_dir_file)) = reserved_data_path {
|
||||
if let Some((old_path, reserved_marker)) = reserved_data_path {
|
||||
let recovery_marker = old_path.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
|
||||
match std::fs::rename(&reserved_marker, &recovery_marker) {
|
||||
Ok(()) if sync => os::fsync_directory_handle_std(&old_path, old_dir_file.as_ref())?,
|
||||
Ok(()) if sync => os::fsync_dir_std(&old_path)?,
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
@@ -2855,11 +2774,9 @@ impl LocalDisk {
|
||||
journal_raw,
|
||||
journal,
|
||||
sole_transaction,
|
||||
had_old_data_dir,
|
||||
rollback_dir_fd,
|
||||
},
|
||||
);
|
||||
Ok(Ok(Some(old_dir.filter(|_| had_old_data_dir))))
|
||||
Ok(Ok(Some(old_dir)))
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -2912,13 +2829,7 @@ impl LocalDisk {
|
||||
Some(entry.bucket_path),
|
||||
entry.file_path,
|
||||
entry.staging_root,
|
||||
Some((
|
||||
entry.journal_raw,
|
||||
entry.journal,
|
||||
entry.sole_transaction,
|
||||
entry.had_old_data_dir,
|
||||
entry.rollback_dir_fd,
|
||||
)),
|
||||
Some((entry.journal_raw, entry.journal, entry.sole_transaction)),
|
||||
),
|
||||
None => {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
@@ -2949,7 +2860,7 @@ impl LocalDisk {
|
||||
lease,
|
||||
sync,
|
||||
move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
|
||||
if let Some((journal_raw, journal, sole_transaction, had_old_data_dir, rollback_dir_fd)) = cached_receipt
|
||||
if let Some((journal_raw, journal, sole_transaction)) = cached_receipt
|
||||
&& rollback.join(PREPARED).is_file()
|
||||
{
|
||||
let tmp_receipt = staging_root.join(format!("receipt-{transaction}.tmp"));
|
||||
@@ -2963,7 +2874,7 @@ impl LocalDisk {
|
||||
os::record_native_publication(&receipt_path);
|
||||
std::fs::rename(&tmp_receipt, &receipt_path)?;
|
||||
if sync {
|
||||
os::fsync_directory_handle_std(&rollback, rollback_dir_fd.as_deref())?;
|
||||
os::fsync_dir_std(&rollback)?;
|
||||
}
|
||||
Ok(())
|
||||
})();
|
||||
@@ -2980,8 +2891,6 @@ impl LocalDisk {
|
||||
file_path: root,
|
||||
staging_root,
|
||||
journal_raw,
|
||||
had_old_data_dir,
|
||||
rollback_dir_fd,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -3118,34 +3027,29 @@ impl LocalDisk {
|
||||
None
|
||||
};
|
||||
let cached_published = rollback_open_path.as_deref().and_then(take_published_delete_receipt);
|
||||
let (object_path, bucket_path, staging_root, root, cached_journal_raw, cached_had_old_data_dir, mut rollback_dir_fd) =
|
||||
match cached_published {
|
||||
Some(entry) => (
|
||||
entry.object_path,
|
||||
entry.bucket_path,
|
||||
entry.staging_root,
|
||||
entry.file_path,
|
||||
Some(entry.journal_raw),
|
||||
Some(entry.had_old_data_dir),
|
||||
entry.rollback_dir_fd,
|
||||
),
|
||||
None => {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if !safe_path(object, false) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
let obj_path = self.get_object_path(volume, object)?;
|
||||
(
|
||||
obj_path,
|
||||
self.io_get_bucket_path(volume)?,
|
||||
self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?,
|
||||
self.io_get_object_path(volume, object)?,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
let (object_path, bucket_path, staging_root, root, cached_journal_raw) = match cached_published {
|
||||
Some(entry) => (
|
||||
entry.object_path,
|
||||
entry.bucket_path,
|
||||
entry.staging_root,
|
||||
entry.file_path,
|
||||
Some(entry.journal_raw),
|
||||
),
|
||||
None => {
|
||||
self.reject_multipart_delete_recovery_mutation(volume, object)?;
|
||||
if !safe_path(object, false) {
|
||||
return Err(DiskError::InvalidPath);
|
||||
}
|
||||
};
|
||||
let obj_path = self.get_object_path(volume, object)?;
|
||||
(
|
||||
obj_path,
|
||||
self.io_get_bucket_path(volume)?,
|
||||
self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?,
|
||||
self.io_get_object_path(volume, object)?,
|
||||
None,
|
||||
)
|
||||
}
|
||||
};
|
||||
let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
|
||||
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
|
||||
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
|
||||
@@ -3219,61 +3123,56 @@ impl LocalDisk {
|
||||
let recovery_committed = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let reserved = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
|
||||
let mut had_physical_data_dirs = false;
|
||||
let fast_cleanup = cached_journal_raw.is_some();
|
||||
if cached_had_old_data_dir != Some(false) {
|
||||
let bases: &[&Path] = if fast_cleanup { &[&root] } else { &[&root, &rollback] };
|
||||
for data_dir in &data_dirs {
|
||||
for base in bases {
|
||||
let dir_path = base.join(data_dir.to_string());
|
||||
match std::fs::read_dir(&dir_path) {
|
||||
Ok(entries) => {
|
||||
had_physical_data_dirs = true;
|
||||
let mut removable = Vec::new();
|
||||
let mut preserve_dir = false;
|
||||
for entry in recovery_entries(entries, work_budget.as_deref()) {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let Some(name) = name.to_str() else {
|
||||
preserve_dir = true;
|
||||
continue;
|
||||
};
|
||||
if entry.file_type()?.is_file()
|
||||
&& (delete_recovery_part_file(name)
|
||||
|| name == committed
|
||||
|| name == recovery_committed
|
||||
|| name == reserved
|
||||
|| is_delete_transaction_marker(name, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX))
|
||||
{
|
||||
removable.push(entry.path());
|
||||
} else {
|
||||
preserve_dir = true;
|
||||
}
|
||||
}
|
||||
for path in removable {
|
||||
ignore_not_found(std::fs::remove_file(path))?;
|
||||
}
|
||||
let mut dir_removed = false;
|
||||
if !preserve_dir {
|
||||
match std::fs::remove_dir(&dir_path) {
|
||||
Ok(()) => dir_removed = true,
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => dir_removed = true,
|
||||
Err(err) if is_dir_not_empty_error(&err) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
if !dir_removed && sync {
|
||||
os::fsync_dir_std(&dir_path)?;
|
||||
for data_dir in &data_dirs {
|
||||
for base in [&root, &rollback] {
|
||||
let dir_path = base.join(data_dir.to_string());
|
||||
match std::fs::read_dir(&dir_path) {
|
||||
Ok(entries) => {
|
||||
had_physical_data_dirs = true;
|
||||
let mut removable = Vec::new();
|
||||
let mut preserve_dir = false;
|
||||
for entry in recovery_entries(entries, work_budget.as_deref()) {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name();
|
||||
let Some(name) = name.to_str() else {
|
||||
preserve_dir = true;
|
||||
continue;
|
||||
};
|
||||
if entry.file_type()?.is_file()
|
||||
&& (delete_recovery_part_file(name)
|
||||
|| name == committed
|
||||
|| name == recovery_committed
|
||||
|| name == reserved
|
||||
|| is_delete_transaction_marker(name, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX))
|
||||
{
|
||||
removable.push(entry.path());
|
||||
} else {
|
||||
preserve_dir = true;
|
||||
}
|
||||
}
|
||||
for path in removable {
|
||||
ignore_not_found(std::fs::remove_file(path))?;
|
||||
}
|
||||
if sync {
|
||||
os::fsync_dir_std(&dir_path)?;
|
||||
}
|
||||
if !preserve_dir {
|
||||
match std::fs::remove_dir(&dir_path) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound || is_dir_not_empty_error(&err) => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
Err(err) if err.kind() == ErrorKind::NotFound => {}
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
}
|
||||
if had_physical_data_dirs && sync && root.is_dir() {
|
||||
os::fsync_dir_std(&root)?;
|
||||
}
|
||||
let fast_cleanup = cached_journal_raw.is_some();
|
||||
if fast_cleanup {
|
||||
let backup_or_marker = if has_original {
|
||||
Some(STORAGE_FORMAT_FILE_BACKUP)
|
||||
@@ -3322,15 +3221,12 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
}
|
||||
let rollback_sync = rollback_dir_fd.take();
|
||||
if sync && let Err(err) = os::fsync_directory_handle_std(&rollback, rollback_sync.as_deref()) {
|
||||
drop(rollback_sync);
|
||||
if sync && let Err(err) = os::fsync_dir_std(&rollback) {
|
||||
if has_target || had_physical_data_dirs {
|
||||
let _ = persist_receipt(true);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
drop(rollback_sync);
|
||||
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
|
||||
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &receipt_path);
|
||||
if let Err(err) = ignore_not_found(std::fs::remove_file(&receipt_path)) {
|
||||
@@ -6613,93 +6509,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_discovery_charges_uuid_named_children_once_and_keeps_evidence() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
|
||||
std::fs::create_dir_all(&path).expect("physical prefix");
|
||||
// UUID-named child keys of the folder: each holds its own `xl.meta` and a
|
||||
// data directory, neither of which can be evidence for the folder's key.
|
||||
const CHILDREN: u32 = 2000;
|
||||
for _ in 0..CHILDREN {
|
||||
let child = path.join(Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(child.join(Uuid::new_v4().to_string())).expect("child key data dir");
|
||||
std::fs::write(child.join(STORAGE_FORMAT_FILE), b"meta").expect("child key metadata");
|
||||
}
|
||||
// A UUID-named sub-prefix with children of its own.
|
||||
const SUB_PREFIX_CHILDREN: u32 = 640;
|
||||
let sub_prefix = path.join(Uuid::new_v4().to_string());
|
||||
for index in 0..SUB_PREFIX_CHILDREN {
|
||||
std::fs::create_dir_all(sub_prefix.join(format!("child-{index}"))).expect("sub-prefix child");
|
||||
}
|
||||
// Evidence of two interrupted transactions of the key itself: a journaled
|
||||
// rollback directory, and a data-directory marker whose rollback is gone.
|
||||
let journaled = Uuid::new_v4();
|
||||
std::fs::create_dir_all(path.join(journaled.to_string())).expect("rollback directory");
|
||||
std::fs::write(path.join(journaled.to_string()).join(PREPARED), b"{}").expect("journal");
|
||||
let marker_only = Uuid::new_v4();
|
||||
let data_dir = path.join(Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(&data_dir).expect("residual data directory");
|
||||
std::fs::write(data_dir.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{marker_only}")), [])
|
||||
.expect("reservation marker");
|
||||
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded(
|
||||
"recovery",
|
||||
"prefix",
|
||||
super::super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES,
|
||||
)
|
||||
.await
|
||||
.expect("bounded probe");
|
||||
assert!(page.complete, "a folder of UUID-named keys is a complete inventory");
|
||||
// One entry per descent (children, sub-prefix, rollback and data
|
||||
// directory) and per evidence file; inert names at 1/64.
|
||||
let inert = 2 * CHILDREN + SUB_PREFIX_CHILDREN;
|
||||
assert_eq!(page.visited, CHILDREN + 3 + 2 + inert / PLAIN_CHILDREN_PER_BUDGET_ENTRY);
|
||||
let found = page.transactions.iter().map(|id| id.transaction).collect::<BTreeSet<_>>();
|
||||
assert!(found.contains(&journaled), "a journaled rollback directory is evidence");
|
||||
assert!(found.contains(&marker_only), "a marker-only residue is evidence");
|
||||
assert_eq!(found.len(), 2, "no child key is mistaken for a transaction");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_discovery_stops_on_request_and_skips_only_inert_irregular_entries() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
let path = disk.delete_discovery_path("recovery", Some("prefix")).expect("prefix path");
|
||||
let child = path.join(Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(&child).expect("UUID-named child key");
|
||||
// A stop request (cancellation or the wall-clock limit) ends the walk
|
||||
// before any entry, leaving continuation state: never evidence of absence.
|
||||
let session = disk
|
||||
.new_delete_discovery_session("recovery", Some("prefix"), &path)
|
||||
.expect("private session");
|
||||
let (transactions, next, visited) = session
|
||||
.advance_bounded(256, super::super::super::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, &|| true)
|
||||
.expect("stopped walk");
|
||||
assert!(transactions.is_empty());
|
||||
assert_eq!(visited, 0);
|
||||
assert!(next.is_some(), "a stopped walk is incomplete");
|
||||
drop(next);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
// An irregular entry whose name cannot be evidence is skipped unread.
|
||||
std::os::unix::fs::symlink(&path, child.join("inert-link")).expect("inert symlink");
|
||||
let page = disk
|
||||
.discover_delete_transactions_bounded("recovery", "prefix", 4096)
|
||||
.await
|
||||
.expect("inert irregular entries are not evidence");
|
||||
assert!(page.complete && page.transactions.is_empty());
|
||||
// One with an evidence name still fails closed.
|
||||
std::os::unix::fs::symlink(&path, child.join(PREPARED)).expect("evidence-named symlink");
|
||||
assert!(
|
||||
disk.discover_delete_transactions_bounded("recovery", "prefix", 4096)
|
||||
.await
|
||||
.is_err(),
|
||||
"an evidence-named symlink is corruption, not a skipped name"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_bounded_discovery_refuses_partial_inventory_without_public_cursor() {
|
||||
let (_temp, disk, _, _) = fixture().await;
|
||||
@@ -6802,11 +6611,8 @@ mod tests {
|
||||
let filler_id = Uuid::new_v4().to_string();
|
||||
let filler = path.join(&filler_id);
|
||||
fs::create_dir_all(&filler).await.expect("non-evidence directory");
|
||||
// Inert names are skipped at 1/64, so fill the step with entries that
|
||||
// pass the evidence-name filter but are directories: each costs a full
|
||||
// budget entry and names no transaction.
|
||||
for _ in 0..5000 {
|
||||
fs::create_dir(filler.join(format!("{DELETE_DATA_DIR_MARKER_PREFIX}{}", Uuid::new_v4())))
|
||||
for index in 0..5000 {
|
||||
fs::write(filler.join(format!("part-{index:04}")), [])
|
||||
.await
|
||||
.expect("non-evidence entry");
|
||||
}
|
||||
|
||||
@@ -4,11 +4,6 @@
|
||||
//! Bucket-to-staging-root candidates: one marker per staging root, recorded
|
||||
//! before the root is published and pruned once the root is gone. Source
|
||||
//! metadata stays authoritative.
|
||||
//!
|
||||
//! A marker without a staging root is either a leftover or an upload whose
|
||||
//! root is still being published. The publisher proves the second case by
|
||||
//! holding a [`PublicationPin`] from before it records the marker until its
|
||||
//! filesystem work has returned; no elapsed time is taken as that proof.
|
||||
|
||||
use super::{LocalDisk, read_metadata_only_std};
|
||||
use crate::disk::{
|
||||
@@ -34,6 +29,9 @@ use uuid::Uuid;
|
||||
|
||||
const INDEX: &str = ".rustfs.sys/multipart-index-v1";
|
||||
const REBUILD_DIRENTS_PER_JOB: usize = 64;
|
||||
/// A marker is recorded before its staging root is published. A marker younger
|
||||
/// than this without a root is an upload in flight, never a leftover to prune.
|
||||
const MULTIPART_INDEX_PRUNE_GRACE: std::time::Duration = std::time::Duration::from_secs(300);
|
||||
type DirectoryEntries = Box<dyn Iterator<Item = io::Result<String>> + Send>;
|
||||
type PhysicalRoot = (u64, [u8; 16]);
|
||||
static OWNERS: LazyLock<parking_lot::Mutex<HashMap<PhysicalRoot, Weak<IndexOwner>>>> =
|
||||
@@ -58,15 +56,9 @@ pub(super) struct IndexOwner {
|
||||
// drop so a cancelled scan can be resumed by the next caller.
|
||||
driving: AtomicBool,
|
||||
disabled: AtomicBool,
|
||||
// Markers whose staging root some publisher is creating right now, with
|
||||
// the number of publishers. Boot-local like the index itself: every writer
|
||||
// of this physical drive in this process shares this owner.
|
||||
publishing: parking_lot::Mutex<HashMap<PathBuf, usize>>,
|
||||
#[cfg(test)]
|
||||
rebuild_head_pause: parking_lot::Mutex<Option<RebuildHeadPause>>,
|
||||
#[cfg(test)]
|
||||
publication_pause: parking_lot::Mutex<Option<RebuildHeadPause>>,
|
||||
#[cfg(test)]
|
||||
observed_rebuild_queue: parking_lot::Mutex<Option<Arc<os::DiskBlockingQueue>>>,
|
||||
#[cfg(test)]
|
||||
rebuild_single_job: AtomicBool,
|
||||
@@ -128,6 +120,23 @@ impl IndexDirectory {
|
||||
std::fs::File::open(self.path.join(name))
|
||||
}
|
||||
}
|
||||
fn modified(&self, name: &str) -> io::Result<std::time::SystemTime> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
self.open_file(name)?.metadata()?.modified()
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
std::fs::metadata(self.path.join(name))?.modified()
|
||||
}
|
||||
}
|
||||
/// Whether the marker is still inside the publication grace period. A
|
||||
/// marker dated in the future (clock stepped back) is never a leftover.
|
||||
fn marker_is_young(&self, name: &str) -> bool {
|
||||
self.modified(name)
|
||||
.ok()
|
||||
.is_some_and(|modified| modified.elapsed().map_or(true, |age| age < MULTIPART_INDEX_PRUNE_GRACE))
|
||||
}
|
||||
fn remove_file(&self, name: &str) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
@@ -183,10 +192,6 @@ impl RebuildCursor {
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
// The head is decoded; release the source pin before indexing. On
|
||||
// Windows the pin denies delete sharing, so holding it would make a
|
||||
// concurrent abort of this upload fail with a sharing violation.
|
||||
drop(source);
|
||||
let source_root = relative
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
@@ -275,12 +280,9 @@ impl IndexOwner {
|
||||
initialized: OnceLock::new(),
|
||||
driving: AtomicBool::new(false),
|
||||
disabled: AtomicBool::new(false),
|
||||
publishing: parking_lot::Mutex::new(HashMap::new()),
|
||||
#[cfg(test)]
|
||||
rebuild_head_pause: parking_lot::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
publication_pause: parking_lot::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
observed_rebuild_queue: parking_lot::Mutex::new(None),
|
||||
#[cfg(test)]
|
||||
rebuild_single_job: AtomicBool::new(false),
|
||||
@@ -392,8 +394,8 @@ impl IndexOwner {
|
||||
// The index is boot-local: every start rescans the staging area
|
||||
// before serving it, so a marker lost to a crash is recreated before
|
||||
// it can be consulted. Creation is enough; no fsync of the marker or
|
||||
// its ancestors. An existing marker needs nothing: the caller's
|
||||
// publication pin, not the marker's age, keeps it from being pruned.
|
||||
// its ancestors. An existing marker is touched so pruning sees the
|
||||
// new upload's age, not the previous one's.
|
||||
match openat(
|
||||
&parent,
|
||||
source_root,
|
||||
@@ -405,7 +407,16 @@ impl IndexOwner {
|
||||
return Err(io::Error::other("index marker is not a file"));
|
||||
}
|
||||
}
|
||||
Err(rustix::io::Errno::EXIST) => {}
|
||||
Err(rustix::io::Errno::EXIST) => {
|
||||
// Best effort: a filesystem that refuses to set times keeps
|
||||
// the previous upload's age, which only shortens the prune
|
||||
// grace for this key; index completeness does not depend on it.
|
||||
if let Ok(marker) =
|
||||
openat(&parent, source_root, OFlags::WRONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, Mode::empty())
|
||||
{
|
||||
let _ = std::fs::File::from(marker).set_modified(std::time::SystemTime::now());
|
||||
}
|
||||
}
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
@@ -421,22 +432,6 @@ impl IndexOwner {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Declare that the caller is about to publish the staging root of
|
||||
/// `marker`. Must precede the marker lease the caller records under, so a
|
||||
/// prune that found no pin has released that lease before the marker is
|
||||
/// (re)created.
|
||||
fn pin_publication(self: &Arc<Self>, marker: PathBuf) -> PublicationPin {
|
||||
*self.publishing.lock().entry(marker.clone()).or_default() += 1;
|
||||
PublicationPin {
|
||||
owner: Arc::clone(self),
|
||||
marker,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_publishing(&self, marker: &Path) -> bool {
|
||||
self.publishing.lock().contains_key(marker)
|
||||
}
|
||||
|
||||
fn ensure_current_root(&self) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
let current = {
|
||||
@@ -518,12 +513,9 @@ impl IndexOwner {
|
||||
}
|
||||
}
|
||||
|
||||
/// Record the marker of `path`'s staging root. The returned pin must stay
|
||||
/// alive until the filesystem work that publishes the root has returned,
|
||||
/// including work that outlives a cancelled caller.
|
||||
pub(super) async fn record_info(self: &Arc<Self>, path: &str, info: &FileInfo) -> Result<Option<PublicationPin>> {
|
||||
pub(super) async fn record_info(self: &Arc<Self>, path: &str, info: &FileInfo) -> Result<()> {
|
||||
if canonical_upload(path).is_none() {
|
||||
return Ok(None);
|
||||
return Ok(());
|
||||
}
|
||||
let owner = self.clone();
|
||||
let path = path.to_owned();
|
||||
@@ -539,7 +531,6 @@ impl IndexOwner {
|
||||
.join(path.split('/').next().unwrap_or_default())
|
||||
})
|
||||
.unwrap_or_else(|| owner.base.join(INDEX));
|
||||
let pin = self.pin_publication(marker.clone());
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
match owner.record(&path, &info) {
|
||||
@@ -553,7 +544,7 @@ impl IndexOwner {
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
Ok(Some(pin))
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Option<PreparedHeadRecord> {
|
||||
@@ -587,23 +578,20 @@ impl IndexOwner {
|
||||
.join(upload.split('/').next().unwrap_or_default())
|
||||
})
|
||||
.unwrap_or_else(|| self.base.join(INDEX));
|
||||
let pin = self.pin_publication(marker.clone());
|
||||
let lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
|
||||
Some(PreparedHeadRecord {
|
||||
owner: Arc::clone(self),
|
||||
upload: upload.to_owned(),
|
||||
info,
|
||||
pin,
|
||||
_lease: lease,
|
||||
})
|
||||
}
|
||||
|
||||
/// Record the marker of the staging root that `path` belongs to. See
|
||||
/// [`Self::record_info`] for how long the returned pin must live.
|
||||
pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<Option<PublicationPin>> {
|
||||
pub(super) async fn record_head(self: &Arc<Self>, path: &str, bytes: &[u8]) -> Result<()> {
|
||||
let Some(prepared) = self.prepare_record_head(path, bytes).await else {
|
||||
return Ok(None);
|
||||
return Ok(());
|
||||
};
|
||||
prepared.run().await.map(Some)
|
||||
prepared.run().await
|
||||
}
|
||||
|
||||
async fn page(self: &Arc<Self>, bucket: &str, marker: Option<&str>, disk_id: Uuid) -> Result<MultipartIndexPage> {
|
||||
@@ -654,20 +642,17 @@ impl IndexOwner {
|
||||
// to every key ever multipart-initiated on this drive.
|
||||
let mut live = Vec::with_capacity(roots.len());
|
||||
let mut stale = Vec::new();
|
||||
let markers = owner.base.join(INDEX).join(sha(&bucket));
|
||||
for root in roots {
|
||||
// The marker precedes its staging root: a pinned one is an
|
||||
// upload in flight and stays a candidate. The pin is read
|
||||
// before the root, so a publisher that finished in between has
|
||||
// already made its root visible.
|
||||
if owner.is_publishing(&markers.join(&root)) {
|
||||
live.push(root);
|
||||
continue;
|
||||
}
|
||||
match owner.directory(&Path::new(RUSTFS_META_MULTIPART_BUCKET).join(&root)) {
|
||||
Ok(_) => live.push(root),
|
||||
Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory) => {
|
||||
stale.push(root);
|
||||
// The marker precedes its staging root; a young one is an
|
||||
// upload in flight and stays a candidate.
|
||||
if directory.as_ref().is_some_and(|index| index.marker_is_young(&root)) {
|
||||
live.push(root);
|
||||
} else {
|
||||
stale.push(root);
|
||||
}
|
||||
}
|
||||
Err(error) => return Err(DiskError::from(error)),
|
||||
}
|
||||
@@ -692,11 +677,9 @@ impl IndexOwner {
|
||||
}
|
||||
|
||||
/// Remove one marker whose staging root was gone when a page examined it.
|
||||
/// The marker's own namespace lease serializes this with recording. Under
|
||||
/// that lease the publication pin is read first and the staging root
|
||||
/// second: a publisher pinned before the read keeps the marker, one that
|
||||
/// already finished has made its root visible, and one that pins later
|
||||
/// records its marker only after this lease is released.
|
||||
/// The marker's own namespace lease serializes this with `record_info`; the
|
||||
/// staging root and the marker's age are re-checked under that lease, so an
|
||||
/// upload recorded in between keeps its marker.
|
||||
async fn prune_marker(self: &Arc<Self>, bucket: &str, root: &str) -> Result<()> {
|
||||
let owner = self.clone();
|
||||
let bucket_directory = Path::new(INDEX).join(sha(bucket));
|
||||
@@ -707,9 +690,6 @@ impl IndexOwner {
|
||||
// not queue behind every other mutation on the one-worker foreground lane.
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || {
|
||||
owner.ensure_current_root()?;
|
||||
if owner.is_publishing(&marker) {
|
||||
return Ok(());
|
||||
}
|
||||
match owner.directory(&Path::new(RUSTFS_META_MULTIPART_BUCKET).join(&root)) {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory) => {}
|
||||
@@ -720,6 +700,9 @@ impl IndexOwner {
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if directory.marker_is_young(&root) {
|
||||
return Ok(());
|
||||
}
|
||||
match directory.remove_file(&root) {
|
||||
Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error),
|
||||
_ => Ok(()),
|
||||
@@ -730,69 +713,34 @@ impl IndexOwner {
|
||||
}
|
||||
}
|
||||
|
||||
/// Proof that the staging root of one marker is being published. While any pin
|
||||
/// for a marker is alive, pruning leaves that marker alone.
|
||||
#[must_use = "dropping the pin lets a concurrent listing prune the marker before its staging root exists"]
|
||||
pub(super) struct PublicationPin {
|
||||
owner: Arc<IndexOwner>,
|
||||
marker: PathBuf,
|
||||
}
|
||||
|
||||
impl PublicationPin {
|
||||
/// Test seam between recording the marker and publishing the staging root.
|
||||
#[cfg(test)]
|
||||
pub(super) fn pause_for_test(&self) {
|
||||
let pause = self.owner.publication_pause.lock().take();
|
||||
if let Some(pause) = pause {
|
||||
let _pause = os::enter_blocking_test_hook();
|
||||
let _ = pause.started.send(());
|
||||
let _ = pause.release.recv_timeout(std::time::Duration::from_secs(10));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PublicationPin {
|
||||
fn drop(&mut self) {
|
||||
let mut publishing = self.owner.publishing.lock();
|
||||
if let Some(count) = publishing.get_mut(&self.marker) {
|
||||
*count -= 1;
|
||||
if *count == 0 {
|
||||
publishing.remove(&self.marker);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct PreparedHeadRecord {
|
||||
owner: Arc<IndexOwner>,
|
||||
upload: String,
|
||||
info: FileInfo,
|
||||
pin: PublicationPin,
|
||||
_lease: Arc<os::NamespaceMutationLease>,
|
||||
}
|
||||
|
||||
impl PreparedHeadRecord {
|
||||
/// Record the marker and release its lease. The caller publishes the
|
||||
/// staging root while it holds the returned pin.
|
||||
pub(super) fn run_std(self) -> io::Result<PublicationPin> {
|
||||
pub(super) fn run_std(self) -> io::Result<()> {
|
||||
let Self {
|
||||
owner,
|
||||
upload,
|
||||
info,
|
||||
pin,
|
||||
_lease,
|
||||
} = self;
|
||||
match pin.owner.record(&upload, &info) {
|
||||
match owner.record(&upload, &info) {
|
||||
Err(error) if error.kind() == io::ErrorKind::InvalidData => {
|
||||
pin.owner.disabled.store(true, Ordering::Release);
|
||||
Ok(pin)
|
||||
owner.disabled.store(true, Ordering::Release);
|
||||
Ok(())
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
Ok(()) => Ok(pin),
|
||||
result => result,
|
||||
}
|
||||
}
|
||||
|
||||
async fn run(self) -> Result<PublicationPin> {
|
||||
async fn run(self) -> Result<()> {
|
||||
let lease = Arc::clone(&self._lease);
|
||||
Ok(os::run_blocking_namespace_operation_with_sync(lease, true, move || self.run_std()).await?)
|
||||
os::run_blocking_namespace_operation_with_sync(lease, true, move || self.run_std()).await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1010,274 +958,47 @@ mod tests {
|
||||
let live = format!("{:064x}", 1);
|
||||
let finished = format!("{:064x}", 2);
|
||||
let in_flight = format!("{:064x}", 3);
|
||||
for name in [&live, &finished, &in_flight] {
|
||||
let clock_stepped = format!("{:064x}", 4);
|
||||
for name in [&live, &finished, &in_flight, &clock_stepped] {
|
||||
std::fs::write(directory.join(name), b"1").expect("marker");
|
||||
}
|
||||
// Completion and abort remove the staging root and leave the marker.
|
||||
std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&live)).expect("live staging root");
|
||||
let owner = &disk.multipart_index;
|
||||
let publishing = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&in_flight));
|
||||
let second_publisher = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&in_flight));
|
||||
let page = owner.page("bucket", None, Uuid::new_v4()).await.expect("page");
|
||||
// The in-flight marker has no staging root yet but is pinned: a candidate, not a leftover.
|
||||
assert_eq!(page.roots, vec![live.clone(), in_flight.clone()]);
|
||||
set_marker_modified(&directory.join(&finished), std::time::SystemTime::now() - MULTIPART_INDEX_PRUNE_GRACE * 2);
|
||||
// A marker dated after a backwards clock step is never a leftover.
|
||||
set_marker_modified(
|
||||
&directory.join(&clock_stepped),
|
||||
std::time::SystemTime::now() + MULTIPART_INDEX_PRUNE_GRACE * 2,
|
||||
);
|
||||
let page = disk.multipart_index.page("bucket", None, Uuid::new_v4()).await.expect("page");
|
||||
// The in-flight marker has no staging root yet but is young: a candidate, not a leftover.
|
||||
assert_eq!(page.roots, vec![live.clone(), in_flight.clone(), clock_stepped.clone()]);
|
||||
assert!(page.next_marker.is_none());
|
||||
assert!(directory.join(&live).exists(), "a live root keeps its marker");
|
||||
assert!(directory.join(&in_flight).exists(), "a pinned marker is kept");
|
||||
assert!(!directory.join(&finished).exists(), "a finished root's marker is pruned at once");
|
||||
// One of two publishers of the same root finishing does not unpin it.
|
||||
drop(publishing);
|
||||
let page = owner
|
||||
assert!(directory.join(&in_flight).exists(), "a marker inside the grace period is kept");
|
||||
assert!(directory.join(&clock_stepped).exists(), "a future-dated marker is kept");
|
||||
assert!(!directory.join(&finished).exists(), "a finished root's marker is pruned");
|
||||
// The next page no longer pays for the pruned marker.
|
||||
let page = disk
|
||||
.multipart_index
|
||||
.page("bucket", None, Uuid::new_v4())
|
||||
.await
|
||||
.expect("page with one publisher left");
|
||||
assert_eq!(page.roots, vec![live.clone(), in_flight.clone()]);
|
||||
assert!(directory.join(&in_flight).exists(), "the remaining publisher keeps the marker");
|
||||
// A publication that ended without a staging root leaves a leftover.
|
||||
drop(second_publisher);
|
||||
let page = owner
|
||||
.page("bucket", None, Uuid::new_v4())
|
||||
.await
|
||||
.expect("page after the failed publication");
|
||||
assert_eq!(page.roots, vec![live]);
|
||||
assert!(!directory.join(&in_flight).exists(), "an unpinned marker without a root is pruned");
|
||||
assert!(owner.publishing.lock().is_empty(), "finished publishers leave no pin behind");
|
||||
.expect("page after pruning");
|
||||
assert_eq!(page.roots, vec![live, in_flight, clock_stepped]);
|
||||
}
|
||||
|
||||
fn pause_next_publication(owner: &IndexOwner) -> (tokio::sync::oneshot::Receiver<()>, std::sync::mpsc::Sender<()>) {
|
||||
let (started, waiting) = tokio::sync::oneshot::channel();
|
||||
let (release, released) = std::sync::mpsc::channel();
|
||||
*owner.publication_pause.lock() = Some(RebuildHeadPause {
|
||||
started,
|
||||
release: released,
|
||||
});
|
||||
(waiting, release)
|
||||
/// Move a marker's modification time outside the publication grace period.
|
||||
fn age_marker(marker: &Path) {
|
||||
set_marker_modified(marker, std::time::SystemTime::now() - MULTIPART_INDEX_PRUNE_GRACE * 2);
|
||||
}
|
||||
|
||||
/// The window the pin closes: the marker is recorded and its lease is
|
||||
/// released, the staging root is not published yet, and a listing examines
|
||||
/// the bucket. The marker is dated a day back so that no grace period could
|
||||
/// be what keeps it.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_index_listing_during_source_publication_keeps_the_new_upload_listed() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = Arc::new(open(root.path()).await);
|
||||
let (path, info) = upload("bucket", "published-while-listed");
|
||||
let source_root = path.split('/').next().expect("source sha").to_owned();
|
||||
let marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
|
||||
let source_head = root
|
||||
.path()
|
||||
.join(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&path)
|
||||
.join(STORAGE_FORMAT_FILE);
|
||||
let (paused, release) = pause_next_publication(&disk.multipart_index);
|
||||
let writer = tokio::spawn({
|
||||
let disk = Arc::clone(&disk);
|
||||
let path = path.clone();
|
||||
async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), paused)
|
||||
.await
|
||||
.expect("the publication reaches the pause")
|
||||
.expect("pause sender");
|
||||
assert!(marker.exists(), "the marker precedes the staging root");
|
||||
assert!(!source_head.exists(), "the staging root is not published yet");
|
||||
fn set_marker_modified(marker: &Path, modified: std::time::SystemTime) {
|
||||
std::fs::File::options()
|
||||
.write(true)
|
||||
.open(&marker)
|
||||
.open(marker)
|
||||
.expect("marker")
|
||||
.set_modified(std::time::SystemTime::now() - std::time::Duration::from_secs(86_400))
|
||||
.expect("age the marker");
|
||||
|
||||
let identity = Uuid::new_v4();
|
||||
let page = disk
|
||||
.multipart_index
|
||||
.page("bucket", None, identity)
|
||||
.await
|
||||
.expect("page during publication");
|
||||
assert_eq!(page.roots, vec![source_root.clone()], "an upload in flight stays a candidate");
|
||||
assert!(marker.exists(), "a listing must not prune the marker of an upload in flight");
|
||||
|
||||
release.send(()).expect("resume the publication");
|
||||
writer.await.expect("writer task").expect("the upload metadata is published");
|
||||
assert!(source_head.exists());
|
||||
let page = disk
|
||||
.multipart_index
|
||||
.page("bucket", None, identity)
|
||||
.await
|
||||
.expect("page after publication");
|
||||
assert_eq!(page.roots, vec![source_root], "the published upload is listed");
|
||||
assert!(marker.exists());
|
||||
assert!(disk.multipart_index.publishing.lock().is_empty());
|
||||
}
|
||||
|
||||
/// A timed-out caller drops its future while the filesystem work goes on.
|
||||
/// The pin belongs to that work, so the marker survives until the staging
|
||||
/// root the detached work publishes is visible.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_index_cancelled_publisher_keeps_its_marker_until_the_detached_work_returns() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = Arc::new(open(root.path()).await);
|
||||
let (path, info) = upload("bucket", "caller-timed-out");
|
||||
let source_root = path.split('/').next().expect("source sha").to_owned();
|
||||
let marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
|
||||
let source_head = root
|
||||
.path()
|
||||
.join(RUSTFS_META_MULTIPART_BUCKET)
|
||||
.join(&path)
|
||||
.join(STORAGE_FORMAT_FILE);
|
||||
let (paused, release) = pause_next_publication(&disk.multipart_index);
|
||||
let writer = tokio::spawn({
|
||||
let disk = Arc::clone(&disk);
|
||||
let path = path.clone();
|
||||
async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), paused)
|
||||
.await
|
||||
.expect("the publication reaches the pause")
|
||||
.expect("pause sender");
|
||||
writer.abort();
|
||||
assert!(writer.await.expect_err("the caller is gone").is_cancelled());
|
||||
|
||||
let identity = Uuid::new_v4();
|
||||
let page = disk
|
||||
.multipart_index
|
||||
.page("bucket", None, identity)
|
||||
.await
|
||||
.expect("page after the caller left");
|
||||
assert_eq!(page.roots, vec![source_root.clone()]);
|
||||
assert!(marker.exists(), "the detached publication still owns the marker");
|
||||
|
||||
release.send(()).expect("resume the detached publication");
|
||||
tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
while !source_head.exists() || !disk.multipart_index.publishing.lock().is_empty() {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(5)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the detached publication completes");
|
||||
let page = disk
|
||||
.multipart_index
|
||||
.page("bucket", None, identity)
|
||||
.await
|
||||
.expect("page after the detached publication");
|
||||
assert_eq!(page.roots, vec![source_root], "the upload the detached work published is listed");
|
||||
assert!(marker.exists());
|
||||
}
|
||||
|
||||
/// A page classifies a marker as stale before the publisher pins it; the
|
||||
/// prune that follows must read the pin again under the marker lease.
|
||||
#[tokio::test]
|
||||
async fn multipart_index_prune_rechecks_the_pin_under_the_marker_lease() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = open(root.path()).await;
|
||||
let owner = &disk.multipart_index;
|
||||
let directory = root.path().join(INDEX).join(sha("bucket"));
|
||||
std::fs::create_dir_all(&directory).expect("marker directory");
|
||||
let name = format!("{:064x}", 9);
|
||||
std::fs::write(directory.join(&name), b"1").expect("marker");
|
||||
|
||||
let pin = owner.pin_publication(owner.base.join(INDEX).join(sha("bucket")).join(&name));
|
||||
owner.prune_marker("bucket", &name).await.expect("prune of a pinned marker");
|
||||
assert!(directory.join(&name).exists(), "a publisher that pinned after the page keeps its marker");
|
||||
|
||||
drop(pin);
|
||||
owner.prune_marker("bucket", &name).await.expect("prune of a leftover");
|
||||
assert!(!directory.join(&name).exists(), "the same marker is a leftover once nobody publishes it");
|
||||
}
|
||||
|
||||
/// The publisher pins before it queues on the marker lease. A prune that
|
||||
/// holds the lease and found no pin has therefore unlinked the marker
|
||||
/// before the publisher records it, and the record recreates it.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn multipart_index_publisher_pins_before_it_waits_for_the_marker_lease() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = Arc::new(open(root.path()).await);
|
||||
let owner = Arc::clone(&disk.multipart_index);
|
||||
let (path, info) = upload("bucket", "queued-behind-a-prune");
|
||||
let source_root = path.split('/').next().expect("source sha").to_owned();
|
||||
let marker = owner.base.join(INDEX).join(sha("bucket")).join(&source_root);
|
||||
let visible_marker = root.path().join(INDEX).join(sha("bucket")).join(&source_root);
|
||||
|
||||
// Stand in for a prune that owns the marker lease and already read "no pin".
|
||||
let prune_lease = os::acquire_namespace_mutation_lease_with_owner(&marker, None).await;
|
||||
assert!(!owner.is_publishing(&marker));
|
||||
let writer = tokio::spawn({
|
||||
let disk = Arc::clone(&disk);
|
||||
let path = path.clone();
|
||||
async move { disk.write_metadata("bucket", RUSTFS_META_MULTIPART_BUCKET, &path, info).await }
|
||||
});
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
while !owner.is_publishing(&marker) {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(2)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("the publisher pins while the marker lease is still held by the prune");
|
||||
assert!(!visible_marker.exists(), "the marker is recorded only under the lease");
|
||||
|
||||
drop(prune_lease);
|
||||
writer.await.expect("writer task").expect("the upload metadata is published");
|
||||
assert!(visible_marker.exists(), "the record after the prune recreates the marker");
|
||||
let page = owner.page("bucket", None, Uuid::new_v4()).await.expect("page");
|
||||
assert_eq!(page.roots, vec![source_root]);
|
||||
}
|
||||
|
||||
/// Stale markers at the start of a page, at its cursor, and filling a whole
|
||||
/// page: every page still satisfies the receiver's contract and advances.
|
||||
#[tokio::test]
|
||||
async fn multipart_index_pages_of_stale_markers_keep_valid_cursors() {
|
||||
let root = tempfile::tempdir().expect("disk root");
|
||||
let disk = open(root.path()).await;
|
||||
let directory = root.path().join(INDEX).join(sha("bucket"));
|
||||
std::fs::create_dir_all(&directory).expect("marker directory");
|
||||
// Page one is entirely stale. Page two is live except its first name
|
||||
// and the name its cursor lands on. Two live names follow.
|
||||
let total = MULTIPART_INDEX_PAGE_SIZE * 2 + 2;
|
||||
let stale = |i: usize| i <= MULTIPART_INDEX_PAGE_SIZE || i == MULTIPART_INDEX_PAGE_SIZE * 2 - 1;
|
||||
for i in 0..total {
|
||||
let name = format!("{i:064x}");
|
||||
std::fs::write(directory.join(&name), b"1").expect("marker");
|
||||
if !stale(i) {
|
||||
std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&name)).expect("staging root");
|
||||
}
|
||||
}
|
||||
let disk_id = Uuid::new_v4();
|
||||
let first = disk.multipart_index.page("bucket", None, disk_id).await.expect("first page");
|
||||
first
|
||||
.validate("bucket", None, disk_id)
|
||||
.expect("an empty page with a cursor validates");
|
||||
assert!(first.roots.is_empty(), "a fully stale page has no roots");
|
||||
let cursor = first.next_marker.clone().expect("more markers follow a fully stale page");
|
||||
assert_eq!(cursor, format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE - 1));
|
||||
|
||||
let second = disk
|
||||
.multipart_index
|
||||
.page("bucket", Some(&cursor), disk_id)
|
||||
.await
|
||||
.expect("second page");
|
||||
second
|
||||
.validate("bucket", Some(&cursor), disk_id)
|
||||
.expect("stale first and cursor names validate");
|
||||
assert_eq!(second.roots.len(), MULTIPART_INDEX_PAGE_SIZE - 2);
|
||||
assert_eq!(second.roots.first(), Some(&format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE + 1)));
|
||||
let cursor = second
|
||||
.next_marker
|
||||
.clone()
|
||||
.expect("the cursor is the stale last examined name");
|
||||
assert_eq!(cursor, format!("{:064x}", MULTIPART_INDEX_PAGE_SIZE * 2 - 1));
|
||||
assert!(second.roots.last() < Some(&cursor));
|
||||
|
||||
let third = disk
|
||||
.multipart_index
|
||||
.page("bucket", Some(&cursor), disk_id)
|
||||
.await
|
||||
.expect("third page");
|
||||
third.validate("bucket", Some(&cursor), disk_id).expect("last page validates");
|
||||
assert_eq!(third.roots.len(), 2);
|
||||
assert!(third.next_marker.is_none());
|
||||
let remaining = std::fs::read_dir(&directory).expect("marker directory").count();
|
||||
assert_eq!(remaining, total - MULTIPART_INDEX_PAGE_SIZE - 2, "every stale marker is pruned");
|
||||
.set_modified(modified)
|
||||
.expect("set marker time");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1294,6 +1015,7 @@ mod tests {
|
||||
std::fs::create_dir_all(root.path().join(RUSTFS_META_MULTIPART_BUCKET).join(&name)).expect("staging root");
|
||||
}
|
||||
}
|
||||
age_marker(&directory.join(&stale));
|
||||
let disk_id = Uuid::new_v4();
|
||||
let first = disk.multipart_index.page("bucket", None, disk_id).await.expect("first page");
|
||||
first
|
||||
|
||||
@@ -813,8 +813,7 @@ impl SourceStore {
|
||||
Ok(())
|
||||
}
|
||||
pub async fn replace_head(&self, bytes: &[u8]) -> Result<()> {
|
||||
let index_pin = self
|
||||
.multipart_index
|
||||
self.multipart_index
|
||||
.record_head(&format!("{}/xl.meta", self.path), bytes)
|
||||
.await?;
|
||||
invalidate_active_multipart_source_head(&self.source);
|
||||
@@ -823,7 +822,6 @@ impl SourceStore {
|
||||
let root = self.publication_root.clone();
|
||||
let bytes = bytes.to_vec();
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
let _index_pin = index_pin;
|
||||
os::replace_multipart_source_metadata(&path, &bytes, &base, &root)
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -2585,13 +2585,7 @@ pub struct BoundedDeleteRecoveryDiscovery {
|
||||
pub complete: bool,
|
||||
}
|
||||
|
||||
/// Entries one bounded discovery call may charge on one drive. Object-scoped
|
||||
/// discovery charges each transaction-shaped (UUID) child of the key's
|
||||
/// directory once and every name that cannot be delete evidence at
|
||||
/// 1/`PLAIN_CHILDREN_PER_BUDGET_ENTRY`, so this admits a missing-key delete of a
|
||||
/// folder prefix with this many UUID-named children per drive. The request
|
||||
/// budget (`DeleteRecoveryWorkBudget`) still bounds the total across keys.
|
||||
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 262_144;
|
||||
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
|
||||
pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
||||
@@ -619,6 +619,10 @@ pub(crate) mod prepare_rename_test_hooks {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn has_after_component_opened() -> bool {
|
||||
!AFTER_COMPONENT_OPENED.lock().is_empty()
|
||||
}
|
||||
|
||||
/// Hooks keyed by a directory component name, run before a Windows
|
||||
/// directory pin retries a transient sharing conflict on it.
|
||||
#[cfg(windows)]
|
||||
@@ -1288,28 +1292,6 @@ pub(crate) fn fsync_dir_limited_std(dir: &Path) -> io::Result<()> {
|
||||
fsync_dir_std(dir)
|
||||
}
|
||||
|
||||
pub(crate) fn fsync_commit_ancestor_directory_limited_std(
|
||||
dir: &Path,
|
||||
base_dir: &Path,
|
||||
step_from_destination: usize,
|
||||
guard: &RenameCommitGuard,
|
||||
) -> io::Result<()> {
|
||||
#[cfg(target_os = "linux")]
|
||||
if base_dir == guard.directories.destination_base
|
||||
&& guard.directories.destination_path.ancestors().nth(1 + step_from_destination) == Some(dir)
|
||||
&& let Some(ancestors) = guard.directories.destination_ancestors.get()
|
||||
&& let Some(idx) = ancestors.len().checked_sub(1 + step_from_destination)
|
||||
&& let Some(file) = ancestors.get(idx)
|
||||
{
|
||||
#[cfg(test)]
|
||||
fsync_dir_recorder::record_limited(dir);
|
||||
return fsync_directory_handle_std(dir, Some(file));
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let _ = (base_dir, step_from_destination, guard);
|
||||
fsync_dir_limited_std(dir)
|
||||
}
|
||||
|
||||
/// Reuse the published directory without bypassing group commit or sync admission.
|
||||
pub(crate) async fn fsync_commit_directory(
|
||||
dir: &Path,
|
||||
@@ -6014,7 +5996,6 @@ struct LinuxCommitDirectories {
|
||||
destination_path: PathBuf,
|
||||
source: LinuxCommitDirectory,
|
||||
destination: std::sync::OnceLock<LinuxCommitDirectory>,
|
||||
destination_ancestors: std::sync::OnceLock<Vec<std::fs::File>>,
|
||||
destination_base: PathBuf,
|
||||
}
|
||||
|
||||
@@ -6031,8 +6012,6 @@ impl LinuxCommitDirectories {
|
||||
.ok_or_else(|| io::Error::other("missing destination directory guard"))?,
|
||||
);
|
||||
let directory = LinuxCommitDirectory::new(directory)?;
|
||||
let ancestors = parents.into_iter().map(std::fs::File::from).collect();
|
||||
let _ = self.destination_ancestors.set(ancestors);
|
||||
Ok(self.destination.get_or_init(|| directory))
|
||||
}
|
||||
|
||||
@@ -6042,7 +6021,7 @@ impl LinuxCommitDirectories {
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "file parent does not match its commit guard"))
|
||||
}
|
||||
|
||||
fn validate(&self, check_destination: bool) -> io::Result<()> {
|
||||
fn validate(&self) -> io::Result<()> {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
// A pinned directory may have been unlinked or replaced. Do not report a
|
||||
// successful publication into a detached object tree.
|
||||
@@ -6051,9 +6030,8 @@ impl LinuxCommitDirectories {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "commit directory identity changed"));
|
||||
}
|
||||
let destination = self.prepare_destination()?;
|
||||
if check_destination
|
||||
&& ((self.source.device, self.source.inode) != (destination.device, destination.inode)
|
||||
|| self.source_path.as_os_str() != self.destination_path.as_os_str())
|
||||
if (self.source.device, self.source.inode) != (destination.device, destination.inode)
|
||||
|| self.source_path.as_os_str() != self.destination_path.as_os_str()
|
||||
{
|
||||
let current = self.root.open_directory(&self.destination_path)?.metadata()?;
|
||||
if (current.dev(), current.ino()) != (destination.device, destination.inode) {
|
||||
@@ -6067,9 +6045,8 @@ impl LinuxCommitDirectories {
|
||||
record_native_publication(destination);
|
||||
let source_name = self.file_name(source, true)?;
|
||||
let destination_name = self.file_name(destination, false)?;
|
||||
let destination_was_prepared = self.destination.get().is_some();
|
||||
let destination_parent = self.prepare_destination()?;
|
||||
self.validate(destination_was_prepared)?;
|
||||
self.validate()?;
|
||||
let mut attempt = 0;
|
||||
loop {
|
||||
if let Some(prepared) = prepared {
|
||||
@@ -6291,7 +6268,7 @@ pub(crate) fn multipart_payload_commit_guard_std(
|
||||
source_parent: &Path,
|
||||
) -> io::Result<RenameCommitGuard> {
|
||||
let original = &metadata_guard.directories;
|
||||
original.validate(true)?;
|
||||
original.validate()?;
|
||||
let source = LinuxCommitDirectory::new(original.root.open_directory(source_parent)?)?;
|
||||
let destination = original.prepare_destination()?;
|
||||
// Clone the pinned file identity, never recapture the target pathname
|
||||
@@ -6308,7 +6285,6 @@ pub(crate) fn multipart_payload_commit_guard_std(
|
||||
destination_path: original.destination_path.clone(),
|
||||
source,
|
||||
destination: std::sync::OnceLock::from(destination),
|
||||
destination_ancestors: std::sync::OnceLock::new(),
|
||||
destination_base: original.destination_base.clone(),
|
||||
}),
|
||||
})
|
||||
@@ -6390,7 +6366,6 @@ pub(crate) fn prepare_rename_commit_guard(
|
||||
destination_path: destination_parent.to_path_buf(),
|
||||
source,
|
||||
destination,
|
||||
destination_ancestors: std::sync::OnceLock::new(),
|
||||
destination_base: destination_base.to_path_buf(),
|
||||
}),
|
||||
})
|
||||
@@ -7500,7 +7475,6 @@ fn same_parent_commit_guard(
|
||||
device: dir.device,
|
||||
inode: dir.inode,
|
||||
}),
|
||||
destination_ancestors: std::sync::OnceLock::new(),
|
||||
destination_base: base.to_path_buf(),
|
||||
}),
|
||||
});
|
||||
@@ -8540,7 +8514,37 @@ pub(crate) fn mkdir_all_below_existing_base_std(
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
|
||||
#[cfg(target_os = "linux")]
|
||||
let base = publication_root.open_directory(base_dir)?.into();
|
||||
let base = {
|
||||
use rustix::fs::{ResolveFlags, openat2};
|
||||
use std::os::fd::AsFd;
|
||||
|
||||
let owned_base = (base_dir != publication_root.io_path.as_path() && base_dir != publication_root.path.as_path())
|
||||
.then(|| publication_root.open_directory(base_dir).map(std::os::fd::OwnedFd::from))
|
||||
.transpose()?;
|
||||
#[cfg(test)]
|
||||
let has_hook = prepare_rename_test_hooks::has_after_component_opened();
|
||||
#[cfg(not(test))]
|
||||
let has_hook = false;
|
||||
if !has_hook {
|
||||
let base_fd = owned_base
|
||||
.as_ref()
|
||||
.map_or_else(|| publication_root.directory.as_fd(), AsFd::as_fd);
|
||||
let target = if relative.as_os_str().is_empty() {
|
||||
Path::new(".")
|
||||
} else {
|
||||
relative
|
||||
};
|
||||
match openat2(base_fd, target, flags, Mode::empty(), ResolveFlags::BENEATH | ResolveFlags::NO_SYMLINKS) {
|
||||
Ok(leaf) => return Ok(vec![leaf]),
|
||||
Err(Errno::NOENT | Errno::NOSYS) => {}
|
||||
Err(err) => return Err(err.into()),
|
||||
}
|
||||
}
|
||||
match owned_base {
|
||||
Some(base) => base,
|
||||
None => publication_root.open_directory(base_dir)?.into(),
|
||||
}
|
||||
};
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let base = {
|
||||
// The configured endpoint is trusted; every component beneath it
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use crate::disk::error::Error as DiskError;
|
||||
use crate::erasure::codec::buffer_pool::{get_ec_buffer, return_ec_buffer};
|
||||
use pin_project_lite::pin_project;
|
||||
use rustfs_utils::{HashAlgorithm, LogThrottle};
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use std::future::poll_fn;
|
||||
use std::io::IoSlice;
|
||||
use std::pin::Pin;
|
||||
@@ -28,13 +28,6 @@ const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_ERASURE: &str = "erasure";
|
||||
const EVENT_BITROT_SHORT_SHARD_READ: &str = "bitrot_short_shard_read";
|
||||
const EVENT_BITROT_HASH_MISMATCH: &str = "bitrot_hash_mismatch";
|
||||
/// A failing drive raises these once per shard block of every read it serves.
|
||||
/// The events carry no object identity (the caller logs that), so one line per
|
||||
/// interval with the suppressed count loses nothing but repetition.
|
||||
const BITROT_FAILURE_LOG_INTERVAL_MS: u64 = 5_000;
|
||||
static BITROT_SHORT_SHARD_READ_LOGS: LogThrottle = LogThrottle::new(BITROT_FAILURE_LOG_INTERVAL_MS);
|
||||
static BITROT_HASH_MISMATCH_LOGS: LogThrottle = LogThrottle::new(BITROT_FAILURE_LOG_INTERVAL_MS);
|
||||
static BITROT_READ_ERROR_LOGS: LogThrottle = LogThrottle::new(BITROT_FAILURE_LOG_INTERVAL_MS);
|
||||
const MAX_RETAINED_CHUNKS_PER_BLOCK: usize = 64;
|
||||
const MAX_CHUNK_POLLS_PER_YIELD: usize = MAX_RETAINED_CHUNKS_PER_BLOCK + 1;
|
||||
|
||||
@@ -266,41 +259,18 @@ fn reserve_bitrot_scratch(
|
||||
|
||||
/// A truncated shard is `UnexpectedEof`, not a short success (backlog#799 B2).
|
||||
fn short_shard_read(got: usize, want: usize) -> std::io::Error {
|
||||
if let Some(suppressed) = BITROT_SHORT_SHARD_READ_LOGS.claim() {
|
||||
error!(
|
||||
event = EVENT_BITROT_SHORT_SHARD_READ,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE,
|
||||
state = "failed",
|
||||
got,
|
||||
want,
|
||||
suppressed,
|
||||
"short shard read: got {got} of {want} bytes"
|
||||
);
|
||||
}
|
||||
error!(
|
||||
event = EVENT_BITROT_SHORT_SHARD_READ,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE,
|
||||
state = "failed",
|
||||
got,
|
||||
want,
|
||||
"short shard read: got {got} of {want} bytes"
|
||||
);
|
||||
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, DiskError::FileCorrupt)
|
||||
}
|
||||
|
||||
fn log_hash_mismatch(data_len: usize) {
|
||||
if let Some(suppressed) = BITROT_HASH_MISMATCH_LOGS.claim() {
|
||||
error!(
|
||||
event = EVENT_BITROT_HASH_MISMATCH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE,
|
||||
state = "failed",
|
||||
data_len,
|
||||
suppressed,
|
||||
"bitrot hash mismatch"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn log_read_error(error: &std::io::Error) {
|
||||
if let Some(suppressed) = BITROT_READ_ERROR_LOGS.claim() {
|
||||
error!(suppressed, "bitrot reader read error: {error}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Split a `[hash][data]` block, verify the hash (unless `skip_verify`), and
|
||||
/// return the data slice plus the time spent hashing. The caller writes `data`
|
||||
/// into its sink **only after** this returns `Ok`, so a shard that fails the
|
||||
@@ -316,7 +286,14 @@ fn split_and_verify<'a>(hash_algo: &HashAlgorithm, skip_verify: bool, block: &'a
|
||||
let actual_hash = hash_algo.hash_encode(data);
|
||||
let verify = verify_start.elapsed();
|
||||
if actual_hash.as_ref() != hash {
|
||||
log_hash_mismatch(data.len());
|
||||
error!(
|
||||
event = EVENT_BITROT_HASH_MISMATCH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE,
|
||||
state = "failed",
|
||||
data_len = data.len(),
|
||||
"bitrot hash mismatch"
|
||||
);
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
|
||||
}
|
||||
Ok((data, verify))
|
||||
@@ -368,7 +345,7 @@ where
|
||||
.inner
|
||||
.read_buf(&mut (&mut *out).limit(remaining))
|
||||
.await
|
||||
.inspect_err(log_read_error)?;
|
||||
.inspect_err(|e| error!("bitrot reader read error: {e}"))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
@@ -482,7 +459,14 @@ where
|
||||
for chunk in block_chunks() {
|
||||
let take = remaining.min(chunk.len());
|
||||
if actual_hash.as_ref()[hash_offset..hash_offset + take] != chunk[..take] {
|
||||
log_hash_mismatch(want);
|
||||
error!(
|
||||
event = EVENT_BITROT_HASH_MISMATCH,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_ERASURE,
|
||||
state = "failed",
|
||||
data_len = want,
|
||||
"bitrot hash mismatch"
|
||||
);
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, DiskError::FileCorrupt));
|
||||
}
|
||||
hash_offset += take;
|
||||
@@ -610,7 +594,10 @@ where
|
||||
{
|
||||
let mut filled = 0;
|
||||
while filled < buf.len() {
|
||||
let n = reader.read(&mut buf[filled..]).await.inspect_err(log_read_error)?;
|
||||
let n = reader.read(&mut buf[filled..]).await.map_err(|e| {
|
||||
error!("bitrot reader read error: {}", e);
|
||||
e
|
||||
})?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
@@ -1271,34 +1258,6 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
/// The log line is bounded; the rejection is not. Every damaged block is
|
||||
/// refused with the same typed error however many came before it.
|
||||
#[test]
|
||||
fn repeated_bitrot_failures_are_all_rejected_while_their_log_is_bounded() {
|
||||
use crate::disk::error::Error as DiskError;
|
||||
let algo = HashAlgorithm::HighwayHash256;
|
||||
let data = b"one of many damaged blocks";
|
||||
let mut frame = algo.hash_encode(data).as_ref().to_vec();
|
||||
frame.extend_from_slice(data);
|
||||
*frame.last_mut().expect("test frame has payload") ^= 1;
|
||||
|
||||
for _ in 0..1_000 {
|
||||
let error = super::split_and_verify(&algo, false, &frame).expect_err("every mismatch rejects its shard");
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(DiskError::from(error), DiskError::FileCorrupt);
|
||||
let error = super::short_shard_read(3, 4);
|
||||
assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof);
|
||||
assert_eq!(DiskError::from(error), DiskError::FileCorrupt);
|
||||
super::log_read_error(&io::Error::other("drive read failed"));
|
||||
}
|
||||
|
||||
// Each burst used its own window, so a further event of the same kind
|
||||
// inside the interval is suppressed.
|
||||
assert!(super::BITROT_HASH_MISMATCH_LOGS.claim().is_none());
|
||||
assert!(super::BITROT_SHORT_SHARD_READ_LOGS.claim().is_none());
|
||||
assert!(super::BITROT_READ_ERROR_LOGS.claim().is_none());
|
||||
}
|
||||
|
||||
struct FragmentedSource {
|
||||
chunks: VecDeque<Bytes>,
|
||||
}
|
||||
|
||||
@@ -894,13 +894,6 @@ fn record_shard_read_result(
|
||||
}
|
||||
}
|
||||
|
||||
/// A slow or misaligned shard reader is retired once per stripe of every read
|
||||
/// it lags on. The lines name only a shard index, so one per interval with the
|
||||
/// suppressed count is enough.
|
||||
const RETIRED_READER_LOG_INTERVAL_MS: u64 = 5_000;
|
||||
static ABANDONED_READER_LOGS: rustfs_utils::LogThrottle = rustfs_utils::LogThrottle::new(RETIRED_READER_LOG_INTERVAL_MS);
|
||||
static UNALIGNED_PARITY_READER_LOGS: rustfs_utils::LogThrottle = rustfs_utils::LogThrottle::new(RETIRED_READER_LOG_INTERVAL_MS);
|
||||
|
||||
fn retire_abandoned_readers(errs: &mut [Option<Error>], retire_readers: &mut ShardIndexes, active_readers: &[bool]) {
|
||||
for (i, active) in active_readers.iter().enumerate() {
|
||||
if !*active {
|
||||
@@ -911,9 +904,7 @@ fn retire_abandoned_readers(errs: &mut [Option<Error>], retire_readers: &mut Sha
|
||||
errs[i] = Some(Error::from(io::Error::new(ErrorKind::TimedOut, "shard read abandoned after read quorum")));
|
||||
}
|
||||
retire_readers.push(i);
|
||||
if let Some(suppressed) = ABANDONED_READER_LOGS.claim() {
|
||||
warn!(shard_index = i, suppressed, "retiring in-flight shard reader after read quorum");
|
||||
}
|
||||
warn!(shard_index = i, "retiring in-flight shard reader after read quorum");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2033,12 +2024,10 @@ where
|
||||
self.engaged[idx] = true;
|
||||
true
|
||||
} else {
|
||||
if let Some(suppressed) = UNALIGNED_PARITY_READER_LOGS.claim() {
|
||||
warn!(
|
||||
shard_index = idx,
|
||||
stripe_index, suppressed, "retiring parity reader that cannot be aligned to the current stripe"
|
||||
);
|
||||
}
|
||||
warn!(
|
||||
shard_index = idx,
|
||||
stripe_index, "retiring parity reader that cannot be aligned to the current stripe"
|
||||
);
|
||||
self.readers[idx] = None;
|
||||
false
|
||||
}
|
||||
@@ -2855,29 +2844,6 @@ impl Erasure {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// The warning is bounded; the retirement is not. Every call retires every
|
||||
/// reader still in flight and records why, however many calls came before.
|
||||
#[test]
|
||||
fn abandoned_readers_are_retired_on_every_call_while_their_log_is_bounded() {
|
||||
let active_readers = [true, false, true, true];
|
||||
for _ in 0..1_000 {
|
||||
let mut errs: Vec<Option<Error>> = vec![None, None, Some(Error::other("already failed")), None];
|
||||
let mut retire_readers = ShardIndexes::new();
|
||||
retire_abandoned_readers(&mut errs, &mut retire_readers, &active_readers);
|
||||
|
||||
assert_eq!(retire_readers.as_slice(), &[0, 2, 3]);
|
||||
assert!(errs[1].is_none(), "an inactive reader is left alone");
|
||||
assert!(errs[0].is_some() && errs[3].is_some(), "an abandoned reader gets its timeout error");
|
||||
assert!(
|
||||
errs[2]
|
||||
.as_ref()
|
||||
.is_some_and(|error| error.to_string().contains("already failed")),
|
||||
"an existing error is kept"
|
||||
);
|
||||
}
|
||||
assert!(ABANDONED_READER_LOGS.claim().is_none(), "the burst used the window");
|
||||
}
|
||||
use crate::{
|
||||
disk::error::DiskError,
|
||||
erasure::coding::{BitrotReader, BitrotWriter},
|
||||
|
||||
@@ -2149,15 +2149,6 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs endpoint parsing cases whose local paths do not exist. Unix leaves
|
||||
/// missing paths out of the physical-disk independence check, but Windows
|
||||
/// resolves them to their volume, so paths such as `/d1` and `/d2` count as
|
||||
/// one shared device there. These cases cover parsing, not disk
|
||||
/// independence, which has its own tests below.
|
||||
async fn with_disk_independence_check_bypassed<F: std::future::Future<Output = ()>>(cases: F) {
|
||||
async_with_vars([(ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true"))], cases).await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn endpoint_parse_errors_do_not_echo_url_credentials() {
|
||||
let err = Endpoints::try_from(["http://:topsecret@server/path"].as_slice()).unwrap_err();
|
||||
@@ -2166,13 +2157,8 @@ mod test {
|
||||
assert!(!err.to_string().contains("topsecret"));
|
||||
}
|
||||
|
||||
#[serial]
|
||||
#[tokio::test]
|
||||
async fn explicit_local_endpoint_host_selects_only_exact_host_and_server_port() {
|
||||
with_disk_independence_check_bypassed(explicit_local_endpoint_host_selects_only_exact_host_and_server_port_cases()).await;
|
||||
}
|
||||
|
||||
async fn explicit_local_endpoint_host_selects_only_exact_host_and_server_port_cases() {
|
||||
let args = vec![
|
||||
"http://rustfs-0.rustfs-headless.ns.svc.cluster.local./data0",
|
||||
"http://rustfs-0.rustfs-headless.ns.svc.cluster.local.:9443/data1",
|
||||
@@ -2935,13 +2921,8 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[serial]
|
||||
#[tokio::test]
|
||||
async fn test_create_pool_endpoints() {
|
||||
with_disk_independence_check_bypassed(test_create_pool_endpoints_cases()).await;
|
||||
}
|
||||
|
||||
async fn test_create_pool_endpoints_cases() {
|
||||
#[derive(Default)]
|
||||
struct TestCase<'a> {
|
||||
num: usize,
|
||||
@@ -3560,13 +3541,8 @@ mod test {
|
||||
(urls, local_flags)
|
||||
}
|
||||
|
||||
#[serial]
|
||||
#[tokio::test]
|
||||
async fn test_create_server_endpoints() {
|
||||
with_disk_independence_check_bypassed(test_create_server_endpoints_cases()).await;
|
||||
}
|
||||
|
||||
async fn test_create_server_endpoints_cases() {
|
||||
let test_cases = [
|
||||
// Invalid input.
|
||||
("", vec![], false),
|
||||
|
||||
@@ -975,11 +975,9 @@ pub struct DeleteRecoveryWorkBudget {
|
||||
remaining: std::sync::atomic::AtomicU32,
|
||||
}
|
||||
|
||||
/// Per-drive unit a request budget is sized by (a typical discovery plus one
|
||||
/// inspection). Admission charges what a call visited, not this unit, so it only
|
||||
/// scales the request total with the drives a request can reach; it is kept
|
||||
/// independent of the per-call caps, which may let one key's probe use more of
|
||||
/// the same total on a large folder.
|
||||
/// Entries one disk may visit per recovery step (discovery plus inspection).
|
||||
/// Every disk of every set a request fans out to is admitted up to this much
|
||||
/// and charged on completion, so a budget must cover that many disks.
|
||||
const DELETE_RECOVERY_RESERVATION_PER_DISK: u32 = 4096 + 65_536;
|
||||
|
||||
impl Default for DeleteRecoveryWorkBudget {
|
||||
|
||||
@@ -11040,12 +11040,7 @@ mod tests {
|
||||
let live = Uuid::new_v4();
|
||||
let orphan = Uuid::new_v4();
|
||||
let object_dir = dir.path().join("bucket/obj");
|
||||
// Part files are inert to discovery and cost 1/64 of an entry, so the
|
||||
// first step is exhausted by what still costs a full entry each: the
|
||||
// data directories of many versions, all referenced by the metadata.
|
||||
let versions = (0..4200).map(|_| Uuid::new_v4()).collect::<Vec<_>>();
|
||||
let referenced = std::iter::once(live).chain(versions.iter().copied()).collect::<Vec<_>>();
|
||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &referenced).await;
|
||||
write_object_meta_with_data_dirs(&object_dir, "bucket", "obj", &[live]).await;
|
||||
let live_dir = object_dir.join(live.to_string());
|
||||
let orphan_dir = object_dir.join(orphan.to_string());
|
||||
fs::create_dir_all(&live_dir).await.expect("create live multipart directory");
|
||||
@@ -11054,14 +11049,10 @@ mod tests {
|
||||
.await
|
||||
.expect("write orphan payload");
|
||||
let parts_dir = live_dir.clone();
|
||||
let versions_root = object_dir.clone();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
for part in 1..=4200 {
|
||||
std::fs::write(parts_dir.join(format!("part.{part}")), b"live").expect("write live multipart entry");
|
||||
}
|
||||
for version in versions {
|
||||
std::fs::create_dir(versions_root.join(version.to_string())).expect("create referenced version directory");
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("create multipart fixture");
|
||||
@@ -11071,10 +11062,7 @@ mod tests {
|
||||
.await
|
||||
.expect("read first discovery page");
|
||||
assert!(first.transactions.is_empty());
|
||||
assert!(
|
||||
first.next.is_some(),
|
||||
"the data directories of many versions must exhaust the first scan step"
|
||||
);
|
||||
assert!(first.next.is_some(), "ordinary multipart entries must exhaust the first scan budget");
|
||||
let last = disk
|
||||
.discover_delete_transactions("bucket", Some("obj"), first.next, 1)
|
||||
.await
|
||||
|
||||
@@ -1709,37 +1709,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_missing_key_naming_a_folder_of_uuid_named_keys_completes() {
|
||||
let bucket = "recover-uuid-named-folder";
|
||||
let (dirs, store, _) = fixture(bucket).await;
|
||||
// More UUID-named child keys per drive than the former 4096-entry probe
|
||||
// could cover once each child's `xl.meta` and data directory were charged
|
||||
// (three entries per child).
|
||||
const CHILDREN: usize = 1500;
|
||||
for dir in &dirs {
|
||||
let prefix = dir.path().join(bucket).join("uploads");
|
||||
for _ in 0..CHILDREN {
|
||||
let child = prefix.join(Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(child.join(Uuid::new_v4().to_string())).expect("child key data dir");
|
||||
std::fs::write(child.join(crate::disk::STORAGE_FORMAT_FILE), b"meta").expect("child key metadata");
|
||||
}
|
||||
}
|
||||
let error = store
|
||||
.delete_object(bucket, "uploads", ObjectOptions::default())
|
||||
.await
|
||||
.expect_err("the folder's own key does not exist");
|
||||
assert!(
|
||||
crate::error::is_err_object_not_found(&error),
|
||||
"a complete inventory keeps the missing-key no-op semantics: {error:?}"
|
||||
);
|
||||
assert!(dirs.iter().all(|dir| {
|
||||
std::fs::read_dir(dir.path().join(bucket).join("uploads"))
|
||||
.map(|entries| entries.count() == CHILDREN)
|
||||
.unwrap_or(false)
|
||||
}));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_batch_budget_survives_pool_and_set_fanout() {
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
|
||||
@@ -69,92 +69,6 @@ use super::super::{
|
||||
use super::bitrot_self_verify::{BitrotSelfVerifyTarget, drop_failed_writer_disks, verify_written_bitrot_shards};
|
||||
use super::delete_recovery::DeleteRecoveryContext;
|
||||
|
||||
const EVENT_SET_DISK_DELETE_OBJECTS_STAGE: &str = "set_disk_delete_objects_stage";
|
||||
const DELETE_OBJECTS_STAGE_SLOW_MS: u128 = 1_000;
|
||||
|
||||
/// Counts use the stage's actual inputs, not failures inherited from an earlier stage.
|
||||
struct DeleteObjectsStageSummary {
|
||||
stage: &'static str,
|
||||
started: Instant,
|
||||
attempted_object_count: usize,
|
||||
attempted_group_count: usize,
|
||||
fence_lost: bool,
|
||||
failed_object_count: usize,
|
||||
failed_group_count: usize,
|
||||
disk_error_counts: std::collections::BTreeMap<u32, usize>,
|
||||
}
|
||||
|
||||
impl DeleteObjectsStageSummary {
|
||||
fn new(stage: &'static str, attempted_object_count: usize, attempted_group_count: usize) -> Self {
|
||||
Self {
|
||||
stage,
|
||||
started: Instant::now(),
|
||||
attempted_object_count,
|
||||
attempted_group_count,
|
||||
fence_lost: false,
|
||||
failed_object_count: 0,
|
||||
failed_group_count: 0,
|
||||
disk_error_counts: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn record_disk_failure(&mut self, error: &DiskError, object_count: usize) {
|
||||
self.failed_object_count += object_count;
|
||||
self.failed_group_count += 1;
|
||||
// Wire codes preserve typed disk causes without exposing error messages or paths.
|
||||
*self.disk_error_counts.entry(error.to_u32()).or_default() += 1;
|
||||
}
|
||||
|
||||
fn log(self, pool_index: usize, set_index: usize, disk_index: Option<usize>) {
|
||||
let elapsed_ms = self.started.elapsed().as_millis();
|
||||
let state = if self.fence_lost { "fence_lost" } else { "completed" };
|
||||
let result = if self.fence_lost || self.failed_object_count > 0 {
|
||||
"failed"
|
||||
} else {
|
||||
"ok"
|
||||
};
|
||||
if self.fence_lost || self.failed_object_count > 0 || elapsed_ms >= DELETE_OBJECTS_STAGE_SLOW_MS {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_DELETE_OBJECTS_STAGE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
state,
|
||||
result,
|
||||
stage = self.stage,
|
||||
pool_index,
|
||||
set_index,
|
||||
disk_index,
|
||||
attempted_object_count = self.attempted_object_count,
|
||||
attempted_group_count = self.attempted_group_count,
|
||||
failed_object_count = self.failed_object_count,
|
||||
failed_group_count = self.failed_group_count,
|
||||
disk_error_counts = ?self.disk_error_counts,
|
||||
elapsed_ms,
|
||||
"Batch delete stage completed"
|
||||
);
|
||||
} else {
|
||||
debug!(
|
||||
event = EVENT_SET_DISK_DELETE_OBJECTS_STAGE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_SET_DISK,
|
||||
state,
|
||||
result,
|
||||
stage = self.stage,
|
||||
pool_index,
|
||||
set_index,
|
||||
disk_index,
|
||||
attempted_object_count = self.attempted_object_count,
|
||||
attempted_group_count = self.attempted_group_count,
|
||||
failed_object_count = self.failed_object_count,
|
||||
failed_group_count = self.failed_group_count,
|
||||
disk_error_counts = ?self.disk_error_counts,
|
||||
elapsed_ms,
|
||||
"Batch delete stage completed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DeleteCommitContext<'a> {
|
||||
incarnation: Option<Uuid>,
|
||||
@@ -10061,7 +9975,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
let dist_erasure = self.ctx.is_dist_erasure().await;
|
||||
let mut dist_batch_lock_ids = vec![Vec::new(); self.lockers.len()];
|
||||
|
||||
let mut raw_lock_summary = DeleteObjectsStageSummary::new("raw_acquire", objects.len(), unique_lock_count);
|
||||
if opts.no_lock {
|
||||
locked_objects = unique_objects;
|
||||
} else if dist_erasure {
|
||||
@@ -10101,13 +10014,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
for (i, lock_name) in lock_names.iter().enumerate() {
|
||||
if let Some(err) = failed_map.get(&(bucket.to_string(), lock_name.to_string())) {
|
||||
del_errs[i] = Some(Error::other(err.to_string()));
|
||||
raw_lock_summary.failed_object_count += 1;
|
||||
}
|
||||
}
|
||||
if !opts.no_lock {
|
||||
raw_lock_summary.failed_group_count = failed_map.len();
|
||||
raw_lock_summary.log(self.pool_index, self.set_index, None);
|
||||
}
|
||||
|
||||
let outer_namespace_lock_fence = opts.namespace_lock_fence.clone();
|
||||
let mut transaction_guards = std::collections::BTreeMap::new();
|
||||
@@ -10115,18 +10023,12 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// Every raw batch lock is acquired first. Canonical sorting also
|
||||
// gives aliases of one disk key exactly one transaction owner.
|
||||
let keys: std::collections::BTreeSet<_> = locked_objects.iter().cloned().collect();
|
||||
// Batch unresolved keys per peer under one total acquisition budget.
|
||||
// Confirmed cleanup precedes retries; each key keeps its own quorum
|
||||
// and failure result at the set layer.
|
||||
// Bulk delete is the only multi-key transaction-lock holder; one
|
||||
// batched lock round replaces one quorum round-trip per key and is
|
||||
// deadlock-free by construction (a single `lock_batch` RPC per lock
|
||||
// client, global (shard, key) order at each server). Per-key
|
||||
// failures keep the set layer's per-key error reporting.
|
||||
let key_names: Vec<&str> = keys.iter().map(|key| key.as_str()).collect();
|
||||
let mut transaction_summary = DeleteObjectsStageSummary::new(
|
||||
"transaction_acquire",
|
||||
lock_names
|
||||
.iter()
|
||||
.filter(|name| locked_objects.contains(name.as_ref()))
|
||||
.count(),
|
||||
key_names.len(),
|
||||
);
|
||||
let acquired = self
|
||||
.acquire_delete_transaction_locks_batch(bucket, &key_names, super::super::get_lock_acquire_timeout())
|
||||
.await;
|
||||
@@ -10137,17 +10039,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
transaction_guards.insert(key, guard);
|
||||
}
|
||||
Err(error) => {
|
||||
transaction_summary.failed_group_count += 1;
|
||||
for (index, lock_name) in lock_names.iter().enumerate() {
|
||||
if lock_name.as_ref() == key {
|
||||
del_errs[index] = Some(error.clone());
|
||||
transaction_summary.failed_object_count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
transaction_summary.log(self.pool_index, self.set_index, None);
|
||||
}
|
||||
let transaction_guards = Arc::new(transaction_guards);
|
||||
let current_generations = opts.conditional_delete_generations.as_deref();
|
||||
@@ -10605,18 +10504,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
let mut fence_summary = DeleteObjectsStageSummary::new(
|
||||
"fence_before_prepare",
|
||||
del_errs.iter().filter(|error| error.is_none()).count(),
|
||||
0,
|
||||
);
|
||||
fence_summary.fence_lost = true;
|
||||
if dist_erasure {
|
||||
self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await;
|
||||
}
|
||||
for (index, object) in objects.iter().enumerate() {
|
||||
if del_errs[index].is_none() {
|
||||
fence_summary.failed_object_count += 1;
|
||||
del_errs[index] = Some(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "delete_objects_commit",
|
||||
bucket: bucket.to_string(),
|
||||
@@ -10626,7 +10518,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
});
|
||||
}
|
||||
}
|
||||
fence_summary.log(self.pool_index, self.set_index, None);
|
||||
return (del_objects, del_errs, accounting);
|
||||
}
|
||||
|
||||
@@ -10693,9 +10584,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
|
||||
// let mut errors = Vec::with_capacity(disks.len());
|
||||
|
||||
for (disk_index, disk) in disks.iter().enumerate() {
|
||||
for disk in disks.iter() {
|
||||
let disk_namespace_owner = namespace_owner.clone();
|
||||
let group_inputs = &vers;
|
||||
let vers = vers.clone();
|
||||
let namespace_lock_fence = opts.namespace_lock_fence.clone();
|
||||
let lock_lost_during_commit = Arc::clone(&lock_lost_during_commit);
|
||||
@@ -10704,12 +10594,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
lock_lost_during_commit.store(true, std::sync::atomic::Ordering::Release);
|
||||
return (0..vers.len()).map(|_| Some(DiskError::DiskOngoingReq)).collect();
|
||||
}
|
||||
let mut prepare_summary = DeleteObjectsStageSummary::new(
|
||||
"prepare",
|
||||
group_inputs.iter().map(|group| group.versions.len()).sum(),
|
||||
group_inputs.len(),
|
||||
);
|
||||
let errors = if let Some(disk) = disk {
|
||||
if let Some(disk) = disk {
|
||||
let options = DeleteOptions {
|
||||
old_data_dir: Some(rollback_dir),
|
||||
delete_recovery_bucket_incarnation: opts.expected_bucket_incarnation_id,
|
||||
@@ -10777,28 +10662,14 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
errs.push(Some(DiskError::DiskNotFound));
|
||||
}
|
||||
errs
|
||||
};
|
||||
for (error, group) in errors.iter().zip(group_inputs) {
|
||||
if let Some(error) = error {
|
||||
prepare_summary.record_disk_failure(error, group.versions.len());
|
||||
}
|
||||
}
|
||||
prepare_summary.log(self.pool_index, self.set_index, Some(disk_index));
|
||||
errors
|
||||
});
|
||||
}
|
||||
|
||||
let results = join_all(futures).await;
|
||||
if transaction_guards.values().any(|guard| guard.is_lock_lost()) {
|
||||
let mut fence_summary = DeleteObjectsStageSummary::new(
|
||||
"transaction_fence_after_prepare",
|
||||
del_errs.iter().filter(|error| error.is_none()).count(),
|
||||
0,
|
||||
);
|
||||
fence_summary.fence_lost = true;
|
||||
for (index, object) in objects.iter().enumerate() {
|
||||
if del_errs[index].is_none() {
|
||||
fence_summary.failed_object_count += 1;
|
||||
accounting[index] = None;
|
||||
del_errs[index] = Some(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "delete_transaction",
|
||||
@@ -10809,7 +10680,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
});
|
||||
}
|
||||
}
|
||||
fence_summary.log(self.pool_index, self.set_index, None);
|
||||
if dist_erasure {
|
||||
self.release_dist_delete_object_locks_batch(dist_batch_lock_ids).await;
|
||||
}
|
||||
@@ -10876,15 +10746,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
}
|
||||
|
||||
if lock_lost_during_commit.load(std::sync::atomic::Ordering::Acquire) {
|
||||
let mut fence_summary = DeleteObjectsStageSummary::new(
|
||||
"fence_during_prepare",
|
||||
del_errs.iter().filter(|error| error.is_none()).count(),
|
||||
0,
|
||||
);
|
||||
fence_summary.fence_lost = true;
|
||||
for (index, object) in objects.iter().enumerate() {
|
||||
if del_errs[index].is_none() {
|
||||
fence_summary.failed_object_count += 1;
|
||||
del_errs[index] = Some(Error::NamespaceLockQuorumUnavailable {
|
||||
mode: "delete_objects_commit",
|
||||
bucket: bucket.to_string(),
|
||||
@@ -10894,7 +10757,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
});
|
||||
}
|
||||
}
|
||||
fence_summary.log(self.pool_index, self.set_index, None);
|
||||
}
|
||||
|
||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
@@ -10913,11 +10775,6 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
// Mark all attempted groups before starting publication. A lost
|
||||
// response can hide a persisted decision and must never roll back.
|
||||
let groups = groups.collect::<Vec<_>>();
|
||||
let mut receipt_summary = DeleteObjectsStageSummary::new(
|
||||
"receipt",
|
||||
groups.iter().map(|index| vers[*index].versions.len()).sum(),
|
||||
groups.len(),
|
||||
);
|
||||
published_groups.extend(groups.iter().map(|index| vers[*index].name.clone()));
|
||||
// Journal publish fans out per object per drive; keep enough
|
||||
// publications in flight to feed the drives' durable-mutation
|
||||
@@ -10965,13 +10822,11 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
for (index, result) in publications {
|
||||
let group = &vers[index];
|
||||
if let Err(error) = result {
|
||||
receipt_summary.record_disk_failure(&error, group.versions.len());
|
||||
for version in &group.versions {
|
||||
del_errs[version.idx] = Some(to_object_err(error.clone().into(), vec![bucket, &group.name]));
|
||||
}
|
||||
}
|
||||
}
|
||||
receipt_summary.log(self.pool_index, self.set_index, None);
|
||||
}
|
||||
let mut rollback_futures = Vec::new();
|
||||
let committed_receipt_indices =
|
||||
@@ -29042,24 +28897,3 @@ mod delete_recovery_inventory_tests {
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
mod data_movement_prefix_cleanup_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod delete_objects_stage_summary_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn batch_delete_diagnostics_count_groups_and_versions_separately() {
|
||||
let mut summary = DeleteObjectsStageSummary::new("receipt", 6, 3);
|
||||
summary.record_disk_failure(&DiskError::ErasureWriteQuorum, 3);
|
||||
summary.record_disk_failure(&DiskError::ErasureWriteQuorum, 2);
|
||||
|
||||
assert_eq!(summary.attempted_object_count, 6);
|
||||
assert_eq!(summary.attempted_group_count, 3);
|
||||
assert_eq!(summary.failed_object_count, 5);
|
||||
assert_eq!(summary.failed_group_count, 2);
|
||||
assert_eq!(
|
||||
summary.disk_error_counts,
|
||||
std::collections::BTreeMap::from([(DiskError::ErasureWriteQuorum.to_u32(), 2)])
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2533,7 +2533,6 @@ fn return_batch_delete_lock_error_with_accounting(
|
||||
}
|
||||
|
||||
const EVENT_DETACHED_DELETE_FAILED: &str = "detached_delete_failed";
|
||||
const EVENT_BULK_DELETE_LOCK_ACQUIRE: &str = "bulk_delete_lock_acquire";
|
||||
const LOG_SUBSYSTEM_STORE_DELETE: &str = "store_delete";
|
||||
|
||||
/// A detached delete only fails to join when it panicked or the runtime is
|
||||
@@ -3568,10 +3567,12 @@ impl ECStore {
|
||||
}
|
||||
|
||||
/// Acquire the per-object write locks a bulk delete needs before any
|
||||
/// mutation. Each peer receives one batch of unresolved keys per round;
|
||||
/// retries share the original deadline and require confirmed cleanup of
|
||||
/// partial grants. Outcomes remain independent per key so other keys can
|
||||
/// proceed when a contended key exhausts its budget.
|
||||
/// mutation, one batched lock round for the whole key set. Bulk delete is
|
||||
/// the only multi-key write-lock holder, and the batched round is
|
||||
/// deadlock-free by construction (one `lock_batch` RPC per lock client,
|
||||
/// global key order at each server), so no optimistic wave or serial
|
||||
/// fallback is needed. Outcomes are per key: a locked-out key fails alone
|
||||
/// and the rest of the batch proceeds.
|
||||
async fn acquire_delete_objects_write_locks(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -3583,7 +3584,6 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let object_names = sorted_unique_delete_object_names(objects);
|
||||
let acquire_started = Instant::now();
|
||||
let acquired = self.pools[0].disk_set[0]
|
||||
.acquire_object_write_locks_batch(bucket, &object_names, get_lock_acquire_timeout())
|
||||
.await;
|
||||
@@ -3614,35 +3614,6 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
}
|
||||
if failures.is_empty() {
|
||||
debug!(
|
||||
event = EVENT_BULK_DELETE_LOCK_ACQUIRE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_STORE_DELETE,
|
||||
result = "success",
|
||||
stage = "raw_acquire",
|
||||
requested_objects = objects.len(),
|
||||
unique_keys = object_names.len(),
|
||||
successful_keys = guards.len(),
|
||||
failed_keys = failures.len(),
|
||||
elapsed_ms = acquire_started.elapsed().as_millis(),
|
||||
"Bulk delete namespace lock acquisition completed"
|
||||
);
|
||||
} else {
|
||||
warn!(
|
||||
event = EVENT_BULK_DELETE_LOCK_ACQUIRE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_STORE_DELETE,
|
||||
result = "partial_failure",
|
||||
stage = "raw_acquire",
|
||||
requested_objects = objects.len(),
|
||||
unique_keys = object_names.len(),
|
||||
successful_keys = guards.len(),
|
||||
failed_keys = failures.len(),
|
||||
elapsed_ms = acquire_started.elapsed().as_millis(),
|
||||
"Bulk delete namespace lock acquisition failed for some keys"
|
||||
);
|
||||
}
|
||||
opts.no_lock = true;
|
||||
for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
|
||||
opts.add_namespace_lock_lost_signal(signal);
|
||||
@@ -14434,7 +14405,8 @@ mod batch_delete_lock_tests {
|
||||
let second = run(second);
|
||||
|
||||
// Whatever mix of per-key lock outcomes the overlap produces, both
|
||||
// batches must finish within their acquisition and mutation budgets.
|
||||
// batches must finish: the batched round is deadlock-free by
|
||||
// construction (global key order at each lock server).
|
||||
for batch in [first, second] {
|
||||
let (_, errors, _) = tokio::time::timeout(Duration::from_secs(30), batch)
|
||||
.await
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "rustfs-gateway-adapter"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
homepage.workspace = true
|
||||
description = "RustFS implementations of the rustfs-gateway extension traits (credentials, policy facts, virtual hosts, bucket owner and request id)."
|
||||
keywords = ["rustfs", "gateway", "s3", "iam"]
|
||||
categories = ["web-programming", "authentication"]
|
||||
authors.workspace = true
|
||||
publish = false
|
||||
|
||||
[package.metadata.gateway]
|
||||
# Documentary only: this crate is a ring-2 consumer of the gateway (it depends on
|
||||
# RustFS business crates). The RustFS repository does not run the gateway's ring guard.
|
||||
ring = 2
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
http = { workspace = true }
|
||||
rustfs-gateway = { workspace = true }
|
||||
rustfs-credentials = { workspace = true }
|
||||
rustfs-iam = { workspace = true }
|
||||
rustfs-policy = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
rustfs-gateway-http = { workspace = true }
|
||||
time = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros", "rt"] }
|
||||
@@ -1,548 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`AuthzAuditSink`]: the legacy denial record for a refusal the gateway enforced.
|
||||
//!
|
||||
//! Responsible for: one [`GatewayDenialRecord`] for each authorization decision the gateway made
|
||||
//! at an enforcing stage that ended the request with a refusal (`Deny` or `Indeterminate`), after
|
||||
//! the legacy audit switch and the legacy denial rate limiter have both admitted it.
|
||||
//! NOT responsible for: the authoritative authorization decision. On the gateway stack that is the
|
||||
//! legacy access check in owned form, run by the handler before the use case, and it writes its own
|
||||
//! denial record (`DenialContext::record`, `rustfs/src/storage/access.rs`). The gateway
|
||||
//! `Authorizer` is only an earlier gate that may refuse what the legacy check certainly refuses, so
|
||||
//! a request it lets through is not decided yet, and a request it refuses never reaches the check.
|
||||
//! Nor is this module responsible for building the `rustfs_audit::AuditEntry`, filling its
|
||||
//! request-line fields, or dispatching it: that is the binary's [`DenialAuditSink`].
|
||||
//! Upstream: the gateway service, once per decision (`AuthzAuditSink::on_decision`). Downstream:
|
||||
//! the binary's [`DenialAuditSink`].
|
||||
//!
|
||||
//! # What is recorded, and what is not
|
||||
//!
|
||||
//! - An `Allow` is never recorded: at the early gate it is not the final decision, and a handler
|
||||
//! denial after it would contradict the record.
|
||||
//! - The missing-object visibility probe is never recorded. For `GetObject` the gateway also asks
|
||||
//! `s3:ListBucket`, only to choose between `404` and `403` for an absent object; that answer
|
||||
//! refuses nothing, yet it reaches the sink as an `Input` event of its own. Legacy keeps the same
|
||||
//! probe out of the audit (`suppress_denial_audit`, `rustfs/src/app/object/head.rs`).
|
||||
//! - Every other `Deny` or `Indeterminate` ended its request: a route-stage refusal is answered
|
||||
//! before the body is read, and an input-stage refusal either fails the stage or prevents the
|
||||
//! gateway core from constructing the authorized request. One refused request therefore yields
|
||||
//! exactly one record.
|
||||
//!
|
||||
//! # Telling the visibility probe apart
|
||||
//!
|
||||
//! At the pinned gateway rev every event's `resources` starts with the request the decision was
|
||||
//! about. For the route and input events that is the route target, whose `action` equals its
|
||||
//! `route_action` (`crates/gateway/src/service.rs`, the route request and every asked question,
|
||||
//! and the input stage's route request). The visibility event alone is built from the route
|
||||
//! target with only `action` replaced by `s3:ListBucket`, keeping the target's `route_action`
|
||||
//! (`crates/gateway/src/ext/authz_audit.rs`, `emit_input_safely`; `crates/gateway/src/ext/authorizer.rs`,
|
||||
//! `InputAuthzRequest::new`). An `Input` event whose first resource asks an action other than its
|
||||
//! route action is the visibility event.
|
||||
//!
|
||||
//! # Known limit: the denied action and resource
|
||||
//!
|
||||
//! `trigger` (and `api.name`), `api.bucket` and `api.object` are the event's `action`, `bucket` and
|
||||
//! `key`: the operation's declared action and primary target. The event carries no per-resource
|
||||
//! decision, so the sink cannot name a derived resource that was refused. This is exact only while
|
||||
//! every early-gate refusal concerns the declared action and target. An operation whose denial can
|
||||
//! concern a second resource or a version-specific action (a copy source, a versioned read) must
|
||||
//! therefore be decided by the handler, never refused by the early gate: legacy records those with
|
||||
//! the source bucket and object, and with `s3:GetObjectVersion`.
|
||||
//!
|
||||
//! # What may not reach a record
|
||||
//!
|
||||
//! The record is built field by field from the event's `action`, `bucket`, `key`, `request_id`,
|
||||
//! `decision` and the verified identity's access key id. Nothing is copied from `resources`,
|
||||
//! `subject`, `auth_scheme`, a version id or a copy-source identity, and nothing is formatted from a
|
||||
//! `Debug` implementation. The event carries no secret key, session token or signature.
|
||||
|
||||
use rustfs_gateway::{AuthzAuditEvent, AuthzAuditSink, AuthzStage, Decision};
|
||||
|
||||
/// `tags.denialReason` of a `Deny` the gateway enforced. The event carries the decision, not one of
|
||||
/// the legacy reasons (`iam_explicit_deny`, `no_policy_allows_action`, ...).
|
||||
pub const DENY_REASON: &str = "authorization_denied";
|
||||
/// `tags.denialReason` of an `Indeterminate` the gateway enforced: no policy reading could answer.
|
||||
pub const INDETERMINATE_REASON: &str = "authorization_indeterminate";
|
||||
|
||||
/// What the legacy denial rate limiter answered for one refusal.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum DenialAdmission {
|
||||
/// Record it; `suppressed` refusals of the same caller class were dropped since the previous
|
||||
/// record, and are reported as `tags.suppressedDenials` when non-zero.
|
||||
Emit {
|
||||
/// Refusals dropped since the previous record of this caller class.
|
||||
suppressed: u64,
|
||||
},
|
||||
/// Drop it.
|
||||
Suppress,
|
||||
}
|
||||
|
||||
/// One refusal the gateway enforced, in the fields of the legacy authorization-denial entry the
|
||||
/// event can supply.
|
||||
///
|
||||
/// The binary builds the entry as legacy `authorization_denial_entry` (`rustfs/src/storage/helper.rs`)
|
||||
/// does: `version` `1.0`, `event` `AuditS3Operation`, `trigger` and `api.name` [`Self::action`],
|
||||
/// `api.status` `failure`, `api.status_code` `403`, `error` `Access Denied`, `tags.denialReason`
|
||||
/// [`Self::denial_reason`], and `tags.suppressedDenials` only when [`Self::suppressed_denials`] is
|
||||
/// non-zero.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct GatewayDenialRecord {
|
||||
/// `trigger` and `api.name`: the operation's declared IAM action, `s3:GetObject`.
|
||||
pub action: String,
|
||||
/// `api.bucket`, when the request addressed one.
|
||||
pub bucket: Option<String>,
|
||||
/// `api.object`, when the request addressed one.
|
||||
pub object: Option<String>,
|
||||
/// `requestID`: the identifier the caller received.
|
||||
pub request_id: String,
|
||||
/// `accessKey`: the verified caller; `None` for an anonymous caller, as legacy omits it.
|
||||
pub access_key: Option<String>,
|
||||
/// `tags.denialReason`: [`DENY_REASON`] or [`INDETERMINATE_REASON`].
|
||||
pub denial_reason: &'static str,
|
||||
/// `tags.suppressedDenials`, from [`DenialAdmission::Emit`].
|
||||
pub suppressed_denials: u64,
|
||||
}
|
||||
|
||||
/// Where admitted refusals go. Implemented by the binary over the legacy denial auditor.
|
||||
///
|
||||
/// The three methods are called in order, at most once each per refusal, on the request's task,
|
||||
/// and must neither block nor panic.
|
||||
///
|
||||
/// # What the implementation owes the legacy entry
|
||||
///
|
||||
/// - [`Self::enabled`] is the legacy audit switch (`is_audit_module_enabled`), and [`Self::admit`]
|
||||
/// the process-wide legacy `DenialRateLimiter` itself, not a second instance: gateway and handler
|
||||
/// refusals share one token bucket per caller class.
|
||||
/// - `time`, `parentUser` and `requestClaims` are filled as legacy fills them.
|
||||
/// - `remotehost`, `userAgent`, `requestHost`, `requestPath` and `requestQuery` come from the
|
||||
/// request the outer layer saw. **`requestQuery` is not the URI query.** Legacy fills it with
|
||||
/// `rustfs_targets::net::extract_params_header` over the request *headers*, which replaces the
|
||||
/// value of every sensitive header (`authorization`, `x-amz-security-token`, the SSE-C key
|
||||
/// headers, ...) with the redaction marker. The raw URI query must never be put there: on a
|
||||
/// presigned request it carries `X-Amz-Signature`, `X-Amz-Credential` and
|
||||
/// `X-Amz-Security-Token`.
|
||||
pub trait DenialAuditSink: Send + Sync + 'static {
|
||||
/// Whether denial auditing is switched on. Asked first; `false` drops the refusal before the
|
||||
/// rate limiter is consulted.
|
||||
fn enabled(&self) -> bool;
|
||||
/// Asks the rate limiter for one refusal of this caller class. Asked before any record exists.
|
||||
fn admit(&self, anonymous: bool) -> DenialAdmission;
|
||||
/// Receives one admitted record.
|
||||
fn record(&self, record: GatewayDenialRecord);
|
||||
}
|
||||
|
||||
/// An [`AuthzAuditSink`] recording only the refusals the gateway enforced.
|
||||
pub struct RustfsAuthzAudit<S> {
|
||||
sink: S,
|
||||
}
|
||||
|
||||
impl<S: DenialAuditSink> RustfsAuthzAudit<S> {
|
||||
/// An audit sink reporting into `sink`.
|
||||
#[must_use]
|
||||
pub fn new(sink: S) -> Self {
|
||||
Self { sink }
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: DenialAuditSink> AuthzAuditSink for RustfsAuthzAudit<S> {
|
||||
fn on_decision(&self, event: &AuthzAuditEvent<'_>) {
|
||||
let denial_reason = match event.decision {
|
||||
Decision::Allow => return,
|
||||
Decision::Deny => DENY_REASON,
|
||||
Decision::Indeterminate => INDETERMINATE_REASON,
|
||||
};
|
||||
if is_visibility_probe(event) || !self.sink.enabled() {
|
||||
return;
|
||||
}
|
||||
let anonymous = event.identity.is_none();
|
||||
let DenialAdmission::Emit { suppressed } = self.sink.admit(anonymous) else {
|
||||
return;
|
||||
};
|
||||
self.sink.record(GatewayDenialRecord {
|
||||
action: event.action.to_owned(),
|
||||
bucket: event.bucket.map(|bucket| bucket.as_str().to_owned()),
|
||||
object: event.key.map(|key| key.as_str().to_owned()),
|
||||
request_id: event.request_id.as_str().to_owned(),
|
||||
access_key: event.identity.map(|identity| identity.access_key_id().to_owned()),
|
||||
denial_reason,
|
||||
suppressed_denials: suppressed,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether `event` is the missing-object visibility probe, which refuses nothing. See the module
|
||||
/// documentation for why the first resource identifies it.
|
||||
fn is_visibility_probe(event: &AuthzAuditEvent<'_>) -> bool {
|
||||
event.stage == AuthzStage::Input
|
||||
&& event
|
||||
.resources
|
||||
.first()
|
||||
.is_some_and(|first| first.action != first.route_action)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use core::time::Duration;
|
||||
use rustfs_gateway::{
|
||||
AuthSchemeRef, AuthzRequest, BucketName, Identity, ObjectKey, PolicySnapshot, RequestId, ResourceShape, SessionBinding,
|
||||
Subject, TargetOrigin,
|
||||
};
|
||||
use std::sync::Mutex;
|
||||
|
||||
const ACCESS_KEY: &str = "AKIDCALLER0001";
|
||||
const BUCKET: &str = "audit-bucket";
|
||||
const KEY: &str = "reports/q3.csv";
|
||||
|
||||
/// A sink with a scripted switch and limiter that records every call it receives.
|
||||
struct Probe {
|
||||
enabled: bool,
|
||||
admission: DenialAdmission,
|
||||
switch_asked: Mutex<usize>,
|
||||
limiter_asked: Mutex<Vec<bool>>,
|
||||
records: Mutex<Vec<GatewayDenialRecord>>,
|
||||
}
|
||||
|
||||
impl Probe {
|
||||
fn new(enabled: bool, admission: DenialAdmission) -> Self {
|
||||
Self {
|
||||
enabled,
|
||||
admission,
|
||||
switch_asked: Mutex::new(0),
|
||||
limiter_asked: Mutex::new(Vec::new()),
|
||||
records: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
fn open() -> Self {
|
||||
Self::new(true, DenialAdmission::Emit { suppressed: 0 })
|
||||
}
|
||||
|
||||
fn records(&self) -> Vec<GatewayDenialRecord> {
|
||||
self.records.lock().expect("not poisoned").clone()
|
||||
}
|
||||
|
||||
fn switch_asked(&self) -> usize {
|
||||
*self.switch_asked.lock().expect("not poisoned")
|
||||
}
|
||||
|
||||
fn limiter_asked(&self) -> Vec<bool> {
|
||||
self.limiter_asked.lock().expect("not poisoned").clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl DenialAuditSink for Probe {
|
||||
fn enabled(&self) -> bool {
|
||||
*self.switch_asked.lock().expect("not poisoned") += 1;
|
||||
self.enabled
|
||||
}
|
||||
|
||||
fn admit(&self, anonymous: bool) -> DenialAdmission {
|
||||
self.limiter_asked.lock().expect("not poisoned").push(anonymous);
|
||||
self.admission
|
||||
}
|
||||
|
||||
fn record(&self, record: GatewayDenialRecord) {
|
||||
self.records.lock().expect("not poisoned").push(record);
|
||||
}
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
request_id: RequestId,
|
||||
bucket: BucketName,
|
||||
key: ObjectKey,
|
||||
identity: Identity,
|
||||
snapshot: PolicySnapshot,
|
||||
}
|
||||
|
||||
fn fixture() -> Fixture {
|
||||
Fixture {
|
||||
request_id: RequestId::from_bits(0x5EED),
|
||||
bucket: BucketName::new(BUCKET).expect("a valid bucket"),
|
||||
key: ObjectKey::new(KEY).expect("a valid key"),
|
||||
identity: Identity::new(ACCESS_KEY).expect("a valid access key"),
|
||||
snapshot: PolicySnapshot::empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The route target of a GetObject, as the gateway builds it: its action is its route action.
|
||||
fn target<'a>(f: &'a Fixture, identity: Option<&'a Identity>) -> AuthzRequest<'a> {
|
||||
AuthzRequest {
|
||||
operation: "GetObject",
|
||||
action: "s3:GetObject",
|
||||
resource: ResourceShape::Object,
|
||||
bucket: Some(&f.bucket),
|
||||
key: Some(&f.key),
|
||||
copy_source_identity: None,
|
||||
version_id: None,
|
||||
route_action: "s3:GetObject",
|
||||
route_bucket: Some(&f.bucket),
|
||||
route_key: Some(&f.key),
|
||||
identity,
|
||||
target_origin: TargetOrigin::Path,
|
||||
subject: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// The visibility question, built from the route target as `InputAuthzRequest::new` builds it.
|
||||
fn visibility<'a>(route: &AuthzRequest<'a>) -> AuthzRequest<'a> {
|
||||
AuthzRequest {
|
||||
action: "s3:ListBucket",
|
||||
resource: ResourceShape::Bucket,
|
||||
copy_source_identity: None,
|
||||
version_id: None,
|
||||
..*route
|
||||
}
|
||||
}
|
||||
|
||||
/// A route or input event of a GetObject about `resources`.
|
||||
fn event<'a>(
|
||||
f: &'a Fixture,
|
||||
stage: AuthzStage,
|
||||
decision: Decision,
|
||||
resources: &'a [AuthzRequest<'a>],
|
||||
identity: Option<&'a Identity>,
|
||||
) -> AuthzAuditEvent<'a> {
|
||||
AuthzAuditEvent {
|
||||
request_id: &f.request_id,
|
||||
stage,
|
||||
operation: "GetObject",
|
||||
action: "s3:GetObject",
|
||||
resource: ResourceShape::Object,
|
||||
bucket: Some(&f.bucket),
|
||||
key: Some(&f.key),
|
||||
resources,
|
||||
auth_scheme: if identity.is_some() {
|
||||
AuthSchemeRef::SigV4Header
|
||||
} else {
|
||||
AuthSchemeRef::Anonymous
|
||||
},
|
||||
identity,
|
||||
target_origin: TargetOrigin::Path,
|
||||
policy_snapshot: Some(f.snapshot.id()),
|
||||
decision,
|
||||
elapsed: Duration::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
/// The visibility event, as `emit_input_safely` builds it from the input event.
|
||||
fn visibility_event<'a>(
|
||||
input: &AuthzAuditEvent<'a>,
|
||||
request: &'a AuthzRequest<'a>,
|
||||
decision: Decision,
|
||||
) -> AuthzAuditEvent<'a> {
|
||||
AuthzAuditEvent {
|
||||
stage: AuthzStage::Input,
|
||||
action: request.action,
|
||||
resource: request.resource,
|
||||
bucket: request.bucket,
|
||||
key: request.key,
|
||||
resources: std::slice::from_ref(request),
|
||||
decision,
|
||||
..*input
|
||||
}
|
||||
}
|
||||
|
||||
/// Drives the events of one GetObject through `audit` in the gateway's order: the route stage,
|
||||
/// then (when the route allowed) the input stage followed by its visibility question.
|
||||
fn get_object(
|
||||
audit: &RustfsAuthzAudit<Probe>,
|
||||
f: &Fixture,
|
||||
identity: Option<&Identity>,
|
||||
route: Decision,
|
||||
input: Decision,
|
||||
probe: Decision,
|
||||
) {
|
||||
let route_target = target(f, identity);
|
||||
let asked = [route_target];
|
||||
audit.on_decision(&event(f, AuthzStage::Route, route, &asked, identity));
|
||||
if route != Decision::Allow {
|
||||
return;
|
||||
}
|
||||
let input_event = event(f, AuthzStage::Input, input, &asked, identity);
|
||||
audit.on_decision(&input_event);
|
||||
let question = visibility(&route_target);
|
||||
audit.on_decision(&visibility_event(&input_event, &question, probe));
|
||||
}
|
||||
|
||||
fn expected(denial_reason: &'static str, access_key: Option<&str>, suppressed_denials: u64) -> GatewayDenialRecord {
|
||||
GatewayDenialRecord {
|
||||
action: "s3:GetObject".to_owned(),
|
||||
bucket: Some(BUCKET.to_owned()),
|
||||
object: Some(KEY.to_owned()),
|
||||
request_id: RequestId::from_bits(0x5EED).as_str().to_owned(),
|
||||
access_key: access_key.map(str::to_owned),
|
||||
denial_reason,
|
||||
suppressed_denials,
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive — a route-stage refusal is one record carrying the legacy denial entry's values,
|
||||
/// and `Indeterminate` keeps a reason of its own.
|
||||
#[test]
|
||||
fn p_route_refusal_is_one_record_with_the_legacy_values() {
|
||||
for (decision, reason) in [(Decision::Deny, DENY_REASON), (Decision::Indeterminate, INDETERMINATE_REASON)] {
|
||||
let f = fixture();
|
||||
let audit = RustfsAuthzAudit::new(Probe::open());
|
||||
get_object(&audit, &f, Some(&f.identity), decision, Decision::Allow, Decision::Allow);
|
||||
assert_eq!(audit.sink.records(), [expected(reason, Some(ACCESS_KEY), 0)], "{decision:?}");
|
||||
assert_eq!(audit.sink.limiter_asked(), [false], "{decision:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive — an input-stage refusal of a GetObject is exactly one record, though its
|
||||
/// visibility question was refused as well.
|
||||
#[test]
|
||||
fn p_enforcing_input_refusal_is_recorded_once() {
|
||||
let f = fixture();
|
||||
let audit = RustfsAuthzAudit::new(Probe::open());
|
||||
get_object(&audit, &f, Some(&f.identity), Decision::Allow, Decision::Deny, Decision::Deny);
|
||||
assert_eq!(audit.sink.records(), [expected(DENY_REASON, Some(ACCESS_KEY), 0)]);
|
||||
assert_eq!(audit.sink.limiter_asked(), [false]);
|
||||
}
|
||||
|
||||
/// Positive — the refusals the limiter dropped before this one are carried on the record.
|
||||
#[test]
|
||||
fn p_suppressed_count_is_carried() {
|
||||
let f = fixture();
|
||||
let audit = RustfsAuthzAudit::new(Probe::new(true, DenialAdmission::Emit { suppressed: 7 }));
|
||||
get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
|
||||
assert_eq!(audit.sink.records(), [expected(DENY_REASON, Some(ACCESS_KEY), 7)]);
|
||||
}
|
||||
|
||||
/// Negative — an `Allow` at either stage records nothing and asks neither the switch nor the
|
||||
/// limiter, so it cannot spend a token a real refusal needs.
|
||||
#[test]
|
||||
fn n_allow_is_never_recorded() {
|
||||
let f = fixture();
|
||||
let audit = RustfsAuthzAudit::new(Probe::open());
|
||||
let asked = [target(&f, Some(&f.identity))];
|
||||
for stage in [AuthzStage::Route, AuthzStage::Input] {
|
||||
audit.on_decision(&event(&f, stage, Decision::Allow, &asked, Some(&f.identity)));
|
||||
}
|
||||
assert!(audit.sink.records().is_empty());
|
||||
assert_eq!((audit.sink.switch_asked(), audit.sink.limiter_asked()), (0, Vec::new()));
|
||||
}
|
||||
|
||||
/// Negative — a GetObject that succeeds for a caller refused `s3:ListBucket` records nothing:
|
||||
/// the visibility question refuses no request, whatever it answered.
|
||||
#[test]
|
||||
fn n_visibility_probe_is_never_recorded() {
|
||||
for probe in [Decision::Deny, Decision::Indeterminate] {
|
||||
let f = fixture();
|
||||
let audit = RustfsAuthzAudit::new(Probe::open());
|
||||
get_object(&audit, &f, Some(&f.identity), Decision::Allow, Decision::Allow, probe);
|
||||
assert!(audit.sink.records().is_empty(), "{probe:?}");
|
||||
assert_eq!((audit.sink.switch_asked(), audit.sink.limiter_asked()), (0, Vec::new()), "{probe:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — with the audit switch off a refusal records nothing and the limiter is never
|
||||
/// asked, as legacy checks the switch first.
|
||||
#[test]
|
||||
fn n_audit_switch_off_records_nothing() {
|
||||
let f = fixture();
|
||||
let audit = RustfsAuthzAudit::new(Probe::new(false, DenialAdmission::Emit { suppressed: 0 }));
|
||||
get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
|
||||
assert!(audit.sink.records().is_empty());
|
||||
assert_eq!((audit.sink.switch_asked(), audit.sink.limiter_asked()), (1, Vec::new()));
|
||||
}
|
||||
|
||||
/// Negative — a refusal the limiter suppresses records nothing.
|
||||
#[test]
|
||||
fn n_rate_limiter_suppression_records_nothing() {
|
||||
let f = fixture();
|
||||
let audit = RustfsAuthzAudit::new(Probe::new(true, DenialAdmission::Suppress));
|
||||
get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
|
||||
assert!(audit.sink.records().is_empty());
|
||||
assert_eq!(audit.sink.limiter_asked(), [false]);
|
||||
}
|
||||
|
||||
/// Negative — an anonymous caller has no access key, not even an empty one, and is limited as
|
||||
/// the anonymous class; the authenticated refusal beside it is limited as the other class.
|
||||
#[test]
|
||||
fn n_anonymous_caller_has_no_access_key() {
|
||||
let f = fixture();
|
||||
let audit = RustfsAuthzAudit::new(Probe::open());
|
||||
get_object(&audit, &f, None, Decision::Deny, Decision::Allow, Decision::Allow);
|
||||
get_object(&audit, &f, Some(&f.identity), Decision::Deny, Decision::Allow, Decision::Allow);
|
||||
assert_eq!(
|
||||
audit.sink.records(),
|
||||
[expected(DENY_REASON, None, 0), expected(DENY_REASON, Some(ACCESS_KEY), 0)]
|
||||
);
|
||||
assert_eq!(audit.sink.limiter_asked(), [true, false]);
|
||||
}
|
||||
|
||||
/// Negative — values planted in the event's `resources`, version ids, subject, auth scheme,
|
||||
/// target origin and session binding leave the record exactly as it is without them, and none
|
||||
/// of their text reaches its `Debug` output.
|
||||
#[test]
|
||||
fn n_event_fields_outside_the_record_do_not_reach_it() {
|
||||
const SECRETS: [&str; 3] = [
|
||||
"wJalrXUtnFEMI/K7MDENG/bPxRfiCYSECRETKEY",
|
||||
"AQoDYXdzEJrSESSIONTOKEN",
|
||||
"fe5f80f77d5fa3beca038a248ff027d0445342fe2855ddc963176630326f1024",
|
||||
];
|
||||
const PLANTED_BUCKET: &str = "planted-source-bucket";
|
||||
const PLANTED_KEY: &str = "planted/source-key";
|
||||
let f = fixture();
|
||||
let source_bucket = BucketName::new(PLANTED_BUCKET).expect("a valid bucket");
|
||||
let source_key = ObjectKey::new(PLANTED_KEY).expect("a valid key");
|
||||
let binding = SessionBinding::new(SECRETS[1], 4_000_000_000).expect("a binding");
|
||||
let session = Identity::new(ACCESS_KEY).expect("a key").with_session_binding(binding);
|
||||
let subject = Subject::Caller;
|
||||
let planted_target = AuthzRequest {
|
||||
version_id: Some(SECRETS[0]),
|
||||
target_origin: TargetOrigin::Host,
|
||||
subject: Some(&subject),
|
||||
..target(&f, Some(&session))
|
||||
};
|
||||
let derived = AuthzRequest {
|
||||
action: "s3:GetObjectVersion",
|
||||
bucket: Some(&source_bucket),
|
||||
key: Some(&source_key),
|
||||
version_id: Some(SECRETS[2]),
|
||||
..planted_target
|
||||
};
|
||||
let planted_resources = [planted_target, derived];
|
||||
let plain_resources = [target(&f, Some(&f.identity))];
|
||||
for stage in [AuthzStage::Route, AuthzStage::Input] {
|
||||
let plain = RustfsAuthzAudit::new(Probe::open());
|
||||
plain.on_decision(&event(&f, stage, Decision::Deny, &plain_resources, Some(&f.identity)));
|
||||
let planted = RustfsAuthzAudit::new(Probe::open());
|
||||
planted.on_decision(&AuthzAuditEvent {
|
||||
auth_scheme: AuthSchemeRef::SigV4Presigned,
|
||||
target_origin: TargetOrigin::Host,
|
||||
..event(&f, stage, Decision::Deny, &planted_resources, Some(&session))
|
||||
});
|
||||
let records = planted.sink.records();
|
||||
assert_eq!(records, plain.sink.records(), "{stage:?}");
|
||||
assert_eq!(records.len(), 1, "{stage:?}");
|
||||
let rendered = format!("{records:?}");
|
||||
for planted_text in SECRETS.into_iter().chain([
|
||||
PLANTED_BUCKET,
|
||||
PLANTED_KEY,
|
||||
"GetObjectVersion",
|
||||
"Presigned",
|
||||
"Host",
|
||||
"Caller",
|
||||
"Authz",
|
||||
]) {
|
||||
assert!(!rendered.contains(planted_text), "{planted_text} reached the record: {rendered}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,215 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`BucketOwnerSource`]: the one owner RustFS reports, for a bucket that exists.
|
||||
//!
|
||||
//! Responsible for: confirming through a [`BucketLookup`] that a bucket exists, answering the
|
||||
//! single owner id the deployment reports ([`RustfsBucketOwners::new`]) for it, and turning every
|
||||
//! outcome that is not a usable owner into the gateway's one refusal, [`BucketOwnerError`].
|
||||
//! NOT responsible for: choosing the owner id (the binary passes the constant it already reports
|
||||
//! as the owner in listings), comparing it with `x-amz-expected-bucket-owner` (the gateway does,
|
||||
//! byte for byte), or authorizing anything.
|
||||
//! Upstream: the gateway's expected-owner check, which consults this source only when a request
|
||||
//! carries the header, after authorization allowed it. Downstream: the lookup the binary supplies
|
||||
//! over the ecstore bucket metadata system.
|
||||
//!
|
||||
//! # What an owner is in RustFS
|
||||
//!
|
||||
//! RustFS stores no per-bucket owner: bucket ACL writes are no-ops and nothing writes the ACL
|
||||
//! document, so the only owner RustFS reports anywhere (ListBuckets, ACL and multipart listings)
|
||||
//! is one constant. The source answers exactly that constant. A caller therefore passes the
|
||||
//! gateway's check only by presenting the owner id RustFS lists, which separates no tenants; it is
|
||||
//! a compatibility answer, and the constant's own documentation says it must not decide
|
||||
//! authorization.
|
||||
//!
|
||||
//! # Difference from legacy (ledger entry R-071)
|
||||
//!
|
||||
//! Legacy RustFS ignores `x-amz-expected-bucket-owner` entirely. On the gateway stack a request
|
||||
//! carrying it is served only when the value equals the reported owner id; otherwise it is `403
|
||||
//! AccessDenied`. A missing bucket and a backend failure with the header present are also `403`
|
||||
//! at the pinned gateway, where legacy answers `404` and `5xx`: [`BucketOwnerError`] has one value,
|
||||
//! so it cannot express either. [`RustfsBucketOwners::resolve`] keeps the two apart at this
|
||||
//! adapter's boundary so the gateway can surface them once it grows the classes.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{BoxFuture, BucketName, BucketOwnerError, BucketOwnerSource};
|
||||
|
||||
/// Why a bucket yielded no owner.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum OwnerLookupError {
|
||||
/// The bucket does not exist.
|
||||
NoSuchBucket,
|
||||
/// The metadata backend failed, could not be reached, or the owner is not usable (an empty
|
||||
/// owner id is a misconfiguration), so the bucket's state is unknown.
|
||||
Backend,
|
||||
}
|
||||
|
||||
/// The narrow slice of RustFS bucket metadata an owner answer needs: whether a bucket exists.
|
||||
///
|
||||
/// Exists so the source can be tested against a fake. The binary implements it over the ecstore
|
||||
/// bucket metadata system, mapping `BucketNotFound` to [`OwnerLookupError::NoSuchBucket`] and every
|
||||
/// other error to [`OwnerLookupError::Backend`].
|
||||
pub trait BucketLookup: Send + Sync + 'static {
|
||||
/// `Ok(())` when `bucket` exists.
|
||||
fn check<'a>(&'a self, bucket: &'a str) -> BoxFuture<'a, Result<(), OwnerLookupError>>;
|
||||
}
|
||||
|
||||
/// A [`BucketOwnerSource`] answering one owner id for every bucket that exists.
|
||||
pub struct RustfsBucketOwners<L> {
|
||||
lookup: L,
|
||||
owner: Arc<str>,
|
||||
}
|
||||
|
||||
impl<L: BucketLookup> RustfsBucketOwners<L> {
|
||||
/// A source answering `owner` for every bucket `lookup` finds.
|
||||
///
|
||||
/// An empty `owner` is no owner: every answer is then [`OwnerLookupError::Backend`].
|
||||
#[must_use]
|
||||
pub fn new(lookup: L, owner: impl Into<Arc<str>>) -> Self {
|
||||
Self {
|
||||
lookup,
|
||||
owner: owner.into(),
|
||||
}
|
||||
}
|
||||
|
||||
/// The owner of `bucket`, or why there is none. Missing and failing stay distinct here.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`OwnerLookupError::NoSuchBucket`] when the bucket does not exist;
|
||||
/// [`OwnerLookupError::Backend`] on any lookup failure or when the configured owner is empty.
|
||||
pub async fn resolve(&self, bucket: &str) -> Result<Arc<str>, OwnerLookupError> {
|
||||
self.lookup.check(bucket).await?;
|
||||
if self.owner.is_empty() {
|
||||
return Err(OwnerLookupError::Backend);
|
||||
}
|
||||
Ok(Arc::clone(&self.owner))
|
||||
}
|
||||
}
|
||||
|
||||
impl<L: BucketLookup> BucketOwnerSource for RustfsBucketOwners<L> {
|
||||
fn owner<'a>(&'a self, bucket: &'a BucketName) -> BoxFuture<'a, Result<Arc<str>, BucketOwnerError>> {
|
||||
Box::pin(async move {
|
||||
// The pinned gateway collapses both classes (R-071), so both are the one refusal.
|
||||
self.resolve(bucket.as_str())
|
||||
.await
|
||||
.map_err(|_: OwnerLookupError| BucketOwnerError::unavailable())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
/// Stands for the owner id RustFS reports; the binary passes its own constant.
|
||||
const OWNER: &str = "reported-owner-id";
|
||||
|
||||
struct Fake {
|
||||
answer: Result<(), OwnerLookupError>,
|
||||
asked: Mutex<Vec<String>>,
|
||||
}
|
||||
|
||||
impl Fake {
|
||||
fn answering(answer: Result<(), OwnerLookupError>) -> Self {
|
||||
Self {
|
||||
answer,
|
||||
asked: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BucketLookup for Fake {
|
||||
fn check<'a>(&'a self, bucket: &'a str) -> BoxFuture<'a, Result<(), OwnerLookupError>> {
|
||||
self.asked.lock().unwrap().push(bucket.to_owned());
|
||||
let answer = self.answer;
|
||||
Box::pin(async move { answer })
|
||||
}
|
||||
}
|
||||
|
||||
fn source(answer: Result<(), OwnerLookupError>) -> RustfsBucketOwners<Fake> {
|
||||
RustfsBucketOwners::new(Fake::answering(answer), OWNER)
|
||||
}
|
||||
|
||||
async fn gateway_answer(answer: Result<(), OwnerLookupError>) -> Result<Arc<str>, BucketOwnerError> {
|
||||
source(answer).owner(&BucketName::new("photos").expect("a bucket name")).await
|
||||
}
|
||||
|
||||
/// Positive: an existing bucket's owner is the configured id, byte for byte.
|
||||
#[tokio::test]
|
||||
async fn an_existing_bucket_has_the_configured_owner() {
|
||||
assert_eq!(gateway_answer(Ok(())).await.as_deref(), Ok(OWNER));
|
||||
}
|
||||
|
||||
/// Positive: every bucket gets the same owner, since RustFS stores none per bucket.
|
||||
#[tokio::test]
|
||||
async fn every_bucket_has_the_same_owner() {
|
||||
let source = source(Ok(()));
|
||||
assert_eq!(source.resolve("photos").await.as_deref(), Ok(OWNER));
|
||||
assert_eq!(source.resolve("backups").await.as_deref(), Ok(OWNER));
|
||||
}
|
||||
|
||||
/// Positive: the lookup is asked about exactly the bucket the gateway named.
|
||||
#[tokio::test]
|
||||
async fn the_lookup_is_asked_about_the_named_bucket() {
|
||||
let source = source(Ok(()));
|
||||
source
|
||||
.owner(&BucketName::new("photos").expect("a bucket name"))
|
||||
.await
|
||||
.expect("an owner");
|
||||
assert_eq!(*source.lookup.asked.lock().unwrap(), vec!["photos".to_owned()]);
|
||||
}
|
||||
|
||||
/// Negative: a missing bucket is reported as missing at the adapter boundary.
|
||||
#[tokio::test]
|
||||
async fn a_missing_bucket_is_reported_as_missing() {
|
||||
assert_eq!(
|
||||
source(Err(OwnerLookupError::NoSuchBucket)).resolve("photos").await,
|
||||
Err(OwnerLookupError::NoSuchBucket)
|
||||
);
|
||||
}
|
||||
|
||||
/// Negative: a failing backend is reported as a failure, never as an owner and never as missing.
|
||||
#[tokio::test]
|
||||
async fn a_backend_failure_is_reported_as_a_failure() {
|
||||
assert_eq!(
|
||||
source(Err(OwnerLookupError::Backend)).resolve("photos").await,
|
||||
Err(OwnerLookupError::Backend)
|
||||
);
|
||||
}
|
||||
|
||||
/// Negative: the pinned gateway sees one refusal for both, and never an owner.
|
||||
#[tokio::test]
|
||||
async fn the_gateway_is_refused_in_both_cases() {
|
||||
assert_eq!(
|
||||
gateway_answer(Err(OwnerLookupError::NoSuchBucket)).await,
|
||||
Err(BucketOwnerError::unavailable())
|
||||
);
|
||||
assert_eq!(gateway_answer(Err(OwnerLookupError::Backend)).await, Err(BucketOwnerError::unavailable()));
|
||||
}
|
||||
|
||||
/// Negative: an empty configured owner is no owner; it must not equal an empty assertion.
|
||||
#[tokio::test]
|
||||
async fn an_empty_owner_is_refused() {
|
||||
let source = RustfsBucketOwners::new(Fake::answering(Ok(())), "");
|
||||
assert_eq!(source.resolve("photos").await, Err(OwnerLookupError::Backend));
|
||||
assert_eq!(
|
||||
source.owner(&BucketName::new("photos").expect("a bucket name")).await,
|
||||
Err(BucketOwnerError::unavailable())
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,446 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`CredentialProvider`], answered by RustFS IAM.
|
||||
//!
|
||||
//! Responsible for: resolving one access key id to the secret the gateway verifies a signature
|
||||
//! with, and classifying every other outcome as either a business result
|
||||
//! ([`CredentialLookup::NotFound`], eligible for the gateway's negative cache) or an operational
|
||||
//! fault ([`ProviderError`], counted and never cached).
|
||||
//! NOT responsible for: authorization of any kind, session-token claim validation, or deciding
|
||||
//! what the client is answered with. The gateway answers every unusable credential identically.
|
||||
//!
|
||||
//! # Secret handling
|
||||
//!
|
||||
//! Secrets travel only inside the gateway's [`Credentials`], whose `Debug` redacts them. No type
|
||||
//! in this module derives or implements `Debug`, and no error produced here carries text: the
|
||||
//! IAM error is classified and dropped. A secret that fails the crate's effective-length rule
|
||||
//! (`crate::MIN_SECRET_KEY_LEN` bytes once trailing NULs are removed) is never handed to the
|
||||
//! gateway, whatever the lookup says about its validity.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{BoxFuture, CredentialLookup, CredentialProvider, Credentials, ProviderError, SessionBinding};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::store::Store;
|
||||
use rustfs_iam::sys::IamSys;
|
||||
use rustfs_policy::auth::{ACCOUNT_OFF, UserIdentity};
|
||||
|
||||
use crate::root::{RootSource, is_usable_secret};
|
||||
|
||||
/// The opaque issuer recorded in the [`rustfs_gateway::SessionBinding`] of every RustFS STS
|
||||
/// credential.
|
||||
pub const STS_SESSION_ISSUER: &str = "rustfs-sts";
|
||||
|
||||
/// Access keys with this prefix belong to Keystone token authentication, which never verifies an
|
||||
/// AWS signature.
|
||||
const KEYSTONE_ACCESS_KEY_PREFIX: &str = "keystone:";
|
||||
|
||||
/// The narrow slice of IAM that a credential lookup needs.
|
||||
///
|
||||
/// Exists so the adapter can be tested against a fake: IAM exports no in-memory backend.
|
||||
/// Implemented for [`IamSys`], whose `check_key` loads a user from storage on a cache miss.
|
||||
pub trait IamKeyLookup: Send + Sync + 'static {
|
||||
/// Whether the IAM store has finished loading.
|
||||
fn is_ready(&self) -> bool;
|
||||
|
||||
/// Resolves an access key, with [`IamSys::check_key`]'s contract: `Ok((None, _))` for an
|
||||
/// unknown key, `Ok((Some(identity), valid))` otherwise, where `valid` is false for a
|
||||
/// disabled, expired, or malformed credential.
|
||||
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>>;
|
||||
}
|
||||
|
||||
impl<T: Store> IamKeyLookup for IamSys<T> {
|
||||
fn is_ready(&self) -> bool {
|
||||
IamSys::is_ready(self)
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
Box::pin(IamSys::check_key(self, access_key))
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`CredentialProvider`] over RustFS IAM and the live root credentials.
|
||||
///
|
||||
/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
|
||||
/// policy source it was assembled with.
|
||||
pub struct IamCredentials {
|
||||
iam: Arc<dyn IamKeyLookup>,
|
||||
root: Arc<dyn RootSource>,
|
||||
}
|
||||
|
||||
impl IamCredentials {
|
||||
pub(crate) fn new(iam: Arc<dyn IamKeyLookup>, root: Arc<dyn RootSource>) -> Self {
|
||||
Self { iam, root }
|
||||
}
|
||||
|
||||
async fn resolve(&self, access_key: &str) -> Result<CredentialLookup, ProviderError> {
|
||||
// Deliberate change from the legacy s3s path (`rustfs/src/auth.rs` `get_secret_key`),
|
||||
// which answered a `keystone:` key with an empty secret. Keystone authenticates by token,
|
||||
// so there is no AWS secret to verify a signature against: the key is unknown here.
|
||||
if access_key.starts_with(KEYSTONE_ACCESS_KEY_PREFIX) {
|
||||
return Ok(CredentialLookup::NotFound);
|
||||
}
|
||||
|
||||
// Legacy answers every key with an internal error while the root credentials are not
|
||||
// initialized; without them no key can be classified as root or not.
|
||||
let Some(root) = self.root.current() else {
|
||||
return Err(ProviderError::Unavailable);
|
||||
};
|
||||
// Byte-exact, as the trait requires: the access key id is signed material.
|
||||
if root.access_key() == access_key {
|
||||
return Ok(CredentialLookup::Found(root.credentials().clone_credentials()));
|
||||
}
|
||||
|
||||
if !self.iam.is_ready() {
|
||||
return Err(ProviderError::Unavailable);
|
||||
}
|
||||
|
||||
match self.iam.check_key(access_key).await {
|
||||
// IAM refuses such a secret through `valid`, but this provider does not rely on every
|
||||
// lookup doing so: a short secret is never usable, so it is not a credential at all.
|
||||
Ok((Some(identity), _)) if !is_usable_secret(identity.credentials.secret_key.as_bytes()) => {
|
||||
Ok(CredentialLookup::NotFound)
|
||||
}
|
||||
Ok((Some(identity), valid)) => to_gateway_credentials(access_key, &identity, valid).map(CredentialLookup::Found),
|
||||
Ok((None, _)) => Ok(CredentialLookup::NotFound),
|
||||
Err(error) => classify_iam_error(&error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Classifies an IAM failure without keeping any of its text.
|
||||
fn classify_iam_error(error: &IamError) -> Result<CredentialLookup, ProviderError> {
|
||||
match error {
|
||||
IamError::NoSuchUser(_)
|
||||
| IamError::NoSuchServiceAccount(_)
|
||||
| IamError::NoSuchTempAccount(_)
|
||||
| IamError::NoSuchAccount(_) => Ok(CredentialLookup::NotFound),
|
||||
IamError::IamSysNotInitialized => Err(ProviderError::Unavailable),
|
||||
_ => Err(ProviderError::Backend),
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the gateway credential for one IAM identity. No authorization happens here: a
|
||||
/// disabled or invalid principal still resolves, marked disabled, and the gateway refuses it
|
||||
/// exactly as it refuses an unknown key.
|
||||
fn to_gateway_credentials(access_key: &str, identity: &UserIdentity, valid: bool) -> Result<Credentials, ProviderError> {
|
||||
let cred = &identity.credentials;
|
||||
let mut credentials = Credentials::new(access_key, cred.secret_key.as_bytes()).map_err(|_| ProviderError::Backend)?;
|
||||
|
||||
// A service account carries a claims token internally, but its clients never present it, so
|
||||
// it authenticates as a long-term credential (as `check_session_token` treats it in legacy).
|
||||
if !cred.session_token.is_empty() && !cred.is_service_account() {
|
||||
// A temporary credential without an expiry cannot become a gateway session, which always
|
||||
// ends. Refusing it as a fault is the fail-closed answer; a never-ending session is not.
|
||||
let expiration = cred.expiration.ok_or(ProviderError::Backend)?;
|
||||
let binding = SessionBinding::new(STS_SESSION_ISSUER, expiration.unix_timestamp()).map_err(|_| ProviderError::Backend)?;
|
||||
credentials = credentials
|
||||
.with_session(&cred.session_token, binding)
|
||||
.map_err(|_| ProviderError::Backend)?;
|
||||
}
|
||||
|
||||
if !valid || cred.status == ACCOUNT_OFF {
|
||||
credentials = credentials.disable();
|
||||
}
|
||||
Ok(credentials)
|
||||
}
|
||||
|
||||
impl CredentialProvider for IamCredentials {
|
||||
fn lookup<'a>(&'a self, access_key_id: &'a str) -> BoxFuture<'a, Result<CredentialLookup, ProviderError>> {
|
||||
Box::pin(self.resolve(access_key_id))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::root::{RootCell, RootCredentials};
|
||||
use rustfs_credentials::Credentials as IamCredential;
|
||||
use rustfs_gateway::RequestNow;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
const ACCESS_KEY: &str = "AKIDUSER0001";
|
||||
const SECRET: &str = "user-secret-key-0001";
|
||||
|
||||
type Answer = Box<dyn Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync>;
|
||||
|
||||
struct FakeIam {
|
||||
ready: bool,
|
||||
answer: Answer,
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl IamKeyLookup for FakeIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, Result<(Option<UserIdentity>, bool), IamError>> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let answer = (self.answer)();
|
||||
Box::pin(async move { answer })
|
||||
}
|
||||
}
|
||||
|
||||
fn fake(
|
||||
ready: bool,
|
||||
answer: impl Fn() -> Result<(Option<UserIdentity>, bool), IamError> + Send + Sync + 'static,
|
||||
) -> Arc<FakeIam> {
|
||||
Arc::new(FakeIam {
|
||||
ready,
|
||||
answer: Box::new(answer),
|
||||
calls: AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
fn user(status: &str) -> IamCredential {
|
||||
IamCredential {
|
||||
access_key: ACCESS_KEY.to_owned(),
|
||||
secret_key: SECRET.to_owned(),
|
||||
status: status.to_owned(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn found(cred: IamCredential, valid: bool) -> Result<(Option<UserIdentity>, bool), IamError> {
|
||||
Ok((Some(UserIdentity::new(cred)), valid))
|
||||
}
|
||||
|
||||
fn root_cell() -> Arc<RootCell> {
|
||||
let root = RootCredentials::new("rootadmin", "root-secret-key", "sts-signing-key").expect("valid root credentials");
|
||||
Arc::new(RootCell::new(Some(root)))
|
||||
}
|
||||
|
||||
async fn lookup(iam: Arc<FakeIam>, access_key: &str) -> Result<CredentialLookup, ProviderError> {
|
||||
IamCredentials::new(iam, root_cell()).lookup(access_key).await
|
||||
}
|
||||
|
||||
fn expect_found(result: Result<CredentialLookup, ProviderError>) -> Credentials {
|
||||
match result {
|
||||
Ok(CredentialLookup::Found(credentials)) => credentials,
|
||||
Ok(CredentialLookup::NotFound) => panic!("expected Found, got NotFound"),
|
||||
Err(error) => panic!("expected Found, got {error:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn is_not_found(result: &Result<CredentialLookup, ProviderError>) -> bool {
|
||||
matches!(result, Ok(CredentialLookup::NotFound))
|
||||
}
|
||||
|
||||
/// Positive — an enabled user resolves to exactly its secret, as a long-term credential.
|
||||
#[tokio::test]
|
||||
async fn credentials_found_user_resolves_to_its_secret() {
|
||||
let iam = fake(true, || found(user("on"), true));
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
assert_eq!(credentials.secret().expose(), SECRET.as_bytes());
|
||||
assert_eq!(credentials.identity().access_key_id(), ACCESS_KEY);
|
||||
assert!(!credentials.is_disabled());
|
||||
assert!(credentials.session_binding().is_none());
|
||||
}
|
||||
|
||||
/// Negative — a key IAM does not know is a business result, not a fault.
|
||||
#[tokio::test]
|
||||
async fn credentials_unknown_key_is_not_found() {
|
||||
let iam = fake(true, || Ok((None, false)));
|
||||
assert!(is_not_found(&lookup(iam, "AKIDUNKNOWN").await));
|
||||
}
|
||||
|
||||
/// Negative — every "no such principal" error IAM can surface is `NotFound`. Answering these
|
||||
/// as `Backend` would keep them out of the negative cache and count every guessed key as a
|
||||
/// backend fault.
|
||||
#[tokio::test]
|
||||
async fn credentials_no_such_principal_errors_are_not_found() {
|
||||
let errors: [fn() -> IamError; 4] = [
|
||||
|| IamError::NoSuchUser(ACCESS_KEY.to_owned()),
|
||||
|| IamError::NoSuchServiceAccount(ACCESS_KEY.to_owned()),
|
||||
|| IamError::NoSuchTempAccount(ACCESS_KEY.to_owned()),
|
||||
|| IamError::NoSuchAccount(ACCESS_KEY.to_owned()),
|
||||
];
|
||||
for error in errors {
|
||||
let iam = fake(true, move || Err(error()));
|
||||
let result = lookup(iam, ACCESS_KEY).await;
|
||||
assert!(is_not_found(&result), "{:?} must be NotFound", error());
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — any other IAM failure is a backend fault, never `NotFound` and never `Found`.
|
||||
#[tokio::test]
|
||||
async fn credentials_backend_error_is_backend() {
|
||||
let iam = fake(true, || Err(IamError::StringError("storage read failed".to_owned())));
|
||||
assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
|
||||
}
|
||||
|
||||
/// Negative — a `keystone:` key never reaches IAM and never resolves to a secret.
|
||||
///
|
||||
/// Deliberate change from the legacy s3s path, which answered such a key with an empty
|
||||
/// secret: Keystone authenticates by token, so there is no AWS secret to verify against.
|
||||
#[tokio::test]
|
||||
async fn credentials_keystone_prefix_is_not_found_without_iam() {
|
||||
let iam = fake(true, || found(user("on"), true));
|
||||
let result = lookup(iam.clone(), "keystone:project-user").await;
|
||||
assert!(is_not_found(&result));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Negative — a switched-off or otherwise invalid user resolves, but disabled, and the same
|
||||
/// user enabled does not.
|
||||
#[tokio::test]
|
||||
async fn credentials_disabled_user_is_disabled() {
|
||||
let off = expect_found(lookup(fake(true, || found(user("off"), false)), ACCESS_KEY).await);
|
||||
assert!(off.is_disabled());
|
||||
assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
|
||||
|
||||
let invalid = expect_found(lookup(fake(true, || found(user("on"), false)), ACCESS_KEY).await);
|
||||
assert!(invalid.is_disabled());
|
||||
|
||||
let on = expect_found(lookup(fake(true, || found(user("on"), true)), ACCESS_KEY).await);
|
||||
assert!(!on.is_disabled());
|
||||
}
|
||||
|
||||
/// Positive — an STS credential carries its token and its lifetime as one session binding.
|
||||
#[tokio::test]
|
||||
async fn credentials_sts_carries_session_binding() {
|
||||
let expires_at = 4_000_000_000_i64;
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "sts-session-token".to_owned();
|
||||
cred.parent_user = "parent".to_owned();
|
||||
cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
let binding = credentials.session_binding().expect("an STS credential has a session");
|
||||
assert_eq!(binding.expires_at_unix_seconds(), expires_at);
|
||||
assert_eq!(binding.issuer(), STS_SESSION_ISSUER);
|
||||
let now = RequestNow::from_unix_seconds(expires_at - 1);
|
||||
assert!(credentials.admit(Some(b"sts-session-token"), now).is_ok());
|
||||
assert!(credentials.admit(None, now).is_err());
|
||||
}
|
||||
|
||||
/// Negative — an STS credential with no expiry cannot be expressed as a session the gateway
|
||||
/// would ever expire, so it is a backend fault rather than a session that never ends.
|
||||
#[tokio::test]
|
||||
async fn credentials_sts_without_expiry_is_backend() {
|
||||
let iam = fake(true, || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "sts-session-token".to_owned();
|
||||
found(cred, true)
|
||||
});
|
||||
assert!(matches!(lookup(iam, ACCESS_KEY).await, Err(ProviderError::Backend)));
|
||||
}
|
||||
|
||||
/// Positive — a service account carries a token internally but authenticates as a long-term
|
||||
/// credential: clients never present that token.
|
||||
#[tokio::test]
|
||||
async fn credentials_service_account_is_long_term() {
|
||||
let iam = fake(true, || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "service-account-claims-token".to_owned();
|
||||
cred.parent_user = "parent".to_owned();
|
||||
cred.claims = Some([(rustfs_credentials::IAM_POLICY_CLAIM_NAME_SA.to_owned(), "inherited-policy".into())].into());
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
assert!(credentials.session_binding().is_none());
|
||||
assert!(credentials.admit(None, RequestNow::from_unix_seconds(0)).is_ok());
|
||||
}
|
||||
|
||||
/// Negative — before IAM is ready a lookup is `Unavailable`, not `NotFound`: a key must not
|
||||
/// be negatively cached because the store had not loaded yet.
|
||||
#[tokio::test]
|
||||
async fn credentials_iam_not_ready_is_unavailable() {
|
||||
let iam = fake(false, || found(user("on"), true));
|
||||
assert!(matches!(lookup(iam.clone(), ACCESS_KEY).await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let uninitialized = fake(true, || Err(IamError::IamSysNotInitialized));
|
||||
assert!(matches!(lookup(uninitialized, ACCESS_KEY).await, Err(ProviderError::Unavailable)));
|
||||
}
|
||||
|
||||
/// Positive — the injected root credentials resolve without IAM, even before it is ready, and
|
||||
/// nothing else is answered with the root secret.
|
||||
#[tokio::test]
|
||||
async fn credentials_root_resolves_without_iam() {
|
||||
let iam = fake(false, || found(user("on"), true));
|
||||
let provider = IamCredentials::new(iam.clone(), root_cell());
|
||||
let root = expect_found(provider.lookup("rootadmin").await);
|
||||
assert_eq!(root.secret().expose(), b"root-secret-key");
|
||||
assert!(matches!(provider.lookup("ROOTADMIN").await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Negative — a lookup that calls an identity valid but holds an empty or short secret is not
|
||||
/// trusted: such a secret is not a credential, so the key is `NotFound`, never `Found`.
|
||||
#[tokio::test]
|
||||
async fn credentials_short_secret_is_never_found() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.secret_key = weak.to_owned();
|
||||
found(cred, true)
|
||||
});
|
||||
let result = lookup(iam, ACCESS_KEY).await;
|
||||
assert!(is_not_found(&result), "{weak:?} must be NotFound");
|
||||
}
|
||||
for usable in ["12345678", "abc\0defg"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.secret_key = usable.to_owned();
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
assert_eq!(credentials.secret().expose(), usable.as_bytes());
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a switched-off account is disabled even when the lookup calls it valid.
|
||||
#[tokio::test]
|
||||
async fn credentials_off_status_alone_disables() {
|
||||
let off = expect_found(lookup(fake(true, || found(user("off"), true)), ACCESS_KEY).await);
|
||||
assert!(off.is_disabled());
|
||||
assert!(off.admit(None, RequestNow::from_unix_seconds(0)).is_err());
|
||||
}
|
||||
|
||||
/// Negative — an STS credential admits only its own session's token: the token of another
|
||||
/// session, or none, is refused.
|
||||
#[tokio::test]
|
||||
async fn credentials_sts_refuses_another_sessions_token() {
|
||||
let expires_at = 4_000_000_000_i64;
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user("on");
|
||||
cred.session_token = "session-token-a".to_owned();
|
||||
cred.parent_user = "parent".to_owned();
|
||||
cred.expiration = Some(OffsetDateTime::from_unix_timestamp(expires_at).expect("valid timestamp"));
|
||||
found(cred, true)
|
||||
});
|
||||
let credentials = expect_found(lookup(iam, ACCESS_KEY).await);
|
||||
let now = RequestNow::from_unix_seconds(expires_at - 1);
|
||||
assert!(credentials.admit(Some(b"session-token-b"), now).is_err());
|
||||
assert!(credentials.admit(Some(b"session-token-a"), now).is_ok());
|
||||
}
|
||||
|
||||
/// Negative — without root credentials no key can be classified, so every lookup is
|
||||
/// `Unavailable` and IAM is not asked.
|
||||
#[tokio::test]
|
||||
async fn credentials_without_root_are_unavailable() {
|
||||
let iam = fake(true, || found(user("on"), true));
|
||||
let provider = IamCredentials::new(iam.clone(), Arc::new(RootCell::new(None)));
|
||||
assert!(matches!(provider.lookup(ACCESS_KEY).await, Err(ProviderError::Unavailable)));
|
||||
assert!(matches!(provider.lookup("rootadmin").await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
}
|
||||
@@ -1,306 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`HostResolver`], built from RustFS's configured domain list.
|
||||
//!
|
||||
//! Responsible for: turning the `RUSTFS_SERVER_DOMAINS` list into the gateway's legacy RustFS
|
||||
//! virtual-host resolver.
|
||||
//! NOT responsible for: parsing a `Host` header, matching a domain, validating bucket names (all
|
||||
//! done by [`LegacyRustfsVirtualHosts`]), reading the environment, or deciding that a console
|
||||
//! deployment serves no domains (the caller passes an empty list then, as `s3_host_domains` in
|
||||
//! `rustfs/src/server/http.rs` returns `None`). An empty list needs no branch here:
|
||||
//! [`LegacyRustfsVirtualHosts`] built from no domain reads every host path-style, exactly as the
|
||||
//! gateway's `PathStyleOnly` does.
|
||||
//! Upstream: the RustFS configuration (`Config::server_domains`). Downstream: the gateway's
|
||||
//! `ServiceBuilder`, which holds the resolver as `Arc<dyn HostResolver>`.
|
||||
//!
|
||||
//! # What "the same as `MultiDomain`" means
|
||||
//!
|
||||
//! RustFS today hands the same list to the s3s framework's multi-domain host, which ignores ports
|
||||
//! on both sides when matching a domain, takes the whole prefix in front of a matched domain as
|
||||
//! the bucket, compares a domain exactly (so a host is matched on the case it arrives in), and,
|
||||
//! for a host no domain matches, falls back to CNAME-style: a host that is itself a bucket name
|
||||
//! (lower-cased) is that bucket. [`LegacyRustfsVirtualHosts`] reproduces exactly that.
|
||||
//!
|
||||
//! What reads **path-style**: an empty domain list; a configured domain itself, with or without a
|
||||
//! port; an IP address, or a socket address (`10.0.0.1`, `[::1]:9000`); a host that is a valid
|
||||
//! domain but, lower-cased, is no bucket name (it carries a port, is over 63 bytes, or starts
|
||||
//! with `xn--`). What reads as a **bucket**: anything under a configured domain, and any other
|
||||
//! bucket-shaped host, which is a bucket of its own (`localhost`, `badexample.com`, `EXAMPLE.COM`).
|
||||
//! What is **refused** before routing: a host that is no domain (`under_score.org`, a bare
|
||||
//! `[::1]`, a trailing dot), and a bucket under a domain that the bucket rules refuse
|
||||
//! (`PHOTOS.example.com`, `ab.example.com`). The tests below pin each of those readings.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{HostResolver, LegacyDomainError, LegacyRustfsVirtualHosts};
|
||||
|
||||
/// Builds the host resolver for RustFS's configured domain list.
|
||||
///
|
||||
/// An empty list reads every host path-style. Otherwise the list is read as legacy RustFS reads
|
||||
/// it: a later entry whose name without its port repeats an earlier one is dropped, and a list
|
||||
/// that legacy RustFS refuses at start-up is refused here.
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`LegacyDomainError`] for an entry that is not a domain, or for two entries where one equals or
|
||||
/// is a subdomain of the other.
|
||||
pub fn rustfs_host_resolver<I, S>(domains: I) -> Result<Arc<dyn HostResolver>, LegacyDomainError>
|
||||
where
|
||||
I: IntoIterator<Item = S>,
|
||||
S: AsRef<str>,
|
||||
{
|
||||
Ok(Arc::new(LegacyRustfsVirtualHosts::new(domains)?))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use http::Method;
|
||||
use rustfs_gateway::{Addressing, HostQuery, HostRefusal, TargetKind};
|
||||
|
||||
/// What the resolver reads `GET path` with `Host: host` as: the host-named bucket (`None` for
|
||||
/// path style), the routed target, and the refusal, if any.
|
||||
fn read(resolver: &dyn HostResolver, path: &str, host: &str) -> (Option<String>, TargetKind, Option<HostRefusal>) {
|
||||
let method = Method::GET;
|
||||
let request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(path)
|
||||
.header("host", host)
|
||||
.body(())
|
||||
.expect("a valid request");
|
||||
let effective = rustfs_gateway_http::effective_host(&request).expect("an acceptable host");
|
||||
let query = HostQuery {
|
||||
host: &effective,
|
||||
path,
|
||||
method: &method,
|
||||
};
|
||||
let resolved = resolver.resolve(&query);
|
||||
let bucket = match &resolved.addressing {
|
||||
Addressing::VirtualHosted { bucket, .. } => Some(bucket.as_str().to_owned()),
|
||||
Addressing::Path => None,
|
||||
};
|
||||
(bucket, resolved.target, resolver.refusal(&query))
|
||||
}
|
||||
|
||||
fn with_domains(domains: &[&str]) -> Arc<dyn HostResolver> {
|
||||
rustfs_host_resolver(domains).expect("a list RustFS accepts")
|
||||
}
|
||||
|
||||
fn bucket(domains: &[&str], host: &str) -> Option<String> {
|
||||
read(&*with_domains(domains), "/key", host).0
|
||||
}
|
||||
|
||||
/// Positive: a host under a configured domain names the bucket in front of it.
|
||||
#[test]
|
||||
fn host_under_domain_names_the_bucket() {
|
||||
assert_eq!(
|
||||
read(&*with_domains(&["example.com"]), "/key", "photos.example.com"),
|
||||
(Some("photos".to_owned()), TargetKind::Object, None)
|
||||
);
|
||||
assert_eq!(read(&*with_domains(&["example.com"]), "/", "photos.example.com").1, TargetKind::Bucket);
|
||||
}
|
||||
|
||||
/// Positive: the port on the host, or on the configured domain, does not take part in matching.
|
||||
#[test]
|
||||
fn ports_are_ignored_on_both_sides() {
|
||||
assert_eq!(bucket(&["example.com"], "photos.example.com:9000").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&["example.com:9000"], "photos.example.com").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&["example.com:9000"], "photos.example.com:1234").as_deref(), Some("photos"));
|
||||
}
|
||||
|
||||
/// Positive: the whole prefix in front of the domain is the bucket, dots kept.
|
||||
#[test]
|
||||
fn whole_prefix_is_the_bucket() {
|
||||
assert_eq!(bucket(&["example.com"], "my.dotted.bkt.example.com").as_deref(), Some("my.dotted.bkt"));
|
||||
}
|
||||
|
||||
/// Positive: with several domains, each one resolves its own subdomains.
|
||||
#[test]
|
||||
fn every_configured_domain_resolves() {
|
||||
let domains = ["example.com", "s3.local"];
|
||||
assert_eq!(bucket(&domains, "abc.example.com").as_deref(), Some("abc"));
|
||||
assert_eq!(bucket(&domains, "xyz.s3.local").as_deref(), Some("xyz"));
|
||||
}
|
||||
|
||||
/// Positive: the domain itself is the endpoint, so the bucket comes from the path.
|
||||
#[test]
|
||||
fn the_domain_itself_is_path_style() {
|
||||
assert_eq!(bucket(&["example.com"], "example.com"), None);
|
||||
assert_eq!(bucket(&["example.com"], "example.com:9000"), None);
|
||||
}
|
||||
|
||||
/// Negative: no configured domain means path style only, for every host shape.
|
||||
#[test]
|
||||
fn no_domain_is_path_style_only() {
|
||||
for host in ["photos.example.com", "localhost", "localhost:9000", "10.0.0.1"] {
|
||||
assert_eq!(bucket(&[], host), None, "{host}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: an address, and a host carrying a port that no domain matches, are path style.
|
||||
#[test]
|
||||
fn addresses_and_unmatched_ports_are_path_style() {
|
||||
for host in ["10.0.0.1", "10.0.0.1:9000", "localhost:9000", "other.org:9000"] {
|
||||
assert_eq!(bucket(&["example.com"], host), None, "{host}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: a host that only ends in the same letters as a domain is not under it. It is not
|
||||
/// stripped to `bad`; the only reading it gets is the CNAME-style one, the whole host.
|
||||
#[test]
|
||||
fn suffix_lookalike_is_not_under_the_domain() {
|
||||
assert_eq!(bucket(&["example.com"], "badexample.com").as_deref(), Some("badexample.com"));
|
||||
assert_eq!(bucket(&["example.com"], "example.com.evil.org").as_deref(), Some("example.com.evil.org"));
|
||||
}
|
||||
|
||||
/// Negative: matching a configured domain is exact on case, as the legacy multi-domain host does it, so a
|
||||
/// mixed-case host is not stripped to its prefix; it is the lower-cased whole host.
|
||||
#[test]
|
||||
fn mixed_case_host_is_not_stripped_to_a_prefix() {
|
||||
assert_eq!(bucket(&["example.com"], "Photos.Example.COM").as_deref(), Some("photos.example.com"));
|
||||
}
|
||||
|
||||
/// Negative: a host that is no domain at all is refused, not routed.
|
||||
#[test]
|
||||
fn a_host_that_is_no_domain_is_refused() {
|
||||
let resolver = with_domains(&["example.com"]);
|
||||
assert_eq!(read(&*resolver, "/k", "under_score.org").2, Some(HostRefusal::UnusableHost));
|
||||
}
|
||||
|
||||
/// Negative: a bucket the host names that bucket rules refuse is refused.
|
||||
#[test]
|
||||
fn a_host_named_bucket_the_rules_refuse_is_refused() {
|
||||
let resolver = with_domains(&["example.com"]);
|
||||
assert_eq!(read(&*resolver, "/k", "ab.example.com").2, Some(HostRefusal::RefusedBucket));
|
||||
assert_eq!(read(&*resolver, "/k", "bad_host.example.com").2, Some(HostRefusal::RefusedBucket));
|
||||
}
|
||||
|
||||
/// Negative: a list legacy RustFS refuses at start-up is refused.
|
||||
#[test]
|
||||
fn domain_lists_legacy_rustfs_refuses_are_refused() {
|
||||
for list in [
|
||||
vec!["example.com", "a.example.com"],
|
||||
vec!["example.com", "com"],
|
||||
vec!["bad_domain"],
|
||||
vec![""],
|
||||
vec!["example.com:notaport"],
|
||||
] {
|
||||
assert!(rustfs_host_resolver(&list).is_err(), "{list:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive: a later repeat of a name, whatever its port, is dropped rather than refused, as
|
||||
/// `s3_host_domains` drops it before the legacy multi-domain host sees the list.
|
||||
#[test]
|
||||
fn a_repeat_of_a_domain_is_dropped() {
|
||||
assert_eq!(
|
||||
bucket(&["example.com:9411", "example.com:9000"], "xyz.example.com").as_deref(),
|
||||
Some("xyz")
|
||||
);
|
||||
}
|
||||
|
||||
/// The refusal alone, for hosts whose bucket reading is not the point.
|
||||
fn refusal(domains: &[&str], host: &str) -> Option<HostRefusal> {
|
||||
read(&*with_domains(domains), "/k", host).2
|
||||
}
|
||||
|
||||
/// Positive: a bucket-shaped host no domain matches is a bucket of its own, `localhost` and
|
||||
/// an upper-case spelling of the domain among them.
|
||||
#[test]
|
||||
fn bucket_shaped_hosts_outside_every_domain_are_their_own_bucket() {
|
||||
assert_eq!(bucket(&["example.com"], "localhost").as_deref(), Some("localhost"));
|
||||
assert_eq!(bucket(&["example.com"], "EXAMPLE.COM").as_deref(), Some("example.com"));
|
||||
assert_eq!(bucket(&["example.com"], "cdn.other.org").as_deref(), Some("cdn.other.org"));
|
||||
}
|
||||
|
||||
/// Negative: a bucket the rules refuse under a domain is refused, and reads no bucket.
|
||||
#[test]
|
||||
fn an_upper_case_bucket_under_the_domain_is_refused() {
|
||||
assert_eq!(bucket(&["example.com"], "PHOTOS.example.com"), None);
|
||||
assert_eq!(refusal(&["example.com"], "PHOTOS.example.com"), Some(HostRefusal::RefusedBucket));
|
||||
}
|
||||
|
||||
/// Negative: an IPv6 literal. The bare form is no domain and is refused; with a port it is a
|
||||
/// socket address and reads path-style.
|
||||
#[test]
|
||||
fn ipv6_literals_follow_the_address_rules() {
|
||||
assert_eq!(refusal(&["example.com"], "[::1]"), Some(HostRefusal::UnusableHost));
|
||||
assert_eq!(bucket(&["example.com"], "[::1]:9000"), None);
|
||||
assert_eq!(refusal(&["example.com"], "[::1]:9000"), None);
|
||||
}
|
||||
|
||||
/// Negative: a trailing root dot makes the host no domain, so it is refused, not stripped.
|
||||
#[test]
|
||||
fn a_trailing_dot_is_refused() {
|
||||
assert_eq!(refusal(&["example.com"], "photos.example.com."), Some(HostRefusal::UnusableHost));
|
||||
assert_eq!(bucket(&["example.com"], "photos.example.com."), None);
|
||||
}
|
||||
|
||||
/// Negative: an empty label never reaches the resolver; the gateway's wire layer refuses the
|
||||
/// host first (legacy answers 400 as well).
|
||||
#[test]
|
||||
fn an_empty_label_is_refused_before_resolution() {
|
||||
for host in ["a..b.example.com", ".example.com"] {
|
||||
let request = http::Request::builder()
|
||||
.uri("/k")
|
||||
.header("host", host)
|
||||
.body(())
|
||||
.expect("a valid request");
|
||||
assert!(rustfs_gateway_http::effective_host(&request).is_err(), "{host}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: a punycode label is no bucket under a domain; outside every domain the host is
|
||||
/// a valid domain that is no bucket name, so it reads path-style.
|
||||
#[test]
|
||||
fn a_punycode_label_is_never_a_bucket() {
|
||||
assert_eq!(refusal(&["example.com"], "xn--bcher-kva.example.com"), Some(HostRefusal::RefusedBucket));
|
||||
assert_eq!(bucket(&["example.com"], "xn--bcher-kva.de"), None);
|
||||
assert_eq!(refusal(&["example.com"], "xn--bcher-kva.de"), None);
|
||||
}
|
||||
|
||||
/// Negative: a label over 63 bytes is no bucket, under a domain or outside one.
|
||||
#[test]
|
||||
fn a_label_over_63_bytes_is_never_a_bucket() {
|
||||
let long = "a".repeat(64);
|
||||
let under = format!("{long}.example.com");
|
||||
assert_eq!(refusal(&["example.com"], &under), Some(HostRefusal::RefusedBucket));
|
||||
let outside = format!("{long}.org");
|
||||
assert_eq!(bucket(&["example.com"], &outside), None);
|
||||
assert_eq!(refusal(&["example.com"], &outside), None);
|
||||
let at_limit = "a".repeat(63);
|
||||
assert_eq!(bucket(&["example.com"], &format!("{at_limit}.example.com")), Some(at_limit));
|
||||
}
|
||||
|
||||
/// Positive and negative: two configured domains where one merely ends in the other's letters
|
||||
/// both work, and each domain itself is path-style.
|
||||
#[test]
|
||||
fn a_domain_and_its_suffix_lookalike_are_both_served() {
|
||||
let domains = ["example.com", "badexample.com"];
|
||||
assert_eq!(bucket(&domains, "photos.example.com").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&domains, "photos.badexample.com").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&domains, "badexample.com"), None);
|
||||
assert_eq!(bucket(&domains, "bad.example.com").as_deref(), Some("bad"));
|
||||
}
|
||||
|
||||
/// Negative: a configured domain is matched exactly, case included.
|
||||
#[test]
|
||||
fn a_configured_domain_is_matched_on_its_case() {
|
||||
assert_eq!(bucket(&["Example.COM"], "photos.Example.COM").as_deref(), Some("photos"));
|
||||
assert_eq!(bucket(&["Example.COM"], "photos.example.com").as_deref(), Some("photos.example.com"));
|
||||
assert_eq!(bucket(&["Example.COM"], "Example.COM"), None);
|
||||
}
|
||||
}
|
||||
@@ -1,107 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! RustFS implementations of the `rustfs-gateway` extension traits.
|
||||
//!
|
||||
//! This is the gateway's ring 2: it may depend on RustFS business crates, and nothing in the
|
||||
//! gateway depends on it. Each module adapts one extension point to the RustFS subsystem that
|
||||
//! owns the answer.
|
||||
|
||||
pub mod authz_audit;
|
||||
pub mod bucket_owner;
|
||||
pub mod credentials;
|
||||
pub mod host;
|
||||
pub mod policy;
|
||||
pub mod root;
|
||||
pub mod trace;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::{Authenticator, CredentialProvider, ServiceBuilder};
|
||||
|
||||
pub use authz_audit::{DenialAdmission, DenialAuditSink, GatewayDenialRecord, RustfsAuthzAudit};
|
||||
pub use bucket_owner::{BucketLookup, OwnerLookupError, RustfsBucketOwners};
|
||||
pub use credentials::{IamCredentials, IamKeyLookup, STS_SESSION_ISSUER};
|
||||
pub use host::rustfs_host_resolver;
|
||||
pub use policy::{IamPolicySource, IamPrincipalLookup, PrincipalKind, RustfsPrincipal};
|
||||
pub use root::{MIN_SECRET_KEY_LEN, RootCredentials, RootCredentialsError, RootSource};
|
||||
pub use trace::{AttachOutcome, RequestIdCarrier, attach_host_request_id, host_request_id};
|
||||
|
||||
/// The IAM-backed credential provider and policy source, assembled together.
|
||||
///
|
||||
/// Both halves read the same IAM handle and the same live [`RootSource`], so a key cannot be root
|
||||
/// to one and an ordinary user to the other. Neither half can be taken out on its own: the fields
|
||||
/// are private, and [`IamAdapters::install`] consumes the pair and hands both to one builder.
|
||||
///
|
||||
/// Mixing the halves of two assemblies does not compile:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// fn mix(a: rustfs_gateway_adapter::IamAdapters, b: rustfs_gateway_adapter::IamAdapters) {
|
||||
/// let _mixed = (a.credentials, b.policy);
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Nor does building a half outside the pair:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// use std::sync::Arc;
|
||||
/// use rustfs_gateway_adapter::{IamPolicySource, IamPrincipalLookup, RootSource};
|
||||
/// fn build(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) {
|
||||
/// let _alone = IamPolicySource::new(iam, root);
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Installing the pair is the one supported path:
|
||||
///
|
||||
/// ```no_run
|
||||
/// use std::sync::Arc;
|
||||
/// use rustfs_gateway::{RegionSet, ServiceBuilder, SigV4Authenticator};
|
||||
/// use rustfs_gateway_adapter::{IamAdapters, IamPrincipalLookup, RootSource};
|
||||
/// fn assemble(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>, regions: RegionSet) -> ServiceBuilder {
|
||||
/// IamAdapters::new(iam, root).install(ServiceBuilder::new(), |provider| SigV4Authenticator::new(provider, regions))
|
||||
/// }
|
||||
/// ```
|
||||
pub struct IamAdapters {
|
||||
credentials: IamCredentials,
|
||||
policy: IamPolicySource,
|
||||
}
|
||||
|
||||
impl IamAdapters {
|
||||
/// Assembles both adapters over one IAM handle and one root source. The binary passes a
|
||||
/// source that reads its live root credentials on every call (see [`RootSource`]).
|
||||
#[must_use]
|
||||
pub fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
|
||||
Self {
|
||||
credentials: IamCredentials::new(iam.clone(), root.clone()),
|
||||
policy: IamPolicySource::new(iam, root),
|
||||
}
|
||||
}
|
||||
|
||||
/// Installs both halves into `builder`: the credential provider through the authenticator
|
||||
/// `authenticator` builds over it (for example `SigV4Authenticator::new(provider, regions)`
|
||||
/// with the RustFS profile applied), and the policy source.
|
||||
///
|
||||
/// Consumes the pair and gives both halves to this one builder. The `authenticator` closure
|
||||
/// receives the provider handle and may keep a clone of it; nothing here prevents that, so
|
||||
/// the closure must not hand it to a second builder.
|
||||
#[must_use]
|
||||
pub fn install<A: Authenticator>(
|
||||
self,
|
||||
builder: ServiceBuilder,
|
||||
authenticator: impl FnOnce(Arc<dyn CredentialProvider>) -> A,
|
||||
) -> ServiceBuilder {
|
||||
let authenticator = authenticator(Arc::new(self.credentials));
|
||||
builder.authenticator(authenticator).policy_source(self.policy)
|
||||
}
|
||||
}
|
||||
@@ -1,779 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The gateway's [`PolicySource`], answered by RustFS IAM.
|
||||
//!
|
||||
//! Responsible for: the one reading of the caller's IAM facts a request is judged against — which
|
||||
//! principal kind it is, the user it acts for, its groups, its token claims, and whether it owns
|
||||
//! the deployment — packed as a [`RustfsPrincipal`] into the gateway's [`PolicySnapshot`].
|
||||
//! NOT responsible for: evaluating any policy, choosing which policy documents apply (that stays
|
||||
//! in `IamSys::is_allowed`, which owns the STS, OIDC, session-policy and service-account rules), or
|
||||
//! verifying the signature or the session token (the gateway did both before calling this).
|
||||
//! Upstream: the gateway's verified [`Identity`]. Downstream: the RustFS authorizer, which reads
|
||||
//! the payload back with `PolicySnapshot::get::<RustfsPrincipal>()`.
|
||||
//!
|
||||
//! # Failure classes
|
||||
//!
|
||||
//! Every refusal is a [`PolicyError`] with a constant label and nothing from IAM or the request:
|
||||
//! IAM not ready, not initialized, or failing, or root credentials not initialized, is
|
||||
//! [`PolicyError::unavailable`]; a principal IAM no longer knows is [`UNKNOWN_PRINCIPAL`]; a
|
||||
//! disabled one, or one whose secret fails the effective-length rule, is
|
||||
//! [`DISABLED_PRINCIPAL`]; an identity
|
||||
//! whose credential kind disagrees with IAM's record is [`SESSION_MISMATCH`]; a token whose claims
|
||||
//! cannot be read is [`PolicyError::malformed`]. None of them is ever an empty snapshot: an empty
|
||||
//! snapshot is what an anonymous caller gets, and answering a fault with it would judge an
|
||||
//! authenticated caller as anonymous.
|
||||
//!
|
||||
//! # Secret handling
|
||||
//!
|
||||
//! [`RustfsPrincipal`] holds no secret key and no session token, and implements no `Debug`, so a
|
||||
//! snapshot formatted into a log cannot print its claims either. The root credentials and the STS
|
||||
//! token-signing key are read per request from the shared [`RootSource`] and never stored here.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_credentials::Credentials as IamCredential;
|
||||
use rustfs_gateway::{BoxFuture, Identity, PolicyError, PolicySnapshot, PolicySource};
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::store::Store;
|
||||
use rustfs_iam::sys::{
|
||||
IamSys, SESSION_POLICY_NAME, get_claims_from_token_with_secret, get_claims_from_token_with_secret_allow_missing_exp,
|
||||
is_rustfs_oidc_claims,
|
||||
};
|
||||
use rustfs_policy::auth::ACCOUNT_OFF;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::credentials::IamKeyLookup;
|
||||
use crate::root::{RootCredentials, RootSource, is_usable_secret};
|
||||
|
||||
/// [`PolicyError`] label: IAM does not know the authenticated principal (any longer).
|
||||
pub const UNKNOWN_PRINCIPAL: &str = "unknown-principal";
|
||||
/// [`PolicyError`] label: IAM holds the principal disabled, expired, or otherwise invalid.
|
||||
pub const DISABLED_PRINCIPAL: &str = "disabled-principal";
|
||||
/// [`PolicyError`] label: the identity's credential kind (session or long-term) disagrees with
|
||||
/// what IAM holds for its access key.
|
||||
pub const SESSION_MISMATCH: &str = "session-mismatch";
|
||||
|
||||
/// The narrow slice of IAM a policy reading needs, on top of [`IamKeyLookup`].
|
||||
///
|
||||
/// Implemented for [`IamSys`]; exists so the source can be tested against a fake.
|
||||
pub trait IamPrincipalLookup: IamKeyLookup {
|
||||
/// The built-in IAM groups `user` belongs to now, as `IamSys::user_group_memberships`
|
||||
/// answers. These may include groups granted after a credential was issued.
|
||||
fn group_memberships(&self, user: &str) -> Vec<String>;
|
||||
}
|
||||
|
||||
impl<T: Store> IamPrincipalLookup for IamSys<T> {
|
||||
fn group_memberships(&self, user: &str) -> Vec<String> {
|
||||
IamSys::user_group_memberships(self, user)
|
||||
}
|
||||
}
|
||||
|
||||
/// Which kind of credential the principal authenticated with.
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PrincipalKind {
|
||||
/// The deployment's root credentials.
|
||||
Root,
|
||||
/// A long-term IAM user.
|
||||
User,
|
||||
/// A service account, acting for its parent user.
|
||||
ServiceAccount,
|
||||
/// An STS temporary credential, acting for its parent user.
|
||||
Temporary,
|
||||
}
|
||||
|
||||
/// The caller's IAM facts: the payload of every [`PolicySnapshot`] [`IamPolicySource`] produces.
|
||||
///
|
||||
/// Carries no secret key and no session token, and deliberately implements no `Debug`.
|
||||
pub struct RustfsPrincipal {
|
||||
kind: PrincipalKind,
|
||||
account: String,
|
||||
username: String,
|
||||
parent_user: String,
|
||||
credential_groups: Option<Vec<String>>,
|
||||
group_memberships: Vec<String>,
|
||||
claims: Option<HashMap<String, Value>>,
|
||||
is_owner: bool,
|
||||
}
|
||||
|
||||
impl RustfsPrincipal {
|
||||
/// The kind of credential the caller authenticated with.
|
||||
#[must_use]
|
||||
pub fn kind(&self) -> PrincipalKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
/// The access key the caller signed with (IAM `Args::account`).
|
||||
#[must_use]
|
||||
pub fn account(&self) -> &str {
|
||||
&self.account
|
||||
}
|
||||
|
||||
/// The user the caller acts as: the parent user for an STS credential or a service account,
|
||||
/// the access key otherwise.
|
||||
#[must_use]
|
||||
pub fn username(&self) -> &str {
|
||||
&self.username
|
||||
}
|
||||
|
||||
/// The parent user IAM records for the credential; empty for root and long-term users.
|
||||
#[must_use]
|
||||
pub fn parent_user(&self) -> &str {
|
||||
&self.parent_user
|
||||
}
|
||||
|
||||
/// The groups recorded on the credential itself (IAM `Args::groups`).
|
||||
#[must_use]
|
||||
pub fn credential_groups(&self) -> &Option<Vec<String>> {
|
||||
&self.credential_groups
|
||||
}
|
||||
|
||||
/// The groups IAM currently lists for [`Self::username`], read once with the snapshot.
|
||||
#[must_use]
|
||||
pub fn group_memberships(&self) -> &[String] {
|
||||
&self.group_memberships
|
||||
}
|
||||
|
||||
/// The claims the credential's token carries; `None` when it has none.
|
||||
#[must_use]
|
||||
pub fn claims(&self) -> Option<&HashMap<String, Value>> {
|
||||
self.claims.as_ref()
|
||||
}
|
||||
|
||||
/// Whether the caller owns the deployment (IAM `Args::is_owner`): root, or a credential of
|
||||
/// root that no session policy or external OIDC identity narrows.
|
||||
#[must_use]
|
||||
pub fn is_owner(&self) -> bool {
|
||||
self.is_owner
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`PolicySource`] over RustFS IAM and the live root credentials.
|
||||
///
|
||||
/// Built only by [`crate::IamAdapters::new`], so it always reads the same [`RootSource`] as the
|
||||
/// credential provider it was assembled with.
|
||||
pub struct IamPolicySource {
|
||||
iam: Arc<dyn IamPrincipalLookup>,
|
||||
root: Arc<dyn RootSource>,
|
||||
}
|
||||
|
||||
impl IamPolicySource {
|
||||
pub(crate) fn new(iam: Arc<dyn IamPrincipalLookup>, root: Arc<dyn RootSource>) -> Self {
|
||||
Self { iam, root }
|
||||
}
|
||||
|
||||
async fn read(&self, identity: Option<&Identity>) -> Result<PolicySnapshot, PolicyError> {
|
||||
let Some(identity) = identity else {
|
||||
return Ok(PolicySnapshot::empty());
|
||||
};
|
||||
let principal = self.principal(identity).await?;
|
||||
Ok(PolicySnapshot::of(Arc::new(principal)))
|
||||
}
|
||||
|
||||
async fn principal(&self, identity: &Identity) -> Result<RustfsPrincipal, PolicyError> {
|
||||
let access_key = identity.access_key_id();
|
||||
let has_session = identity.session().is_some();
|
||||
// Without root credentials neither ownership nor STS claims can be decided.
|
||||
let Some(root) = self.root.current() else {
|
||||
return Err(PolicyError::unavailable());
|
||||
};
|
||||
let root_access_key = root.access_key();
|
||||
|
||||
if access_key == root_access_key {
|
||||
// Root has no session token, so a session identity for its key is not one IAM issued.
|
||||
if has_session {
|
||||
return Err(PolicyError::because(SESSION_MISMATCH));
|
||||
}
|
||||
return Ok(RustfsPrincipal {
|
||||
kind: PrincipalKind::Root,
|
||||
account: access_key.to_owned(),
|
||||
username: access_key.to_owned(),
|
||||
parent_user: String::new(),
|
||||
credential_groups: None,
|
||||
group_memberships: Vec::new(),
|
||||
claims: None,
|
||||
is_owner: true,
|
||||
});
|
||||
}
|
||||
|
||||
if !self.iam.is_ready() {
|
||||
return Err(PolicyError::unavailable());
|
||||
}
|
||||
let cred = match self.iam.check_key(access_key).await {
|
||||
Ok((Some(identity), true)) => identity.credentials,
|
||||
Ok((Some(_), false)) => return Err(PolicyError::because(DISABLED_PRINCIPAL)),
|
||||
Ok((None, _)) => return Err(PolicyError::because(UNKNOWN_PRINCIPAL)),
|
||||
Err(error) => return Err(classify_iam_error(&error)),
|
||||
};
|
||||
if cred.status == ACCOUNT_OFF || !is_usable_secret(cred.secret_key.as_bytes()) {
|
||||
return Err(PolicyError::because(DISABLED_PRINCIPAL));
|
||||
}
|
||||
|
||||
let kind = if cred.is_service_account() {
|
||||
PrincipalKind::ServiceAccount
|
||||
} else if !cred.session_token.is_empty() {
|
||||
PrincipalKind::Temporary
|
||||
} else {
|
||||
PrincipalKind::User
|
||||
};
|
||||
// The gateway admitted this identity against the credential IAM returned, so the kinds
|
||||
// agree unless IAM's record changed in between; refuse rather than judge a stale reading.
|
||||
if has_session != (kind == PrincipalKind::Temporary) {
|
||||
return Err(PolicyError::because(SESSION_MISMATCH));
|
||||
}
|
||||
|
||||
let claims = read_claims(kind, &cred, &root)?;
|
||||
let username = match kind {
|
||||
PrincipalKind::Temporary | PrincipalKind::ServiceAccount => cred.parent_user.clone(),
|
||||
PrincipalKind::Root | PrincipalKind::User => cred.access_key.clone(),
|
||||
};
|
||||
let membership_subject = if username.is_empty() { access_key } else { username.as_str() };
|
||||
let group_memberships = self.iam.group_memberships(membership_subject);
|
||||
let is_owner = acts_for_root(&cred, claims.as_ref(), root_access_key);
|
||||
|
||||
Ok(RustfsPrincipal {
|
||||
kind,
|
||||
account: access_key.to_owned(),
|
||||
username,
|
||||
parent_user: cred.parent_user,
|
||||
credential_groups: cred.groups,
|
||||
group_memberships,
|
||||
claims,
|
||||
is_owner,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// The claims of the credential's own token. The gateway admitted the presented token only when it
|
||||
/// equals the stored one, so reading the stored token reads the presented one.
|
||||
fn read_claims(
|
||||
kind: PrincipalKind,
|
||||
cred: &IamCredential,
|
||||
root: &RootCredentials,
|
||||
) -> Result<Option<HashMap<String, Value>>, PolicyError> {
|
||||
// As in legacy `check_claims_from_token_with_context`: no token, no claims.
|
||||
if cred.session_token.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let claims = match kind {
|
||||
PrincipalKind::ServiceAccount => {
|
||||
get_claims_from_token_with_secret_allow_missing_exp(&cred.session_token, &cred.secret_key)
|
||||
.map_err(|_| PolicyError::malformed())?
|
||||
}
|
||||
PrincipalKind::Temporary => get_claims_from_token_with_secret(&cred.session_token, root.sts_signing_key())
|
||||
.map_err(|_| PolicyError::malformed())?,
|
||||
PrincipalKind::Root | PrincipalKind::User => return Ok(None),
|
||||
};
|
||||
Ok(if claims.is_empty() { None } else { Some(claims) })
|
||||
}
|
||||
|
||||
/// Classifies an IAM failure without keeping any of its text.
|
||||
fn classify_iam_error(error: &IamError) -> PolicyError {
|
||||
match error {
|
||||
IamError::NoSuchUser(_)
|
||||
| IamError::NoSuchServiceAccount(_)
|
||||
| IamError::NoSuchTempAccount(_)
|
||||
| IamError::NoSuchAccount(_) => PolicyError::because(UNKNOWN_PRINCIPAL),
|
||||
_ => PolicyError::unavailable(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Root ownership as the legacy edge decides it (`rustfs/src/auth.rs` `has_root_access`): the
|
||||
/// credential is root's or acts for root, and neither a session policy nor an external OIDC
|
||||
/// identity narrows it.
|
||||
fn acts_for_root(cred: &IamCredential, claims: Option<&HashMap<String, Value>>, root_access_key: &str) -> bool {
|
||||
(cred.access_key == root_access_key || cred.parent_user == root_access_key)
|
||||
&& !claims.is_some_and(|claims| claims.contains_key(SESSION_POLICY_NAME) || is_rustfs_oidc_claims(claims))
|
||||
}
|
||||
|
||||
impl PolicySource for IamPolicySource {
|
||||
fn snapshot<'a>(&'a self, identity: Option<&'a Identity>) -> BoxFuture<'a, Result<PolicySnapshot, PolicyError>> {
|
||||
Box::pin(self.read(identity))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::root::RootCell;
|
||||
use rustfs_credentials::{Credentials as IamCredential, IAM_POLICY_CLAIM_NAME_SA};
|
||||
use rustfs_gateway::SessionBinding;
|
||||
use rustfs_iam::error::Error as IamError;
|
||||
use rustfs_iam::sys::SESSION_POLICY_NAME;
|
||||
use rustfs_iam::utils::generate_jwt;
|
||||
use rustfs_policy::auth::UserIdentity;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use time::OffsetDateTime;
|
||||
|
||||
const ROOT_ACCESS_KEY: &str = "rootadmin";
|
||||
const ROOT_SECRET: &str = "root-secret-key-0001";
|
||||
/// Deliberately not the root secret, so every STS test proves which key claims are read with.
|
||||
const SIGNING_KEY: &str = "sts-signing-key-0001";
|
||||
const ACCESS_KEY: &str = "AKIDUSER0001";
|
||||
const SECRET: &str = "user-secret-key-0001";
|
||||
const PARENT: &str = "alice";
|
||||
const EXPIRES_AT: i64 = 4_000_000_000;
|
||||
|
||||
type KeyAnswer = Result<(Option<UserIdentity>, bool), IamError>;
|
||||
type Answer = Box<dyn Fn() -> KeyAnswer + Send + Sync>;
|
||||
|
||||
struct FakeIam {
|
||||
ready: bool,
|
||||
answer: Answer,
|
||||
calls: AtomicUsize,
|
||||
}
|
||||
|
||||
impl IamKeyLookup for FakeIam {
|
||||
fn is_ready(&self) -> bool {
|
||||
self.ready
|
||||
}
|
||||
|
||||
fn check_key<'a>(&'a self, _access_key: &'a str) -> BoxFuture<'a, KeyAnswer> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let answer = (self.answer)();
|
||||
Box::pin(async move { answer })
|
||||
}
|
||||
}
|
||||
|
||||
impl IamPrincipalLookup for FakeIam {
|
||||
/// Distinct groups per name, so a test can tell which name the source asked about.
|
||||
fn group_memberships(&self, user: &str) -> Vec<String> {
|
||||
match user {
|
||||
PARENT => vec!["parent-group".to_owned()],
|
||||
ACCESS_KEY => vec!["key-group".to_owned()],
|
||||
ROOT_ACCESS_KEY => vec!["root-group".to_owned()],
|
||||
_ => Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fake(ready: bool, answer: impl Fn() -> KeyAnswer + Send + Sync + 'static) -> Arc<FakeIam> {
|
||||
Arc::new(FakeIam {
|
||||
ready,
|
||||
answer: Box::new(answer),
|
||||
calls: AtomicUsize::new(0),
|
||||
})
|
||||
}
|
||||
|
||||
fn root() -> RootCredentials {
|
||||
RootCredentials::new(ROOT_ACCESS_KEY, ROOT_SECRET, SIGNING_KEY).expect("valid root credentials")
|
||||
}
|
||||
|
||||
fn source(iam: Arc<FakeIam>) -> IamPolicySource {
|
||||
IamPolicySource::new(iam, Arc::new(RootCell::new(Some(root()))))
|
||||
}
|
||||
|
||||
fn user() -> IamCredential {
|
||||
IamCredential {
|
||||
access_key: ACCESS_KEY.to_owned(),
|
||||
secret_key: SECRET.to_owned(),
|
||||
status: "on".to_owned(),
|
||||
groups: Some(vec!["cred-group".to_owned()]),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn sts_token(claims: &[(&str, Value)], secret: &str) -> String {
|
||||
let mut map: HashMap<String, Value> = claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
|
||||
map.insert("exp".to_owned(), Value::from(EXPIRES_AT));
|
||||
generate_jwt(&map, secret).expect("a token")
|
||||
}
|
||||
|
||||
fn sts(parent: &str, token: String) -> IamCredential {
|
||||
IamCredential {
|
||||
session_token: token,
|
||||
parent_user: parent.to_owned(),
|
||||
expiration: Some(OffsetDateTime::from_unix_timestamp(EXPIRES_AT).expect("valid timestamp")),
|
||||
..user()
|
||||
}
|
||||
}
|
||||
|
||||
fn found(cred: IamCredential) -> KeyAnswer {
|
||||
Ok((Some(UserIdentity::new(cred)), true))
|
||||
}
|
||||
|
||||
fn long_term(access_key: &str) -> Identity {
|
||||
Identity::new(access_key).expect("valid access key")
|
||||
}
|
||||
|
||||
fn session(access_key: &str) -> Identity {
|
||||
let binding = SessionBinding::new(crate::STS_SESSION_ISSUER, EXPIRES_AT).expect("valid binding");
|
||||
long_term(access_key).with_session_binding(binding)
|
||||
}
|
||||
|
||||
async fn principal_of(source: &IamPolicySource, identity: &Identity) -> PolicySnapshot {
|
||||
source.snapshot(Some(identity)).await.expect("a snapshot")
|
||||
}
|
||||
|
||||
fn payload(snapshot: &PolicySnapshot) -> &RustfsPrincipal {
|
||||
snapshot.get::<RustfsPrincipal>().expect("the payload is a RustfsPrincipal")
|
||||
}
|
||||
|
||||
fn refusal(result: Result<PolicySnapshot, PolicyError>) -> PolicyError {
|
||||
match result {
|
||||
Ok(snapshot) => panic!("expected a refusal, got a snapshot (empty: {})", snapshot.is_empty()),
|
||||
Err(error) => error,
|
||||
}
|
||||
}
|
||||
|
||||
/// Compile-time: `RustfsPrincipal` has no `Debug`, so no formatting of a snapshot payload can
|
||||
/// reach its claims. The two blanket impls make the path ambiguous, and so a compile error, as
|
||||
/// soon as `RustfsPrincipal: Debug` holds.
|
||||
#[test]
|
||||
fn policy_principal_has_no_debug() {
|
||||
trait AmbiguousIfDebug<A> {
|
||||
fn some_item() {}
|
||||
}
|
||||
impl<T: ?Sized> AmbiguousIfDebug<()> for T {}
|
||||
impl<T: ?Sized + core::fmt::Debug> AmbiguousIfDebug<u8> for T {}
|
||||
let _ = <RustfsPrincipal as AmbiguousIfDebug<_>>::some_item;
|
||||
}
|
||||
|
||||
/// Positive — a long-term user's facts: itself as username, both group sources, no claims,
|
||||
/// not the owner.
|
||||
#[tokio::test]
|
||||
async fn policy_user_snapshot_carries_iam_facts() {
|
||||
let iam = fake(true, || found(user()));
|
||||
let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
|
||||
let principal = payload(&snapshot);
|
||||
assert!(principal.kind() == PrincipalKind::User);
|
||||
assert_eq!(principal.account(), ACCESS_KEY);
|
||||
assert_eq!(principal.username(), ACCESS_KEY);
|
||||
assert_eq!(principal.parent_user(), "");
|
||||
assert_eq!(principal.credential_groups(), &Some(vec!["cred-group".to_owned()]));
|
||||
assert_eq!(principal.group_memberships(), ["key-group".to_owned()]);
|
||||
assert!(principal.claims().is_none());
|
||||
assert!(!principal.is_owner());
|
||||
}
|
||||
|
||||
/// Positive — an STS credential acts for its parent user, and its claims are the ones its
|
||||
/// token carries, read with the root secret that signed it.
|
||||
#[tokio::test]
|
||||
async fn policy_sts_snapshot_carries_parent_user_and_claims() {
|
||||
let token = sts_token(&[("sub", Value::from("alice-session"))], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(PARENT, token.clone())));
|
||||
let snapshot = principal_of(&source(iam), &session(ACCESS_KEY)).await;
|
||||
let principal = payload(&snapshot);
|
||||
assert!(principal.kind() == PrincipalKind::Temporary);
|
||||
assert_eq!(principal.account(), ACCESS_KEY);
|
||||
assert_eq!(principal.parent_user(), PARENT);
|
||||
assert_eq!(principal.username(), PARENT);
|
||||
let claims = principal.claims().expect("an STS principal has claims");
|
||||
assert_eq!(claims.get("sub"), Some(&Value::from("alice-session")));
|
||||
assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
|
||||
assert!(!principal.is_owner());
|
||||
}
|
||||
|
||||
/// Positive — a service account's claims are read with its own secret, and it acts for its
|
||||
/// parent.
|
||||
#[tokio::test]
|
||||
async fn policy_service_account_snapshot_reads_its_own_token() {
|
||||
let mut token_claims: HashMap<String, Value> = HashMap::new();
|
||||
token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
|
||||
token_claims.insert("parent".to_owned(), Value::from(PARENT));
|
||||
let token = generate_jwt(&token_claims, SECRET).expect("a token");
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user();
|
||||
cred.session_token = token.clone();
|
||||
cred.parent_user = PARENT.to_owned();
|
||||
cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
|
||||
found(cred)
|
||||
});
|
||||
let snapshot = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
|
||||
let principal = payload(&snapshot);
|
||||
assert!(principal.kind() == PrincipalKind::ServiceAccount);
|
||||
assert_eq!(principal.username(), PARENT);
|
||||
assert_eq!(principal.claims().and_then(|c| c.get("parent")), Some(&Value::from(PARENT)));
|
||||
assert_eq!(principal.group_memberships(), ["parent-group".to_owned()]);
|
||||
assert!(!principal.is_owner());
|
||||
}
|
||||
|
||||
/// Positive and negative — root is the owner without asking IAM, even before IAM is ready;
|
||||
/// an STS credential of root is the owner too, but not once a session policy narrows it.
|
||||
#[tokio::test]
|
||||
async fn policy_owner_is_root_or_unrestricted_root_session() {
|
||||
let not_ready = fake(false, || found(user()));
|
||||
let snapshot = principal_of(&source(not_ready.clone()), &long_term(ROOT_ACCESS_KEY)).await;
|
||||
let root = payload(&snapshot);
|
||||
assert!(root.kind() == PrincipalKind::Root);
|
||||
assert!(root.is_owner());
|
||||
assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let plain = sts_token(&[], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, plain.clone())));
|
||||
assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
|
||||
let narrowed = sts_token(&[(SESSION_POLICY_NAME, Value::from("e30"))], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, narrowed.clone())));
|
||||
assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
}
|
||||
|
||||
/// Negative — an anonymous caller is an empty snapshot, and IAM is not asked.
|
||||
#[tokio::test]
|
||||
async fn policy_anonymous_is_empty_snapshot() {
|
||||
let iam = fake(true, || found(user()));
|
||||
let snapshot = source(iam.clone()).snapshot(None).await.expect("an anonymous snapshot");
|
||||
assert!(snapshot.is_empty());
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
/// Negative — IAM that is not ready, not initialized, or failing is `unavailable`: never an
|
||||
/// empty snapshot, which an authorizer would judge as an anonymous caller.
|
||||
#[tokio::test]
|
||||
async fn policy_iam_unavailable_is_unavailable() {
|
||||
let not_ready = fake(false, || found(user()));
|
||||
let error = refusal(source(not_ready.clone()).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::unavailable());
|
||||
assert_eq!(not_ready.calls.load(Ordering::SeqCst), 0);
|
||||
|
||||
let failures: [fn() -> IamError; 2] = [
|
||||
|| IamError::IamSysNotInitialized,
|
||||
|| IamError::StringError("storage read failed".to_owned()),
|
||||
];
|
||||
for failure in failures {
|
||||
let iam = fake(true, move || Err(failure()));
|
||||
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::unavailable(), "{:?}", failure());
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a principal IAM no longer knows is refused as unknown, not as unavailable and
|
||||
/// not as an empty snapshot.
|
||||
#[tokio::test]
|
||||
async fn policy_unknown_principal_is_refused() {
|
||||
let answers: [fn() -> KeyAnswer; 2] = [|| Ok((None, false)), || Err(IamError::NoSuchUser(ACCESS_KEY.to_owned()))];
|
||||
for answer in answers {
|
||||
let error = refusal(source(fake(true, answer)).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(UNKNOWN_PRINCIPAL));
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a disabled or invalid principal is refused rather than given a snapshot.
|
||||
#[tokio::test]
|
||||
async fn policy_disabled_principal_is_refused() {
|
||||
let invalid = fake(true, || Ok((Some(UserIdentity::new(user())), false)));
|
||||
let error = refusal(source(invalid).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
|
||||
|
||||
let off = fake(true, || {
|
||||
let mut cred = user();
|
||||
cred.status = "off".to_owned();
|
||||
found(cred)
|
||||
});
|
||||
let error = refusal(source(off).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL));
|
||||
}
|
||||
|
||||
/// Negative — an STS token not signed by root yields no claims at all: it is `malformed`.
|
||||
#[tokio::test]
|
||||
async fn policy_sts_token_with_foreign_signature_is_malformed() {
|
||||
let forged = sts_token(&[("sub", Value::from("alice-session"))], "not-the-root-secret");
|
||||
let iam = fake(true, move || found(sts(PARENT, forged.clone())));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::malformed());
|
||||
}
|
||||
|
||||
/// Negative — the identity's credential kind must match what IAM holds: a session identity for
|
||||
/// a long-term key, or a long-term identity for an STS key, is refused.
|
||||
#[tokio::test]
|
||||
async fn policy_session_kind_mismatch_is_refused() {
|
||||
let token = sts_token(&[], SIGNING_KEY);
|
||||
let iam = fake(true, move || found(sts(PARENT, token.clone())));
|
||||
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
|
||||
|
||||
let iam = fake(true, || found(user()));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
|
||||
|
||||
let iam = fake(false, || found(user()));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ROOT_ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(SESSION_MISMATCH));
|
||||
}
|
||||
|
||||
fn service_account(parent: &str, extra_claims: &[(&str, Value)]) -> KeyAnswer {
|
||||
let mut token_claims: HashMap<String, Value> = extra_claims.iter().map(|(k, v)| ((*k).to_owned(), v.clone())).collect();
|
||||
token_claims.insert(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"));
|
||||
let mut cred = user();
|
||||
cred.session_token = generate_jwt(&token_claims, SECRET).expect("a token");
|
||||
cred.parent_user = parent.to_owned();
|
||||
cred.claims = Some([(IAM_POLICY_CLAIM_NAME_SA.to_owned(), Value::from("inherited-policy"))].into());
|
||||
found(cred)
|
||||
}
|
||||
|
||||
/// Negative — a valid-flagged identity whose secret is empty or short gets no snapshot.
|
||||
#[tokio::test]
|
||||
async fn policy_short_secret_principal_is_refused() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user();
|
||||
cred.secret_key = weak.to_owned();
|
||||
found(cred)
|
||||
});
|
||||
let error = refusal(source(iam).snapshot(Some(&long_term(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::because(DISABLED_PRINCIPAL), "{weak:?}");
|
||||
}
|
||||
for usable in ["12345678", "abc\0defg"] {
|
||||
let iam = fake(true, move || {
|
||||
let mut cred = user();
|
||||
cred.secret_key = usable.to_owned();
|
||||
found(cred)
|
||||
});
|
||||
assert!(payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).kind() == PrincipalKind::User);
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — a root session issued through external OIDC is not the owner, though its parent
|
||||
/// is root; the same session without the OIDC issuer is.
|
||||
#[tokio::test]
|
||||
async fn policy_oidc_root_session_is_not_owner() {
|
||||
let oidc = sts_token(
|
||||
&[
|
||||
("iss", Value::from("rustfs-oidc")),
|
||||
("oidc_provider", Value::from("corp-idp")),
|
||||
("sub", Value::from("alice@corp")),
|
||||
],
|
||||
SIGNING_KEY,
|
||||
);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, oidc.clone())));
|
||||
assert!(!payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
|
||||
let local = sts_token(
|
||||
&[
|
||||
("iss", Value::from("rustfs-sts")),
|
||||
("oidc_provider", Value::from("corp-idp")),
|
||||
("sub", Value::from("alice@corp")),
|
||||
],
|
||||
SIGNING_KEY,
|
||||
);
|
||||
let iam = fake(true, move || found(sts(ROOT_ACCESS_KEY, local.clone())));
|
||||
assert!(payload(&principal_of(&source(iam), &session(ACCESS_KEY)).await).is_owner());
|
||||
}
|
||||
|
||||
/// Positive and negative — a service account of root is the owner; one narrowed by a session
|
||||
/// policy is not, and neither is a service account of an ordinary user.
|
||||
#[tokio::test]
|
||||
async fn policy_service_account_of_root_is_owner() {
|
||||
let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[]));
|
||||
let principal = principal_of(&source(iam), &long_term(ACCESS_KEY)).await;
|
||||
assert!(payload(&principal).kind() == PrincipalKind::ServiceAccount);
|
||||
assert!(payload(&principal).is_owner());
|
||||
|
||||
let iam = fake(true, || service_account(ROOT_ACCESS_KEY, &[(SESSION_POLICY_NAME, Value::from("e30"))]));
|
||||
assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
|
||||
|
||||
let iam = fake(true, || service_account(PARENT, &[]));
|
||||
assert!(!payload(&principal_of(&source(iam), &long_term(ACCESS_KEY)).await).is_owner());
|
||||
}
|
||||
|
||||
/// Negative — STS claims are read with the STS signing key, not with the root secret: a token
|
||||
/// signed with the root secret while the signing key differs is `malformed`.
|
||||
#[tokio::test]
|
||||
async fn policy_sts_claims_use_the_signing_key() {
|
||||
let root_signed = sts_token(&[("sub", Value::from("alice-session"))], ROOT_SECRET);
|
||||
let iam = fake(true, move || found(sts(PARENT, root_signed.clone())));
|
||||
let error = refusal(source(iam).snapshot(Some(&session(ACCESS_KEY))).await);
|
||||
assert_eq!(error, PolicyError::malformed());
|
||||
}
|
||||
|
||||
/// Negative — without root credentials no principal can be judged: `unavailable`, never an
|
||||
/// empty snapshot, and IAM is not asked.
|
||||
#[tokio::test]
|
||||
async fn policy_without_root_is_unavailable() {
|
||||
let iam = fake(true, || found(user()));
|
||||
let source = IamPolicySource::new(iam.clone(), Arc::new(RootCell::new(None)));
|
||||
for identity in [long_term(ACCESS_KEY), long_term(ROOT_ACCESS_KEY)] {
|
||||
assert_eq!(refusal(source.snapshot(Some(&identity)).await), PolicyError::unavailable());
|
||||
}
|
||||
assert_eq!(iam.calls.load(Ordering::SeqCst), 0);
|
||||
}
|
||||
|
||||
type LiveRoot = Arc<std::sync::Mutex<Option<(String, String)>>>;
|
||||
|
||||
/// The primary path: a source that reads a live accessor on every call, as the binary's does,
|
||||
/// with nothing cached in between.
|
||||
fn live_root_source(live: &LiveRoot) -> Arc<dyn RootSource> {
|
||||
let accessor = live.clone();
|
||||
Arc::new(move || {
|
||||
let guard = accessor.lock().ok()?;
|
||||
let (access_key, secret_key) = guard.as_ref()?;
|
||||
RootCredentials::new(access_key, secret_key, SIGNING_KEY).ok().map(Arc::new)
|
||||
})
|
||||
}
|
||||
|
||||
fn set_live(live: &LiveRoot, value: Option<(&str, &str)>) {
|
||||
*live.lock().expect("live root lock") = value.map(|(ak, sk)| (ak.to_owned(), sk.to_owned()));
|
||||
}
|
||||
|
||||
/// Positive and negative — the two adapters read one live root accessor: a root change is seen
|
||||
/// by both on the next request with no second step, the old root key stops being root to
|
||||
/// both, and an uninitialized or unusable root makes both unavailable.
|
||||
#[tokio::test]
|
||||
async fn adapters_share_one_live_root() {
|
||||
use rustfs_gateway::{CredentialLookup, CredentialProvider, ProviderError};
|
||||
|
||||
let live: LiveRoot = Arc::default();
|
||||
set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
|
||||
let iam = fake(true, || Ok((None, false)));
|
||||
let adapters = crate::IamAdapters::new(iam, live_root_source(&live));
|
||||
|
||||
let found = adapters.credentials.lookup(ROOT_ACCESS_KEY).await;
|
||||
assert!(matches!(found, Ok(CredentialLookup::Found(_))));
|
||||
let snapshot = principal_of(&adapters.policy, &long_term(ROOT_ACCESS_KEY)).await;
|
||||
assert!(payload(&snapshot).is_owner());
|
||||
|
||||
set_live(&live, Some(("newroot", "new-root-secret-01")));
|
||||
match adapters.credentials.lookup("newroot").await {
|
||||
Ok(CredentialLookup::Found(credentials)) => assert_eq!(credentials.secret().expose(), b"new-root-secret-01"),
|
||||
_ => panic!("the new root key must resolve to the new secret"),
|
||||
}
|
||||
assert!(matches!(
|
||||
adapters.credentials.lookup(ROOT_ACCESS_KEY).await,
|
||||
Ok(CredentialLookup::NotFound)
|
||||
));
|
||||
assert!(payload(&principal_of(&adapters.policy, &long_term("newroot")).await).is_owner());
|
||||
let old = refusal(adapters.policy.snapshot(Some(&long_term(ROOT_ACCESS_KEY))).await);
|
||||
assert_eq!(old, PolicyError::because(UNKNOWN_PRINCIPAL));
|
||||
|
||||
for unusable in [None, Some(("newroot", "\0\0\0\0\0\0\0\0"))] {
|
||||
set_live(&live, unusable);
|
||||
assert!(matches!(adapters.credentials.lookup("newroot").await, Err(ProviderError::Unavailable)));
|
||||
assert_eq!(
|
||||
refusal(adapters.policy.snapshot(Some(&long_term("newroot"))).await),
|
||||
PolicyError::unavailable()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive — `install` builds the authenticator exactly once, over a provider that reads the
|
||||
/// pair's root source.
|
||||
#[tokio::test]
|
||||
async fn adapters_install_hands_over_the_paired_provider_once() {
|
||||
use rustfs_gateway::{CredentialLookup, CredentialProvider, RegionSet, ServiceBuilder, SigV4Authenticator};
|
||||
|
||||
let live: LiveRoot = Arc::default();
|
||||
set_live(&live, Some((ROOT_ACCESS_KEY, ROOT_SECRET)));
|
||||
let adapters = crate::IamAdapters::new(fake(true, || Ok((None, false))), live_root_source(&live));
|
||||
let mut handed = Vec::new();
|
||||
let _builder = adapters.install(ServiceBuilder::new(), |provider| {
|
||||
handed.push(provider.clone());
|
||||
SigV4Authenticator::new(provider, RegionSet::new(["us-east-1"]).expect("a region"))
|
||||
});
|
||||
assert_eq!(handed.len(), 1);
|
||||
let found = handed[0].lookup(ROOT_ACCESS_KEY).await;
|
||||
assert!(matches!(found, Ok(CredentialLookup::Found(_))));
|
||||
}
|
||||
}
|
||||
@@ -1,216 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The deployment's root credentials, as every adapter in this crate reads them.
|
||||
//!
|
||||
//! Responsible for: [`RootCredentials`], the validated root key pair plus the key STS session
|
||||
//! tokens are signed with; [`RootSource`], the one live handle both the credential provider and
|
||||
//! the policy source read on every request; and the effective-secret-length rule every secret in
|
||||
//! this crate is held to.
|
||||
//! NOT responsible for: where the root keys come from or when they change. The binary implements
|
||||
//! [`RootSource`] directly over its live root accessor — a closure is enough — so there is no
|
||||
//! second copy of root to keep in step.
|
||||
//! Upstream: the binary's runtime credentials. Downstream: `crate::credentials`, `crate::policy`.
|
||||
//!
|
||||
//! # Why one source
|
||||
//!
|
||||
//! Legacy reads the current root credentials on every request. Two copies taken at assembly could
|
||||
//! disagree, or go stale after a root change, and a key one adapter still treats as root would be
|
||||
//! answered with the root secret or judged as the owner. Both adapters therefore hold the same
|
||||
//! `Arc<dyn RootSource>`, handed to them together by [`crate::IamAdapters::new`].
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_gateway::Credentials;
|
||||
|
||||
/// The shortest effective secret key, in bytes, that any credential in this crate may verify a
|
||||
/// signature with. The same bound IAM applies to its own credentials; a signature over a shorter
|
||||
/// (or empty) key is one anybody can compute.
|
||||
pub const MIN_SECRET_KEY_LEN: usize = 8;
|
||||
|
||||
/// The number of bytes of `secret` a signature actually depends on: its length after trailing NUL
|
||||
/// bytes are removed.
|
||||
///
|
||||
/// HMAC pads a key shorter than its block with zero bytes, and SigV4's first key is `"AWS4"`
|
||||
/// followed by the secret, so trailing NULs add nothing: eight NUL bytes sign exactly like the
|
||||
/// empty secret, and `"abc"` followed by five NULs signs like `"abc"`. A secret of NULs only has
|
||||
/// an effective length of zero. An interior NUL is followed by real key bytes, which change the
|
||||
/// padded key, so it counts like any other byte.
|
||||
pub(crate) fn effective_secret_len(secret: &[u8]) -> usize {
|
||||
secret.iter().rposition(|byte| *byte != 0).map_or(0, |last| last + 1)
|
||||
}
|
||||
|
||||
/// Whether `secret` may verify a signature: its [`effective_secret_len`] is at least
|
||||
/// [`MIN_SECRET_KEY_LEN`]. The one rule for root, IAM credentials and the STS signing key.
|
||||
pub(crate) fn is_usable_secret(secret: &[u8]) -> bool {
|
||||
effective_secret_len(secret) >= MIN_SECRET_KEY_LEN
|
||||
}
|
||||
|
||||
/// Why [`RootCredentials::new`] refused its input. Carries no input text.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum RootCredentialsError {
|
||||
/// The access key id is not one the gateway can hold.
|
||||
InvalidAccessKey,
|
||||
/// The root secret key fails the effective-length rule.
|
||||
WeakSecretKey,
|
||||
/// The STS token-signing key fails the effective-length rule.
|
||||
WeakSigningKey,
|
||||
}
|
||||
|
||||
impl core::fmt::Display for RootCredentialsError {
|
||||
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
|
||||
f.write_str(match self {
|
||||
Self::InvalidAccessKey => "root access key is not usable",
|
||||
Self::WeakSecretKey => "root secret key is shorter than the minimum length",
|
||||
Self::WeakSigningKey => "STS token-signing key is shorter than the minimum length",
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for RootCredentialsError {}
|
||||
|
||||
/// The root key pair and the STS token-signing key, validated together.
|
||||
///
|
||||
/// Implements no `Debug`; the secrets are readable only inside this crate.
|
||||
pub struct RootCredentials {
|
||||
credentials: Credentials,
|
||||
sts_signing_key: Box<str>,
|
||||
}
|
||||
|
||||
impl RootCredentials {
|
||||
/// Validates the root credentials and the key IAM signs STS session tokens with (today the
|
||||
/// root secret, `rustfs_iam::manager::get_token_signing_key`; pass it separately so a future
|
||||
/// key separation changes no adapter).
|
||||
///
|
||||
/// # Errors
|
||||
///
|
||||
/// [`RootCredentialsError`] when either secret fails the effective-length rule (fewer than
|
||||
/// [`MIN_SECRET_KEY_LEN`] bytes once trailing NUL bytes are removed) or the access key id is not
|
||||
/// one the gateway can hold.
|
||||
pub fn new(access_key: &str, secret_key: &str, sts_signing_key: &str) -> Result<Self, RootCredentialsError> {
|
||||
if !is_usable_secret(secret_key.as_bytes()) {
|
||||
return Err(RootCredentialsError::WeakSecretKey);
|
||||
}
|
||||
if !is_usable_secret(sts_signing_key.as_bytes()) {
|
||||
return Err(RootCredentialsError::WeakSigningKey);
|
||||
}
|
||||
let credentials =
|
||||
Credentials::new(access_key, secret_key.as_bytes()).map_err(|_| RootCredentialsError::InvalidAccessKey)?;
|
||||
Ok(Self {
|
||||
credentials,
|
||||
sts_signing_key: sts_signing_key.into(),
|
||||
})
|
||||
}
|
||||
|
||||
/// The root access key id, a public identifier.
|
||||
#[must_use]
|
||||
pub fn access_key(&self) -> &str {
|
||||
self.credentials.identity().access_key_id()
|
||||
}
|
||||
|
||||
pub(crate) fn credentials(&self) -> &Credentials {
|
||||
&self.credentials
|
||||
}
|
||||
|
||||
pub(crate) fn sts_signing_key(&self) -> &str {
|
||||
&self.sts_signing_key
|
||||
}
|
||||
}
|
||||
|
||||
/// The live root credentials, read on every request by both adapters.
|
||||
///
|
||||
/// The binary implements it directly over RustFS's live root accessor, so a root change needs no
|
||||
/// second step. Any `Fn() -> Option<Arc<RootCredentials>>` is a source; for example, over the
|
||||
/// server context:
|
||||
///
|
||||
/// ```ignore
|
||||
/// let root = move || {
|
||||
/// let cred = ctx.action_credentials().get()?;
|
||||
/// let signing_key = ctx.token_signing_key()?;
|
||||
/// RootCredentials::new(&cred.access_key, &cred.secret_key, &signing_key).ok().map(Arc::new)
|
||||
/// };
|
||||
/// let adapters = IamAdapters::new(iam, Arc::new(root));
|
||||
/// ```
|
||||
///
|
||||
/// Root credentials that fail validation read as `None`, which both adapters answer as
|
||||
/// unavailable.
|
||||
pub trait RootSource: Send + Sync + 'static {
|
||||
/// The current root credentials; `None` while they are not initialized, which both adapters
|
||||
/// answer as unavailable rather than guessing.
|
||||
fn current(&self) -> Option<Arc<RootCredentials>>;
|
||||
}
|
||||
|
||||
impl<F> RootSource for F
|
||||
where
|
||||
F: Fn() -> Option<Arc<RootCredentials>> + Send + Sync + 'static,
|
||||
{
|
||||
fn current(&self) -> Option<Arc<RootCredentials>> {
|
||||
self()
|
||||
}
|
||||
}
|
||||
|
||||
/// Test helper: a [`RootSource`] holding fixed root credentials. Production code reads the live
|
||||
/// accessor instead, so that there is no copy of root to keep in step.
|
||||
#[cfg(test)]
|
||||
pub(crate) struct RootCell(Option<Arc<RootCredentials>>);
|
||||
|
||||
#[cfg(test)]
|
||||
impl RootCell {
|
||||
pub(crate) fn new(root: Option<RootCredentials>) -> Self {
|
||||
Self(root.map(Arc::new))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl RootSource for RootCell {
|
||||
fn current(&self) -> Option<Arc<RootCredentials>> {
|
||||
self.0.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Negative — an empty or short root secret is refused at construction, so no source can ever
|
||||
/// hand out a root key anybody could sign with; eight bytes is accepted.
|
||||
#[test]
|
||||
fn root_short_secret_is_refused() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0", "abcdefg\0"] {
|
||||
let refused = RootCredentials::new("rootadmin", weak, "signing-key-0001");
|
||||
assert!(matches!(refused, Err(RootCredentialsError::WeakSecretKey)), "{} bytes", weak.len());
|
||||
}
|
||||
assert!(RootCredentials::new("rootadmin", "12345678", "signing-key-0001").is_ok());
|
||||
}
|
||||
|
||||
/// Positive — an interior NUL is followed by real key bytes and counts: `"abc\0defg"` has an
|
||||
/// effective length of eight and is accepted, as is the same secret with trailing NULs.
|
||||
#[test]
|
||||
fn root_interior_nul_counts_toward_length() {
|
||||
for secret in ["abc\0defg", "abc\0defg\0\0"] {
|
||||
assert!(RootCredentials::new("rootadmin", secret, "signing-key-0001").is_ok(), "{secret:?}");
|
||||
assert!(RootCredentials::new("rootadmin", "root-secret-key-0001", secret).is_ok(), "{secret:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative — an empty or short STS signing key is refused at construction.
|
||||
#[test]
|
||||
fn root_short_signing_key_is_refused() {
|
||||
for weak in ["", "1234567", "\0\0\0\0\0\0\0\0", "abc\0\0\0\0\0"] {
|
||||
let refused = RootCredentials::new("rootadmin", "root-secret-key-0001", weak);
|
||||
assert!(matches!(refused, Err(RootCredentialsError::WeakSigningKey)), "{} bytes", weak.len());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! The request id RustFS's outer layer minted, handed to the gateway as its own when that is safe.
|
||||
//!
|
||||
//! Responsible for: reading the id from the context the outer layer put in the request's
|
||||
//! extensions ([`RequestIdCarrier`]) and, when it is inside the gateway's identifier alphabet and
|
||||
//! length, inserting it as the gateway's [`HostRequestId`].
|
||||
//! NOT responsible for: minting any id, refusing any request (nothing here can fail a request),
|
||||
//! choosing the response headers (the gateway writes `x-amz-request-id` from the id it was
|
||||
//! handed, and RustFS's outer layer overwrites that header on S3 paths), or the layer that mints
|
||||
//! (`rustfs/src/server/layer.rs`). It never reads a header itself.
|
||||
//! Upstream: RustFS's `RequestContext`, which the binary makes a [`RequestIdCarrier`]. Downstream:
|
||||
//! the gateway service, which reads the [`HostRequestId`] extension once, at the top.
|
||||
//!
|
||||
//! # Whose id it is
|
||||
//!
|
||||
//! On S3 paths the outer layer mints a server-owned UUID. On admin, STS and console paths the
|
||||
//! context copies the caller's `x-request-id` (or `x-amz-request-id`), as legacy does, so there
|
||||
//! the id can be caller-chosen. Either way the id is attached only when it is inert, that is 1 to
|
||||
//! 64 bytes of ASCII letters, digits, `-`, `.`, `_` and `~`, which is safe in a header, an XML
|
||||
//! document and a log line. Anything else is not attached and the request is **forwarded
|
||||
//! unchanged**: the gateway then mints its own id. No request is ever refused because of its id.
|
||||
//!
|
||||
//! # Known difference when nothing is attached
|
||||
//!
|
||||
//! The gateway answers with the id it minted (error body `<RequestId>`, its events and audit
|
||||
//! records) while, on S3 paths, the outer layer overwrites the response header with RustFS's id.
|
||||
//! So for such a request the header and the error body name different ids. That needs a caller-
|
||||
//! chosen id outside the alphabet, or a request that never passed the outer layer.
|
||||
//!
|
||||
//! # Why this is not a `TraceSource`
|
||||
//!
|
||||
//! The gateway's `TraceSource::mint` takes no request, so a source cannot read the id RustFS
|
||||
//! already minted; the gateway's documented path for a host that identifies its own requests is
|
||||
//! the [`HostRequestId`] extension, and this module fills it.
|
||||
//!
|
||||
//! # Why a carrier trait
|
||||
//!
|
||||
//! `RequestContext` lives in the `rustfs` binary crate, which a library crate cannot depend on.
|
||||
//! The binary implements [`RequestIdCarrier`] for it, and this module looks it up in the
|
||||
//! extensions by that type.
|
||||
|
||||
use rustfs_gateway::HostRequestId;
|
||||
|
||||
/// A type in the request extensions that holds the id the outer layer minted or propagated.
|
||||
pub trait RequestIdCarrier: Send + Sync + 'static {
|
||||
/// The canonical request id: the value RustFS logs and the response should carry.
|
||||
fn request_id(&self) -> &str;
|
||||
}
|
||||
|
||||
/// What [`attach_host_request_id`] did. Ignoring it is harmless: every outcome forwards the request.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum AttachOutcome {
|
||||
/// The carrier's id was inserted as the request's [`HostRequestId`].
|
||||
Attached,
|
||||
/// The request carries no context from the outer layer; nothing was inserted.
|
||||
NoContext,
|
||||
/// The carrier's id is empty, too long, or outside the gateway's alphabet; nothing was
|
||||
/// inserted and the id was not repaired.
|
||||
UnusableId,
|
||||
}
|
||||
|
||||
/// The gateway's [`HostRequestId`] for the id `carrier` holds, or `None` when it is not inert.
|
||||
#[must_use]
|
||||
pub fn host_request_id<C: RequestIdCarrier + ?Sized>(carrier: &C) -> Option<HostRequestId> {
|
||||
HostRequestId::new(carrier.request_id()).ok()
|
||||
}
|
||||
|
||||
/// Reads the `C` the outer layer put in `request`'s extensions and, when its id is inert, inserts
|
||||
/// it as the request's [`HostRequestId`].
|
||||
///
|
||||
/// Never refuses the request: the caller forwards it whatever the outcome.
|
||||
pub fn attach_host_request_id<C: RequestIdCarrier, B>(request: &mut http::Request<B>) -> AttachOutcome {
|
||||
let Some(carrier) = request.extensions().get::<C>() else {
|
||||
return AttachOutcome::NoContext;
|
||||
};
|
||||
let Some(id) = host_request_id(carrier) else {
|
||||
return AttachOutcome::UnusableId;
|
||||
};
|
||||
request.extensions_mut().insert(id);
|
||||
AttachOutcome::Attached
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[allow(clippy::expect_used, clippy::panic, clippy::unwrap_used)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[derive(Clone)]
|
||||
struct Ctx(String);
|
||||
|
||||
impl RequestIdCarrier for Ctx {
|
||||
fn request_id(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
const UUID: &str = "7c9e6679-7425-40de-944b-e07fc1f90ae7";
|
||||
|
||||
fn request_with(id: &str) -> http::Request<()> {
|
||||
let mut request = http::Request::new(());
|
||||
request.extensions_mut().insert(Ctx(id.to_owned()));
|
||||
request
|
||||
}
|
||||
|
||||
fn handed_over(request: &http::Request<()>) -> Option<String> {
|
||||
request
|
||||
.extensions()
|
||||
.get::<HostRequestId>()
|
||||
.map(|id| id.request_id().as_str().to_owned())
|
||||
}
|
||||
|
||||
fn attach(request: &mut http::Request<()>) -> AttachOutcome {
|
||||
attach_host_request_id::<Ctx, _>(request)
|
||||
}
|
||||
|
||||
/// Positive: the id RustFS minted is the id the gateway is handed, byte for byte.
|
||||
#[test]
|
||||
fn the_minted_id_is_handed_over_unchanged() {
|
||||
let mut request = request_with(UUID);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached);
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(UUID));
|
||||
}
|
||||
|
||||
/// Positive: every shape RustFS mints is accepted, including its two fallbacks.
|
||||
#[test]
|
||||
fn every_shape_rustfs_mints_is_accepted() {
|
||||
for id in [UUID, "trace-0123456789abcdef", "req-0123456789abcdef"] {
|
||||
let mut request = request_with(id);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached, "{id}");
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(id));
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive: a caller-supplied id that is inert (admin and STS paths copy it) is attached, and
|
||||
/// the longest allowed one is accepted.
|
||||
#[test]
|
||||
fn an_inert_caller_supplied_id_is_attached() {
|
||||
let longest = "a".repeat(64);
|
||||
for id in ["abc", "my.trace_id~1-2", longest.as_str()] {
|
||||
let mut request = request_with(id);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached, "{id}");
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(id));
|
||||
}
|
||||
}
|
||||
|
||||
/// Positive: two requests keep their own ids; nothing is shared or minted between them.
|
||||
#[test]
|
||||
fn each_request_keeps_its_own_id() {
|
||||
let mut first = request_with("first-id");
|
||||
let mut second = request_with("second-id");
|
||||
attach(&mut first);
|
||||
attach(&mut second);
|
||||
assert_eq!(handed_over(&first).as_deref(), Some("first-id"));
|
||||
assert_eq!(handed_over(&second).as_deref(), Some("second-id"));
|
||||
}
|
||||
|
||||
/// Positive: with both a carrier and an `x-amz-request-id` header, the carrier's id wins.
|
||||
#[test]
|
||||
fn the_carrier_wins_over_a_request_id_header() {
|
||||
let mut request = http::Request::builder()
|
||||
.header("x-amz-request-id", "caller-chosen")
|
||||
.header("x-request-id", "caller-chosen")
|
||||
.body(())
|
||||
.expect("a request");
|
||||
request.extensions_mut().insert(Ctx(UUID.to_owned()));
|
||||
assert_eq!(attach(&mut request), AttachOutcome::Attached);
|
||||
assert_eq!(handed_over(&request).as_deref(), Some(UUID));
|
||||
}
|
||||
|
||||
/// Negative: a request the outer layer never saw gets no id, and the request is not refused.
|
||||
#[test]
|
||||
fn a_missing_context_hands_over_nothing() {
|
||||
let mut request = http::Request::new(());
|
||||
assert_eq!(attach(&mut request), AttachOutcome::NoContext);
|
||||
assert_eq!(handed_over(&request), None);
|
||||
}
|
||||
|
||||
/// Negative: a header named like the id is not the id; only the context is read.
|
||||
#[test]
|
||||
fn a_request_id_header_alone_is_not_read() {
|
||||
let mut request = http::Request::builder()
|
||||
.header("x-amz-request-id", "caller-chosen")
|
||||
.header("x-request-id", "caller-chosen")
|
||||
.body(())
|
||||
.expect("a request");
|
||||
assert_eq!(attach(&mut request), AttachOutcome::NoContext);
|
||||
assert_eq!(handed_over(&request), None);
|
||||
}
|
||||
|
||||
/// Negative: an id outside the alphabet or the length is never attached or repaired, and the
|
||||
/// outcome says so rather than failing the request.
|
||||
#[test]
|
||||
fn an_id_the_gateway_cannot_carry_is_not_attached() {
|
||||
let too_long = "a".repeat(65);
|
||||
for id in [
|
||||
"",
|
||||
"has space",
|
||||
"line\r\nbreak",
|
||||
"<RequestId>",
|
||||
"quo\"te",
|
||||
"Root=1-5759e988",
|
||||
too_long.as_str(),
|
||||
] {
|
||||
let mut request = request_with(id);
|
||||
assert_eq!(attach(&mut request), AttachOutcome::UnusableId, "{id:?}");
|
||||
assert_eq!(handed_over(&request), None, "{id:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Negative: the unusable id is not repaired into a shorter or cleaned-up one.
|
||||
#[test]
|
||||
fn host_request_id_does_not_repair() {
|
||||
assert!(host_request_id(&Ctx("has space".to_owned())).is_none());
|
||||
assert!(host_request_id(&Ctx(format!("{UUID} "))).is_none());
|
||||
assert!(host_request_id(&Ctx(UUID.to_owned())).is_some());
|
||||
}
|
||||
}
|
||||
@@ -385,15 +385,9 @@ mod tests {
|
||||
.is_err(),
|
||||
"a stale worker cannot mutate a recreated bucket"
|
||||
);
|
||||
// Name targets as the store's endpoints render them (`C:/...` on Windows),
|
||||
// not as raw OS paths.
|
||||
let targets = directories
|
||||
.iter()
|
||||
.map(|dir| {
|
||||
crate::heal::Endpoint::try_from(dir.path().to_str().expect("UTF-8 temp disk path"))
|
||||
.expect("temp disk endpoint")
|
||||
.to_string()
|
||||
})
|
||||
.map(|dir| dir.path().to_string_lossy().into_owned())
|
||||
.collect::<Vec<_>>();
|
||||
assert!(
|
||||
storage
|
||||
|
||||
@@ -917,10 +917,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_configure_request_development_defaults_require_opt_in() {
|
||||
let key_dir = std::env::temp_dir().join("kms-key-dir");
|
||||
let local_raw = serde_json::json!({
|
||||
"backend_type": "local",
|
||||
"key_dir": key_dir
|
||||
"key_dir": "/tmp/kms-key-dir"
|
||||
});
|
||||
let request: ConfigureKmsRequest = serde_json::from_value(local_raw).expect("local request should deserialize");
|
||||
let config = request.to_kms_config();
|
||||
@@ -928,7 +927,7 @@ mod tests {
|
||||
|
||||
let local_opt_in_raw = serde_json::json!({
|
||||
"backend_type": "local",
|
||||
"key_dir": key_dir,
|
||||
"key_dir": "/tmp/kms-key-dir",
|
||||
"allow_insecure_dev_defaults": true
|
||||
});
|
||||
let request: ConfigureKmsRequest = serde_json::from_value(local_opt_in_raw).expect("local request should deserialize");
|
||||
|
||||
+18
-200
@@ -56,7 +56,6 @@
|
||||
//! schedule/cancel deletion, enable/disable) are executed at most once because
|
||||
//! a lost response could otherwise be replayed into a second side effect.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
|
||||
@@ -70,7 +69,6 @@ use tokio_util::sync::CancellationToken;
|
||||
|
||||
use super::{BackendCapabilities, ExpiredKeyRemoval, KmsBackend, empty_key_page, list_keys_page_size};
|
||||
use crate::config::{BackendConfig, KmsConfig};
|
||||
use crate::encryption::{DataKeyEnvelope, is_data_key_envelope};
|
||||
use crate::error::{KmsError, Result};
|
||||
use crate::policy::{self, AttemptError, ErrorClass, OpClass, RetryPolicy, classify_status};
|
||||
use crate::types::*;
|
||||
@@ -290,22 +288,6 @@ pub struct AwsKmsBackend {
|
||||
cancel: CancellationToken,
|
||||
}
|
||||
|
||||
/// The blob to hand to AWS `Decrypt` for a stored ciphertext.
|
||||
///
|
||||
/// [`KmsBackend::generate_data_key`] wraps the AWS blob in a
|
||||
/// [`DataKeyEnvelope`]; [`KmsBackend::encrypt`] returns it bare, and so did
|
||||
/// `generate_data_key` through 1.0.5. Both shapes stay decryptable. The
|
||||
/// envelope's recorded key id and context are not consulted: AWS resolves the
|
||||
/// key from the blob and authenticates the caller's context against it.
|
||||
fn native_ciphertext(ciphertext: &[u8]) -> Result<Cow<'_, [u8]>> {
|
||||
if !is_data_key_envelope(ciphertext) {
|
||||
return Ok(Cow::Borrowed(ciphertext));
|
||||
}
|
||||
let envelope: DataKeyEnvelope = serde_json::from_slice(ciphertext)
|
||||
.map_err(|e| KmsError::cryptographic_error("parse", format!("Failed to parse data key envelope: {e}")))?;
|
||||
Ok(Cow::Owned(envelope.encrypted_key))
|
||||
}
|
||||
|
||||
impl AwsKmsBackend {
|
||||
/// Build the backend from a KMS configuration.
|
||||
///
|
||||
@@ -501,8 +483,7 @@ impl KmsBackend for AwsKmsBackend {
|
||||
// AWS ciphertext blobs identify their own key, so no key id is needed
|
||||
// here — and AWS refuses to decrypt with a disabled or pending-deletion
|
||||
// key, unlike the RustFS-managed backends.
|
||||
let ciphertext = native_ciphertext(&request.ciphertext)?;
|
||||
let ciphertext = ciphertext.as_ref();
|
||||
let ciphertext = request.ciphertext.as_slice();
|
||||
let context = encryption_context(&request.encryption_context);
|
||||
let tokens = grant_tokens(&request.grant_tokens);
|
||||
|
||||
@@ -582,29 +563,10 @@ impl KmsBackend for AwsKmsBackend {
|
||||
.ciphertext_blob
|
||||
.ok_or_else(|| KmsError::backend_error(format!("AWS KMS GenerateDataKey returned no ciphertext for key {key_id}")))?;
|
||||
|
||||
let master_key_id = output.key_id.unwrap_or_else(|| request.key_id.clone());
|
||||
// The envelope is what lets a reader recognize these bytes as
|
||||
// KMS-wrapped: the native blob identifies its key to AWS, but nothing
|
||||
// in it identifies AWS to RustFS.
|
||||
let envelope = DataKeyEnvelope {
|
||||
key_id: uuid::Uuid::new_v4().to_string(),
|
||||
master_key_id: master_key_id.clone(),
|
||||
key_spec: request.key_spec.as_str().to_string(),
|
||||
encrypted_key: ciphertext.into_inner(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: request.encryption_context,
|
||||
created_at: Zoned::now(),
|
||||
// AWS addresses backing keys internally and never exposes a version.
|
||||
master_key_version: None,
|
||||
// AWS binds the context into the blob itself; this field describes
|
||||
// only the local DekCrypto binding, which this backend never uses.
|
||||
context_binding: None,
|
||||
};
|
||||
|
||||
Ok(GenerateDataKeyResponse {
|
||||
key_id: master_key_id,
|
||||
key_id: output.key_id.unwrap_or_else(|| request.key_id.clone()),
|
||||
plaintext_key: plaintext.into_inner(),
|
||||
ciphertext_blob: serde_json::to_vec(&envelope)?,
|
||||
ciphertext_blob: ciphertext.into_inner(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -864,12 +826,6 @@ impl KmsBackend for AwsKmsBackend {
|
||||
.with_production_supported(true)
|
||||
}
|
||||
|
||||
/// Releases through 1.0.5 persisted the `GenerateDataKey` blob without an
|
||||
/// envelope, so objects written by them carry it bare.
|
||||
fn reads_bare_data_key_ciphertext(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
/// Observe, never destroy.
|
||||
///
|
||||
/// AWS runs the deletion window itself, so the sweep's job here is only to
|
||||
@@ -1016,49 +972,15 @@ mod tests {
|
||||
insta::assert_json_snapshot!("aws_backend_capabilities", capabilities_snapshot(backend.capabilities()));
|
||||
}
|
||||
|
||||
/// A data key blob shaped like AWS's: binary, not UTF-8, and nothing in it
|
||||
/// that RustFS could recognize.
|
||||
fn native_blob() -> Vec<u8> {
|
||||
let mut blob = vec![0x01, 0x02, 0x02, 0x00, 0x78];
|
||||
blob.extend((0u8..=178).map(|byte| byte.wrapping_mul(151).wrapping_add(0x80)));
|
||||
blob
|
||||
}
|
||||
|
||||
fn generate_data_key_event(plaintext: &[u8], blob: &[u8]) -> ReplayEvent {
|
||||
ok_event(serde_json::json!({
|
||||
"KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key",
|
||||
"Plaintext": BASE64.encode_to_string(plaintext),
|
||||
"CiphertextBlob": BASE64.encode_to_string(blob),
|
||||
}))
|
||||
}
|
||||
|
||||
fn decrypt_event(plaintext: &[u8]) -> ReplayEvent {
|
||||
ok_event(serde_json::json!({
|
||||
"KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key",
|
||||
"Plaintext": BASE64.encode_to_string(plaintext),
|
||||
"EncryptionAlgorithm": "SYMMETRIC_DEFAULT",
|
||||
}))
|
||||
}
|
||||
|
||||
/// The JSON body of the `index`-th request the backend actually sent.
|
||||
fn sent_body(http_client: &StaticReplayClient, index: usize) -> serde_json::Value {
|
||||
let request = http_client
|
||||
.actual_requests()
|
||||
.nth(index)
|
||||
.expect("the backend should have sent this request");
|
||||
serde_json::from_slice(request.body().bytes().expect("request body should be buffered"))
|
||||
.expect("AWS KMS requests are JSON")
|
||||
}
|
||||
|
||||
/// The data key has to come back in the envelope every reader classifies
|
||||
/// KMS-wrapped keys by. Returning the bare blob made each object written
|
||||
/// through this backend unreadable: the read path took the unrecognized
|
||||
/// bytes for a locally wrapped key and never called AWS at all.
|
||||
#[tokio::test]
|
||||
async fn generate_data_key_wraps_the_native_blob_in_an_envelope() {
|
||||
async fn generate_data_key_maps_the_native_response() {
|
||||
let plaintext = vec![7u8; 32];
|
||||
let blob = native_blob();
|
||||
let (http_client, backend) = scripted_backend(vec![generate_data_key_event(&plaintext, &blob)]);
|
||||
let ciphertext = b"encrypted-data-key".to_vec();
|
||||
let (http_client, backend) = scripted_backend(vec![ok_event(serde_json::json!({
|
||||
"KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key",
|
||||
"Plaintext": BASE64.encode_to_string(&plaintext),
|
||||
"CiphertextBlob": BASE64.encode_to_string(&ciphertext),
|
||||
}))]);
|
||||
|
||||
let response = backend
|
||||
.generate_data_key(generate_request())
|
||||
@@ -1066,89 +988,23 @@ mod tests {
|
||||
.expect("generate_data_key should map the AWS response");
|
||||
|
||||
assert_eq!(response.plaintext_key, plaintext);
|
||||
assert_eq!(response.ciphertext_blob, ciphertext);
|
||||
assert_eq!(response.key_id, "arn:aws:kms:us-east-1:111122223333:key/test-key");
|
||||
assert!(
|
||||
is_data_key_envelope(&response.ciphertext_blob),
|
||||
"the stored data key must be recognizable as KMS-wrapped from its bytes alone"
|
||||
);
|
||||
let envelope: DataKeyEnvelope = serde_json::from_slice(&response.ciphertext_blob).expect("envelope must parse");
|
||||
assert_eq!(envelope.encrypted_key, blob, "the envelope must carry the AWS blob byte for byte");
|
||||
assert_eq!(envelope.master_key_id, "arn:aws:kms:us-east-1:111122223333:key/test-key");
|
||||
assert_eq!(envelope.key_spec, "AES_256");
|
||||
assert_eq!(envelope.encryption_context, generate_request().encryption_context);
|
||||
assert_eq!(envelope.master_key_version, None);
|
||||
assert_eq!(envelope.context_binding, None);
|
||||
assert_eq!(http_client.actual_requests().count(), 1);
|
||||
}
|
||||
|
||||
/// What `generate_data_key` returns is what `decrypt` is later handed, so
|
||||
/// AWS must receive the blob it issued — not the envelope around it — along
|
||||
/// with the caller's context, which AWS authenticates.
|
||||
#[tokio::test]
|
||||
async fn decrypt_opens_a_generated_data_key_with_the_native_blob() {
|
||||
let plaintext = vec![9u8; 32];
|
||||
let blob = native_blob();
|
||||
let (http_client, backend) =
|
||||
scripted_backend(vec![generate_data_key_event(&plaintext, &blob), decrypt_event(&plaintext)]);
|
||||
|
||||
let data_key = backend
|
||||
.generate_data_key(generate_request())
|
||||
.await
|
||||
.expect("generate_data_key should succeed");
|
||||
let response = backend
|
||||
.decrypt(DecryptRequest {
|
||||
ciphertext: data_key.ciphertext_blob,
|
||||
encryption_context: generate_request().encryption_context,
|
||||
grant_tokens: Vec::new(),
|
||||
})
|
||||
.await
|
||||
.expect("a generated data key must decrypt");
|
||||
|
||||
assert_eq!(response.plaintext, plaintext);
|
||||
let sent = sent_body(&http_client, 1);
|
||||
assert_eq!(sent["CiphertextBlob"], BASE64.encode_to_string(&blob));
|
||||
assert_eq!(sent["EncryptionContext"], serde_json::json!({ "bucket": "aws" }));
|
||||
}
|
||||
|
||||
/// The caller's context reaches AWS even when it disagrees with the one
|
||||
/// recorded in the envelope: the recorded copy is unauthenticated, so AWS,
|
||||
/// which bound the real context into the blob, has to be the judge.
|
||||
#[tokio::test]
|
||||
async fn decrypt_sends_the_callers_context_not_the_envelopes() {
|
||||
let plaintext = vec![9u8; 32];
|
||||
let (http_client, backend) = scripted_backend(vec![
|
||||
generate_data_key_event(&plaintext, &native_blob()),
|
||||
error_event(400, "InvalidCiphertextException", "context does not match"),
|
||||
]);
|
||||
|
||||
let data_key = backend
|
||||
.generate_data_key(generate_request())
|
||||
.await
|
||||
.expect("generate_data_key should succeed");
|
||||
backend
|
||||
.decrypt(DecryptRequest {
|
||||
ciphertext: data_key.ciphertext_blob,
|
||||
encryption_context: HashMap::from([("bucket".to_string(), "other".to_string())]),
|
||||
grant_tokens: Vec::new(),
|
||||
})
|
||||
.await
|
||||
.expect_err("AWS rejects a context that does not match the blob");
|
||||
|
||||
assert_eq!(sent_body(&http_client, 1)["EncryptionContext"], serde_json::json!({ "bucket": "other" }));
|
||||
}
|
||||
|
||||
/// Objects written through 1.0.5 store the AWS blob bare, and `encrypt`
|
||||
/// still returns it that way, so a ciphertext that is not an envelope goes
|
||||
/// to AWS untouched.
|
||||
#[tokio::test]
|
||||
async fn decrypt_sends_a_bare_blob_unchanged() {
|
||||
async fn decrypt_recovers_the_plaintext_without_a_key_id() {
|
||||
let plaintext = b"recovered-data-key".to_vec();
|
||||
let blob = native_blob();
|
||||
let (http_client, backend) = scripted_backend(vec![decrypt_event(&plaintext)]);
|
||||
let (_http, backend) = scripted_backend(vec![ok_event(serde_json::json!({
|
||||
"KeyId": "arn:aws:kms:us-east-1:111122223333:key/test-key",
|
||||
"Plaintext": BASE64.encode_to_string(&plaintext),
|
||||
"EncryptionAlgorithm": "SYMMETRIC_DEFAULT",
|
||||
}))]);
|
||||
|
||||
let response = backend
|
||||
.decrypt(DecryptRequest {
|
||||
ciphertext: blob.clone(),
|
||||
ciphertext: b"blob".to_vec(),
|
||||
encryption_context: HashMap::new(),
|
||||
grant_tokens: Vec::new(),
|
||||
})
|
||||
@@ -1157,44 +1013,6 @@ mod tests {
|
||||
|
||||
assert_eq!(response.plaintext, plaintext);
|
||||
assert_eq!(response.encryption_algorithm.as_deref(), Some("SYMMETRIC_DEFAULT"));
|
||||
assert_eq!(sent_body(&http_client, 0)["CiphertextBlob"], BASE64.encode_to_string(&blob));
|
||||
assert!(backend.reads_bare_data_key_ciphertext());
|
||||
}
|
||||
|
||||
/// The object layer's write and read, through the same service entry
|
||||
/// points the S3 handlers use: the key a PUT stores must be the key a GET
|
||||
/// recovers, under the object's own context.
|
||||
#[tokio::test]
|
||||
async fn object_data_key_round_trips_through_the_encryption_service() {
|
||||
use crate::manager::KmsManager;
|
||||
use crate::service::ObjectEncryptionService;
|
||||
|
||||
let plaintext = vec![3u8; 32];
|
||||
let blob = native_blob();
|
||||
let (http_client, backend) =
|
||||
scripted_backend(vec![generate_data_key_event(&plaintext, &blob), decrypt_event(&plaintext)]);
|
||||
let manager = KmsManager::new(std::sync::Arc::new(backend), KmsConfig::aws(Some("us-east-1".to_string())));
|
||||
let service = ObjectEncryptionService::new(manager);
|
||||
let context = ObjectEncryptionContext::new("bucket".to_string(), "dir/object".to_string());
|
||||
|
||||
let (written, stored) = service
|
||||
.create_data_key(&Some("test-key".to_string()), &context)
|
||||
.await
|
||||
.expect("the write path must obtain a data key");
|
||||
assert!(is_data_key_envelope(&stored));
|
||||
assert!(service.reads_bare_data_key_ciphertext());
|
||||
|
||||
let read = service
|
||||
.decrypt_data_key(&stored, &context)
|
||||
.await
|
||||
.expect("the read path must recover the data key");
|
||||
|
||||
assert_eq!(read.plaintext_key, written.plaintext_key);
|
||||
assert_eq!(sent_body(&http_client, 1)["CiphertextBlob"], BASE64.encode_to_string(&blob));
|
||||
assert_eq!(
|
||||
sent_body(&http_client, 1)["EncryptionContext"],
|
||||
sent_body(&http_client, 0)["EncryptionContext"]
|
||||
);
|
||||
}
|
||||
|
||||
/// Asserts that a mapped failure landed on the intended `KmsError` variant.
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
|
||||
use crate::backends::{
|
||||
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, ListedKeyFailure, StateGatedOperation, UnreadableKeys,
|
||||
classify_listed_key_failure, destruction_requires_pending_deletion, ensure_key_status_permits, ensure_tag_keys_are_mutable,
|
||||
paginate_keys, started_at_the_first_key, validate_key_id_segment,
|
||||
classify_listed_key_failure, ensure_key_status_permits, ensure_tag_keys_are_mutable, paginate_keys, started_at_the_first_key,
|
||||
validate_key_id_segment,
|
||||
};
|
||||
use crate::config::KmsConfig;
|
||||
use crate::config::LocalConfig;
|
||||
@@ -2085,6 +2085,8 @@ impl KmsBackend for LocalKmsBackend {
|
||||
}
|
||||
|
||||
async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
|
||||
// For local backend, we'll implement immediate deletion by default
|
||||
// unless a pending window is specified
|
||||
let key_id = &request.key_id;
|
||||
|
||||
// Deletion is a read-modify-write (or read-then-remove) cycle, so hold
|
||||
@@ -2099,13 +2101,6 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.map_err(|_| KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
|
||||
let (deletion_date_str, deletion_date_dt) = if request.force_immediate.unwrap_or(false) {
|
||||
// Expedited destruction only skips the rest of the waiting window:
|
||||
// the key must already be pending deletion (or be a tombstone left
|
||||
// by a crashed removal, which may simply be completed).
|
||||
if matches!(master_key.status, KeyStatus::Active | KeyStatus::Disabled) {
|
||||
return Err(destruction_requires_pending_deletion(key_id));
|
||||
}
|
||||
|
||||
// Tombstone first: mark the record Deleted before removing the
|
||||
// file, so a crash between the two steps leaves a key that is
|
||||
// already unusable and whose removal can simply be re-run.
|
||||
@@ -3622,13 +3617,6 @@ mod tests {
|
||||
assert!(fsync_recorder::dir_sync_count(dir) > dirs_before, "update must fsync the key directory");
|
||||
|
||||
let backend = LocalKmsBackend { client };
|
||||
backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "durable-key".to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("schedule deletion");
|
||||
let dirs_before = fsync_recorder::dir_sync_count(dir);
|
||||
backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
|
||||
@@ -129,19 +129,6 @@ pub(crate) fn ensure_key_status_permits(key_id: &str, status: &KeyStatus, operat
|
||||
ensure_key_state_permits(key_id, &state, operation)
|
||||
}
|
||||
|
||||
/// Refusal for an expedited destruction of a key that was never scheduled.
|
||||
///
|
||||
/// Destruction always passes through `PendingDeletion`, the one state a
|
||||
/// deletion can still be cancelled from; `force_immediate` only skips the rest
|
||||
/// of the waiting window. Every backend that destroys keys refuses with this
|
||||
/// error instead of scheduling the key on the caller's behalf, so a success
|
||||
/// response always means the key is gone.
|
||||
pub(crate) fn destruction_requires_pending_deletion(key_id: &str) -> KmsError {
|
||||
KmsError::invalid_key_state(format!(
|
||||
"Key {key_id} is not pending deletion: schedule the deletion before requesting immediate destruction"
|
||||
))
|
||||
}
|
||||
|
||||
/// Tag key that carries the key's identity rather than user metadata.
|
||||
///
|
||||
/// The key-creation path lifts `name` out of the caller's tag map and uses it
|
||||
@@ -584,18 +571,6 @@ pub trait KmsBackend: Send + Sync {
|
||||
BackendCapabilities::minimal()
|
||||
}
|
||||
|
||||
/// Whether objects on disk may carry this backend's data-key ciphertext
|
||||
/// without a [`crate::encryption::DataKeyEnvelope`] around it.
|
||||
///
|
||||
/// Every backend wraps the data keys it generates in an envelope, which is
|
||||
/// what lets a reader recognize a KMS-wrapped key from its bytes alone. A
|
||||
/// backend returns `true` only when an earlier release persisted its
|
||||
/// provider-native ciphertext bare, so those bytes still have to be routed
|
||||
/// to [`Self::decrypt`] even though nothing in them identifies the backend.
|
||||
fn reads_bare_data_key_ciphertext(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Remove a key whose scheduled deletion deadline has passed.
|
||||
///
|
||||
/// Used by the background deletion worker. Implementations must re-check
|
||||
|
||||
@@ -20,9 +20,9 @@ use crate::backends::vault_credentials::{
|
||||
};
|
||||
use crate::backends::{
|
||||
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, ListedKeyFailure, StateGatedOperation, UnreadableKeys,
|
||||
classify_listed_key_failure, destruction_requires_pending_deletion, empty_key_page, ensure_key_state_permits,
|
||||
ensure_key_status_permits, ensure_rewrap_context_matches, ensure_tag_keys_are_mutable, list_keys_page_size, paginate_keys,
|
||||
started_at_the_first_key, validate_key_id_segment,
|
||||
classify_listed_key_failure, empty_key_page, ensure_key_state_permits, ensure_key_status_permits,
|
||||
ensure_rewrap_context_matches, ensure_tag_keys_are_mutable, list_keys_page_size, paginate_keys, started_at_the_first_key,
|
||||
validate_key_id_segment,
|
||||
};
|
||||
use crate::config::{KmsConfig, VaultConfig};
|
||||
use crate::encryption::{
|
||||
@@ -2135,6 +2135,8 @@ impl KmsBackend for VaultKmsBackend {
|
||||
}
|
||||
|
||||
async fn delete_key(&self, request: DeleteKeyRequest) -> Result<DeleteKeyResponse> {
|
||||
// For Vault backend, we'll mark keys for deletion but not physically delete them
|
||||
// This allows for recovery during the pending window
|
||||
let key_id = &request.key_id;
|
||||
|
||||
// First, check if the key exists and get its metadata
|
||||
@@ -2147,32 +2149,41 @@ impl KmsBackend for VaultKmsBackend {
|
||||
};
|
||||
|
||||
let deletion_date = if request.force_immediate.unwrap_or(false) {
|
||||
// Expedited destruction only skips the rest of the waiting window:
|
||||
// the key must already be pending deletion (or be a tombstone left
|
||||
// by a crashed removal, which may simply be completed).
|
||||
//
|
||||
// Tombstone first: mark the record Deleted before removing it, so
|
||||
// a crash between the two steps leaves a key that is already
|
||||
// unusable and whose removal can simply be re-run. Written
|
||||
// check-and-set with re-validation: a concurrent cancellation
|
||||
// that commits first wins and fails this removal instead of
|
||||
// being overwritten.
|
||||
self.client
|
||||
.update_key_data_with_cas(key_id, |key_data| match key_data.status {
|
||||
KeyStatus::Deleted => Ok(CasMutation::Skip(())),
|
||||
KeyStatus::PendingDeletion => {
|
||||
key_data.status = KeyStatus::Deleted;
|
||||
Ok(CasMutation::Write(()))
|
||||
}
|
||||
KeyStatus::Active | KeyStatus::Disabled => Err(destruction_requires_pending_deletion(key_id)),
|
||||
})
|
||||
.await?;
|
||||
self.client.delete_key(key_id).await?;
|
||||
key_metadata.key_state = KeyState::PendingDeletion;
|
||||
key_metadata.deletion_date = Some(Zoned::now());
|
||||
// Check if key is already in PendingDeletion state (or a tombstone
|
||||
// left by a crashed removal, which may simply be completed)
|
||||
if key_metadata.key_state == KeyState::PendingDeletion || key_metadata.key_state == KeyState::Unavailable {
|
||||
// Tombstone first: mark the record Deleted before removing it,
|
||||
// so a crash between the two steps leaves a key that is already
|
||||
// unusable and whose removal can simply be re-run. Written
|
||||
// check-and-set with re-validation: a concurrent cancellation
|
||||
// that commits first wins and fails this removal instead of
|
||||
// being overwritten.
|
||||
self.client
|
||||
.update_key_data_with_cas(key_id, |key_data| match key_data.status {
|
||||
KeyStatus::Deleted => Ok(CasMutation::Skip(())),
|
||||
KeyStatus::PendingDeletion => {
|
||||
key_data.status = KeyStatus::Deleted;
|
||||
Ok(CasMutation::Write(()))
|
||||
}
|
||||
KeyStatus::Active | KeyStatus::Disabled => {
|
||||
Err(KmsError::invalid_key_state(format!("Key {key_id} is no longer pending deletion")))
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
// Force immediate deletion: physically delete the key from Vault storage
|
||||
self.client.delete_key(key_id).await?;
|
||||
|
||||
// No deletion date: the key is permanently gone, not scheduled.
|
||||
None
|
||||
// Return empty deletion_date to indicate key was permanently deleted
|
||||
None
|
||||
} else {
|
||||
// For non-pending keys, mark as PendingDeletion
|
||||
let marked_at = Zoned::now();
|
||||
self.mark_key_pending_deletion(key_id, &marked_at).await?;
|
||||
key_metadata.key_state = KeyState::PendingDeletion;
|
||||
key_metadata.deletion_date = Some(marked_at);
|
||||
|
||||
None
|
||||
}
|
||||
} else {
|
||||
// Schedule for deletion (default 30 days)
|
||||
ensure_key_state_permits(key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
|
||||
@@ -4018,101 +4029,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Destruction always passes through the cancellable pending state. An
|
||||
/// immediate deletion of a key that was never scheduled is refused without
|
||||
/// writing anything: parking it in PendingDeletion and reporting success
|
||||
/// would tell the caller a key is gone while every object under it stays
|
||||
/// readable.
|
||||
#[tokio::test]
|
||||
async fn wired_immediate_deletion_of_an_enabled_key_is_refused_without_writing() {
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// describe_key: key info plus stored metadata.
|
||||
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
|
||||
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
|
||||
// The check-and-set read observes a key that is not pending.
|
||||
ScriptedResponse::ok(kv2_metadata_read_data(1)),
|
||||
ScriptedResponse::ok(kv2_read_data(&healthy_key_data())),
|
||||
])
|
||||
.await;
|
||||
let config = KmsConfig::vault(
|
||||
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
|
||||
"scripted-token".to_string(),
|
||||
)
|
||||
.with_insecure_development_defaults();
|
||||
let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build");
|
||||
|
||||
let error = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: Some("wired-key".to_string()),
|
||||
})
|
||||
.await
|
||||
.expect_err("a key that is not pending deletion must not be destroyed in one request");
|
||||
assert!(
|
||||
matches!(&error, KmsError::InvalidOperation { message } if message.contains("not pending deletion")),
|
||||
"got {error:?}"
|
||||
);
|
||||
|
||||
let requests = vault.requests();
|
||||
assert!(
|
||||
requests.iter().all(|line| line.starts_with("GET ")),
|
||||
"a refused immediate deletion must neither write nor delete: {requests:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The other half: once the key is pending deletion, one immediate request
|
||||
/// tombstones the record and removes it.
|
||||
#[tokio::test]
|
||||
async fn wired_immediate_deletion_of_a_pending_key_destroys_it() {
|
||||
let mut pending = healthy_key_data();
|
||||
pending.status = KeyStatus::PendingDeletion;
|
||||
pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
|
||||
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// describe_key: key info plus stored metadata.
|
||||
ScriptedResponse::ok(kv2_read_data(&pending)),
|
||||
ScriptedResponse::ok(kv2_read_data(&pending)),
|
||||
// Check-and-set tombstone.
|
||||
ScriptedResponse::ok(kv2_metadata_read_data(1)),
|
||||
ScriptedResponse::ok(kv2_read_data(&pending)),
|
||||
ScriptedResponse::ok(kv2_write_ack()),
|
||||
// The key was never rotated: no version records to purge.
|
||||
ScriptedResponse::empty_list_404(),
|
||||
// Removal of the key record itself.
|
||||
ScriptedResponse::ok(serde_json::Value::Null),
|
||||
])
|
||||
.await;
|
||||
let config = KmsConfig::vault(
|
||||
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
|
||||
"scripted-token".to_string(),
|
||||
)
|
||||
.with_insecure_development_defaults();
|
||||
let backend = VaultKmsBackend::new(config).await.expect("vault kv2 backend should build");
|
||||
|
||||
let response = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: Some("wired-key".to_string()),
|
||||
})
|
||||
.await
|
||||
.expect("immediate deletion of a pending key should succeed");
|
||||
assert!(response.deletion_date.is_none(), "a destroyed key has no future deadline to report");
|
||||
|
||||
let requests = vault.requests();
|
||||
assert_eq!(requests.len(), 7, "{requests:?}");
|
||||
let tombstone = parse_write_body(&vault.request_bodies()[4]);
|
||||
assert_eq!(tombstone["options"]["cas"], serde_json::json!(1), "{tombstone}");
|
||||
assert_eq!(tombstone["data"]["status"], serde_json::json!("Deleted"), "{tombstone}");
|
||||
assert_eq!(
|
||||
requests[6], "DELETE /v1/secret/metadata/rustfs/kms/keys/wired-key",
|
||||
"the key record must be removed: {requests:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wired_schedule_deletion_retries_after_cas_conflict() {
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
|
||||
@@ -21,9 +21,8 @@ use crate::backends::vault_credentials::{
|
||||
};
|
||||
use crate::backends::{
|
||||
BackendCapabilities, ExpiredKeyRemoval, KmsBackend, ListedKeyFailure, StateGatedOperation, UnreadableKeys,
|
||||
classify_listed_key_failure, destruction_requires_pending_deletion, empty_key_page, ensure_key_state_permits,
|
||||
ensure_rewrap_context_matches, ensure_tag_keys_are_mutable, list_keys_page_size, paginate_keys, started_at_the_first_key,
|
||||
validate_key_id_segment,
|
||||
classify_listed_key_failure, empty_key_page, ensure_key_state_permits, ensure_rewrap_context_matches,
|
||||
ensure_tag_keys_are_mutable, list_keys_page_size, paginate_keys, started_at_the_first_key, validate_key_id_segment,
|
||||
};
|
||||
use crate::config::{KmsConfig, VaultTransitConfig};
|
||||
use crate::encryption::{DataKeyEnvelope, generate_key_material};
|
||||
@@ -62,10 +61,6 @@ use zeroize::Zeroizing;
|
||||
/// replay of the stale snapshot.
|
||||
const METADATA_CAS_ATTEMPTS: usize = 3;
|
||||
|
||||
/// The sole error message of the 400 Vault Transit returns when a data-path
|
||||
/// call names a key that does not exist.
|
||||
const TRANSIT_KEY_NOT_FOUND_MESSAGE: &str = "encryption key not found";
|
||||
|
||||
/// TTL bound on cached metadata records. This caps how long one node can keep
|
||||
/// acting on lifecycle state another node has since changed (disable,
|
||||
/// schedule-deletion): the divergence window is one TTL instead of "until
|
||||
@@ -491,14 +486,6 @@ impl VaultTransitKmsClient {
|
||||
match error {
|
||||
vaultrs::error::ClientError::ResponseWrapError => KmsError::key_not_found(key_id),
|
||||
vaultrs::error::ClientError::APIError { code: 404, .. } => KmsError::key_not_found(key_id),
|
||||
// Transit answers decrypt and rewrap for a key that does not exist
|
||||
// with a 400, not the 404 a key read gets. Left as a backend error
|
||||
// it would surface as a retryable outage instead of a missing key.
|
||||
vaultrs::error::ClientError::APIError { code: 400, ref errors }
|
||||
if errors.iter().any(|message| message == TRANSIT_KEY_NOT_FOUND_MESSAGE) =>
|
||||
{
|
||||
KmsError::key_not_found(key_id)
|
||||
}
|
||||
other => KmsError::backend_error(format!("Vault Transit {operation} failed for key {key_id}: {other}")),
|
||||
}
|
||||
}
|
||||
@@ -1722,35 +1709,25 @@ impl KmsBackend for VaultTransitKmsBackend {
|
||||
let mut key_metadata = self.client.key_metadata_response(&key_id).await?;
|
||||
|
||||
let deletion_date = if request.force_immediate.unwrap_or(false) {
|
||||
// Expedited destruction only skips the rest of the waiting window:
|
||||
// the key must already be pending deletion (or be a tombstone left
|
||||
// by a crashed removal, which may simply be completed).
|
||||
//
|
||||
// Tombstone first, as the expired-key sweep does: an Unavailable
|
||||
// record is rejected by every state gate, so a crash before the
|
||||
// Transit key is destroyed leaves a key that is already unusable.
|
||||
// Re-checked against every fresh snapshot: a concurrent
|
||||
// cancellation that commits first fails this removal.
|
||||
let now = Zoned::now();
|
||||
self.client
|
||||
.mutate_key_metadata(&key_id, |metadata| match metadata.key_state {
|
||||
KeyState::PendingDeletion | KeyState::Unavailable => {
|
||||
metadata.key_state = KeyState::Unavailable;
|
||||
if key_metadata.key_state == KeyState::PendingDeletion {
|
||||
if !self.client.read_transit_key(&key_id).await?.deletion_allowed {
|
||||
self.client.allow_transit_key_deletion(&key_id).await?;
|
||||
}
|
||||
self.client.delete_transit_key(&key_id).await?;
|
||||
self.client.delete_key_metadata(&key_id).await?;
|
||||
None
|
||||
} else {
|
||||
let now = Zoned::now();
|
||||
self.client
|
||||
.mutate_key_metadata(&key_id, |metadata| {
|
||||
metadata.key_state = KeyState::PendingDeletion;
|
||||
metadata.deletion_date = Some(now.clone());
|
||||
Ok(())
|
||||
}
|
||||
KeyState::Enabled | KeyState::Disabled | KeyState::PendingImport => {
|
||||
Err(destruction_requires_pending_deletion(&key_id))
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
if !self.client.read_transit_key(&key_id).await?.deletion_allowed {
|
||||
self.client.allow_transit_key_deletion(&key_id).await?;
|
||||
})
|
||||
.await?;
|
||||
key_metadata = self.client.key_metadata_response(&key_id).await?;
|
||||
None
|
||||
}
|
||||
self.client.delete_transit_key(&key_id).await?;
|
||||
self.client.delete_key_metadata(&key_id).await?;
|
||||
key_metadata.key_state = KeyState::PendingDeletion;
|
||||
key_metadata.deletion_date = Some(now);
|
||||
None
|
||||
} else {
|
||||
ensure_key_state_permits(&key_id, &key_metadata.key_state, StateGatedOperation::ScheduleDeletion)?;
|
||||
|
||||
@@ -3028,149 +3005,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// Transit reports a missing key on the data path as a 400. Reads of
|
||||
/// objects under a destroyed key must see a missing key, not a backend
|
||||
/// outage the client is told to retry.
|
||||
#[tokio::test]
|
||||
async fn wired_decrypt_under_a_destroyed_transit_key_reports_key_not_found() {
|
||||
let (vault, client) = scripted_client(vec![ScriptedResponse::error(400, TRANSIT_KEY_NOT_FOUND_MESSAGE)]).await;
|
||||
let envelope = DataKeyEnvelope {
|
||||
key_id: "dek".to_string(),
|
||||
master_key_id: "wired-key".to_string(),
|
||||
key_spec: "AES_256".to_string(),
|
||||
encrypted_key: b"vault:v1:AAAA".to_vec(),
|
||||
nonce: Vec::new(),
|
||||
encryption_context: HashMap::new(),
|
||||
created_at: Zoned::now(),
|
||||
master_key_version: Some(1),
|
||||
context_binding: None,
|
||||
};
|
||||
|
||||
let error = client
|
||||
.decrypt(
|
||||
&DecryptRequest {
|
||||
ciphertext: serde_json::to_vec(&envelope).expect("serialize envelope"),
|
||||
encryption_context: HashMap::new(),
|
||||
grant_tokens: Vec::new(),
|
||||
},
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect_err("decrypting under a destroyed key must fail");
|
||||
assert!(matches!(&error, KmsError::KeyNotFound { .. }), "got {error:?}");
|
||||
assert_eq!(
|
||||
vault.requests(),
|
||||
["POST /v1/transit/decrypt/wired-key"],
|
||||
"a missing key must not be retried"
|
||||
);
|
||||
}
|
||||
|
||||
/// Destruction always passes through the cancellable pending state. An
|
||||
/// immediate deletion of a key that was never scheduled is refused without
|
||||
/// writing anything: parking it in PendingDeletion and reporting success
|
||||
/// would tell the caller a key is gone while every object under it stays
|
||||
/// readable.
|
||||
#[tokio::test]
|
||||
async fn wired_immediate_deletion_of_an_enabled_key_is_refused_without_writing() {
|
||||
let enabled = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// Current key metadata.
|
||||
ScriptedResponse::ok(transit_key_read_data("wired-key")),
|
||||
ScriptedResponse::ok(metadata_read_data(&enabled)),
|
||||
// The check-and-set read observes a key that is not pending.
|
||||
ScriptedResponse::ok(kv2_metadata_read_data(1)),
|
||||
ScriptedResponse::ok(metadata_read_data(&enabled)),
|
||||
])
|
||||
.await;
|
||||
let config = KmsConfig::vault_transit(
|
||||
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
|
||||
"scripted-token".to_string(),
|
||||
)
|
||||
.with_insecure_development_defaults();
|
||||
let backend = VaultTransitKmsBackend::new(config)
|
||||
.await
|
||||
.expect("vault transit backend should build");
|
||||
|
||||
let error = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: Some("wired-key".to_string()),
|
||||
})
|
||||
.await
|
||||
.expect_err("a key that is not pending deletion must not be destroyed in one request");
|
||||
assert!(
|
||||
matches!(&error, KmsError::InvalidOperation { message } if message.contains("not pending deletion")),
|
||||
"got {error:?}"
|
||||
);
|
||||
|
||||
let requests = vault.requests();
|
||||
assert!(
|
||||
requests.iter().all(|line| line.starts_with("GET ")),
|
||||
"a refused immediate deletion must neither write nor delete: {requests:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The other half: once the key is pending deletion, one immediate request
|
||||
/// tombstones the metadata, destroys the Transit key and removes the record.
|
||||
#[tokio::test]
|
||||
async fn wired_immediate_deletion_of_a_pending_key_destroys_it() {
|
||||
let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
pending.key_state = KeyState::PendingDeletion;
|
||||
pending.deletion_date = Some(Zoned::now() + Duration::from_secs(7 * 86400));
|
||||
let vault = ScriptedVault::serve(vec![
|
||||
// Current key metadata for the response.
|
||||
ScriptedResponse::ok(transit_key_read_data("wired-key")),
|
||||
ScriptedResponse::ok(metadata_read_data(&pending)),
|
||||
// Check-and-set tombstone.
|
||||
ScriptedResponse::ok(kv2_metadata_read_data(1)),
|
||||
ScriptedResponse::ok(metadata_read_data(&pending)),
|
||||
ScriptedResponse::ok(kv2_write_ack()),
|
||||
// Transit refuses deletion until the key is configured to allow it.
|
||||
ScriptedResponse::ok(transit_key_read_data("wired-key")),
|
||||
ScriptedResponse::ok(serde_json::Value::Null),
|
||||
// Transit key, then its metadata record.
|
||||
ScriptedResponse::ok(serde_json::Value::Null),
|
||||
ScriptedResponse::ok(serde_json::Value::Null),
|
||||
])
|
||||
.await;
|
||||
let config = KmsConfig::vault_transit(
|
||||
url::Url::parse(&vault.address).expect("scripted vault address should parse"),
|
||||
"scripted-token".to_string(),
|
||||
)
|
||||
.with_insecure_development_defaults();
|
||||
let backend = VaultTransitKmsBackend::new(config)
|
||||
.await
|
||||
.expect("vault transit backend should build");
|
||||
|
||||
let response = backend
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "wired-key".to_string(),
|
||||
pending_window_in_days: None,
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: Some("wired-key".to_string()),
|
||||
})
|
||||
.await
|
||||
.expect("immediate deletion of a pending key should succeed");
|
||||
assert!(response.deletion_date.is_none(), "a destroyed key has no future deadline to report");
|
||||
|
||||
let requests = vault.requests();
|
||||
assert_eq!(requests.len(), 9, "{requests:?}");
|
||||
let tombstone: serde_json::Value = serde_json::from_str(&vault.request_bodies()[4]).expect("tombstone body is JSON");
|
||||
assert_eq!(tombstone["options"]["cas"], serde_json::json!(1), "{tombstone}");
|
||||
assert_eq!(tombstone["data"]["key_state"], serde_json::json!("Unavailable"), "{tombstone}");
|
||||
assert_eq!(
|
||||
requests[7], "DELETE /v1/transit/keys/wired-key",
|
||||
"the transit key must be destroyed: {requests:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
requests[8],
|
||||
format!("DELETE {METADATA_VERSION_PATH}"),
|
||||
"the metadata record must be removed: {requests:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wired_expired_sweep_backs_off_when_cancel_wins_the_cas_race() {
|
||||
let mut pending = TransitKeyMetadata::from_create_request(&CreateKeyRequest::default());
|
||||
|
||||
@@ -1608,14 +1608,10 @@ mod tests {
|
||||
assert!(config.validate().is_err());
|
||||
assert!(config.with_insecure_development_defaults().validate().is_ok());
|
||||
|
||||
#[cfg(windows)]
|
||||
let production_key_dir = PathBuf::from(r"C:\rustfs\kms");
|
||||
#[cfg(not(windows))]
|
||||
let production_key_dir = PathBuf::from("/var/lib/rustfs/kms");
|
||||
let production_config = KmsConfig {
|
||||
backend: KmsBackend::Local,
|
||||
backend_config: BackendConfig::Local(LocalConfig {
|
||||
key_dir: production_key_dir,
|
||||
key_dir: PathBuf::from("/var/lib/rustfs/kms"),
|
||||
master_key: Some("production-master-key".to_string()),
|
||||
file_permissions: Some(0o600),
|
||||
}),
|
||||
|
||||
@@ -466,8 +466,6 @@ impl KmsManager {
|
||||
/// opt-in and a per-request confirmation that echoes the key id; without
|
||||
/// either, the request is refused rather than downgraded to a scheduled
|
||||
/// deletion, so a caller never believes a key is gone when it is not.
|
||||
/// The backend adds the last condition under its own synchronization: the
|
||||
/// key must already be pending deletion.
|
||||
fn check_deletion_request(&self, request: &DeleteKeyRequest) -> Result<()> {
|
||||
if !request.force_immediate.unwrap_or(false) {
|
||||
let days = request.pending_window_in_days.unwrap_or(DEFAULT_PENDING_DELETION_WINDOW_DAYS);
|
||||
@@ -495,7 +493,7 @@ impl KmsManager {
|
||||
|
||||
warn!(
|
||||
key_id = %request.key_id,
|
||||
"immediate KMS key deletion authorized; a key pending deletion is destroyed without waiting out its window and cannot be recovered"
|
||||
"immediate KMS key deletion accepted; key material is destroyed without a waiting window and cannot be recovered"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -670,11 +668,6 @@ impl KmsManager {
|
||||
self.backend.capabilities()
|
||||
}
|
||||
|
||||
/// See [`KmsBackend::reads_bare_data_key_ciphertext`].
|
||||
pub fn reads_bare_data_key_ciphertext(&self) -> bool {
|
||||
self.backend.reads_bare_data_key_ciphertext()
|
||||
}
|
||||
|
||||
/// Direct handle to the configured backend, bypassing the metadata cache.
|
||||
/// Used by background maintenance that must observe fresh state.
|
||||
pub(crate) fn backend(&self) -> Arc<dyn KmsBackend> {
|
||||
@@ -1475,7 +1468,6 @@ mod tests {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, true).await;
|
||||
let key_id = create_named_key(&manager, "confirmed-force-delete").await;
|
||||
schedule_deletion(&manager, &key_id).await;
|
||||
|
||||
manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
@@ -1494,43 +1486,6 @@ mod tests {
|
||||
assert!(matches!(error, KmsError::KeyNotFound { .. }), "expected KeyNotFound, got {error:?}");
|
||||
}
|
||||
|
||||
async fn schedule_deletion(manager: &KmsManager, key_id: &str) {
|
||||
manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect("scheduling the deletion must succeed");
|
||||
}
|
||||
|
||||
/// Destruction always passes through the cancellable pending state: a
|
||||
/// fully authorized immediate deletion of a key that was never scheduled
|
||||
/// is refused, and leaves the key exactly as it was.
|
||||
#[tokio::test]
|
||||
async fn immediate_deletion_of_a_key_that_was_never_scheduled_is_refused() {
|
||||
let temp_dir = tempdir().expect("Failed to create temp dir");
|
||||
let manager = deletion_manager(&temp_dir, true).await;
|
||||
let key_id = create_named_key(&manager, "unscheduled-force-delete").await;
|
||||
let probe = data_key_probe(&manager, &key_id).await;
|
||||
|
||||
let error = manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
force_immediate: Some(true),
|
||||
confirm_key_id: Some(key_id.clone()),
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.expect_err("a key that is not pending deletion must not be destroyed in one request");
|
||||
assert!(
|
||||
matches!(&error, KmsError::InvalidOperation { message } if message.contains("not pending deletion")),
|
||||
"got {error:?}"
|
||||
);
|
||||
|
||||
assert_key_material_intact(&manager, &key_id, &probe).await;
|
||||
}
|
||||
|
||||
/// Reference checker whose answer is fixed, standing in for the server's
|
||||
/// bucket-configuration gate.
|
||||
struct StaticReferences(Vec<String>);
|
||||
@@ -1669,7 +1624,6 @@ mod tests {
|
||||
.await
|
||||
.with_deletion_reference_checker(Some(Arc::new(StaticReferences(Vec::new()))));
|
||||
let key_id = create_named_key(&manager, "unreferenced-confirmed-force-delete").await;
|
||||
schedule_deletion(&manager, &key_id).await;
|
||||
|
||||
manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
|
||||
@@ -304,15 +304,6 @@ impl ObjectEncryptionService {
|
||||
self.kms_manager.backend_capabilities()
|
||||
}
|
||||
|
||||
/// Whether a stored data key that is not a RustFS envelope may still belong
|
||||
/// to the active backend, and so must be offered to
|
||||
/// [`Self::decrypt_data_key`] instead of being treated as foreign.
|
||||
///
|
||||
/// See [`crate::backends::KmsBackend::reads_bare_data_key_ciphertext`].
|
||||
pub fn reads_bare_data_key_ciphertext(&self) -> bool {
|
||||
self.kms_manager.reads_bare_data_key_ciphertext()
|
||||
}
|
||||
|
||||
/// Re-wrap an object's encrypted data key onto its master key's current
|
||||
/// version, without the plaintext data key ever reaching the caller.
|
||||
///
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
|
||||
mod common;
|
||||
|
||||
use common::{TestKms, assert_invalid_operation, ctx, schedule_deletion};
|
||||
use common::{TestKms, assert_invalid_operation, ctx};
|
||||
use rustfs_kms::{
|
||||
CancelKeyDeletionRequest, DeleteKeyRequest, DescribeKeyRequest, GenerateDataKeyRequest, KeySpec, KeyState, KmsManager,
|
||||
};
|
||||
@@ -190,7 +190,6 @@ async fn a_destroyed_key_cannot_be_served_from_cache() {
|
||||
|
||||
// Warm the cache, then destroy the key outright.
|
||||
assert_eq!(describe_state(&manager, &key_id).await, KeyState::Enabled);
|
||||
schedule_deletion(&manager, &key_id).await;
|
||||
manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: key_id.clone(),
|
||||
|
||||
@@ -32,7 +32,7 @@ use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use common::{TestKms, assert_invalid_operation, ctx, schedule_deletion};
|
||||
use common::{TestKms, assert_invalid_operation, ctx};
|
||||
use rustfs_kms::{
|
||||
CreateKeyRequest, DecryptRequest, DeleteKeyRequest, DescribeKeyRequest, GenerateDataKeyRequest, KeySpec, KeyState, KmsConfig,
|
||||
KmsError, KmsServiceManager, KmsServiceStatus,
|
||||
@@ -519,7 +519,6 @@ async fn a_destroyed_key_stays_destroyed_across_a_restart() {
|
||||
kms.create_key("gone-for-good").await;
|
||||
kms.create_key("kept").await;
|
||||
|
||||
schedule_deletion(&manager, "gone-for-good").await;
|
||||
manager
|
||||
.delete_key(DeleteKeyRequest {
|
||||
key_id: "gone-for-good".to_string(),
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user