From b7805caa58163032399ae0609b63198c31da6e2a Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 1 Aug 2026 18:57:23 +0800 Subject: [PATCH] ci: stop archiving every dependency's unpacked source in each cache (#5566) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four consolidated ci keys plus the build keys still do not fit the repository's fixed 10GB Actions cache quota, so LRU keeps evicting them: measured demand is ci-dev 2331MB + ci-feat-proto 2310MB + ci-feat-rio 1951MB + ci-uring 1317MB + two build legs at ~1429MB + cargo-deny 844MB, and main pushes add two more build legs. That is roughly 14.4GB against 10.24GB. The symptom is misleading: Cache Warm reports every job successful and the restores log "full match: true", yet ci-feat-rio and ci-uring disappear from the cache list between runs. cache-all-crates was the wrong default for this repository. With it set to true, rust-cache's cleanup returns before pruning ~/.cargo/registry/src and its config archives the whole registry, so every cache carried the unpacked source tree of every dependency — not, as the name suggests, just a few extra crates. Setting it to false is rust-cache's own default and loses no coverage: the package set comes from `cargo metadata --all-features`, a strict superset of any single lane's feature closure; -sys crates are explicitly exempt from pruning, since their source timestamps would otherwise trigger rebuilds; and everything pruned is re-unpacked from the .crate files still in registry/cache, whose mtimes crates.io normalises, so cargo fingerprints stay valid. Applied to the setup composite and to audit.yml's own rust-cache. Cache Warm now also reports the sizes of registry/src, registry/cache, registry/index, ~/.cargo/git and target/ to the step summary, immediately before rust-cache's post step archives them, so the size of the effect is measured rather than assumed. The new sizes only appear once the cache key next rotates, since rust-cache skips the save entirely on an exact key hit. Deliberately not forcing that by bumping prefix-key: it would invalidate every family at once and produce a repository-wide cold build. Refs: rustfs/backlog#1598, rustfs/backlog#1600 --- .github/actions/setup/action.yml | 13 +++- .github/workflows/audit.yml | 4 +- .github/workflows/cache-warm.yml | 23 +++++-- .github/workflows/runner-hygiene.yml | 81 +++++++++++++++++++++++++ scripts/ci/check_runner_ephemerality.sh | 77 +++++++++++++++++++++++ 5 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/runner-hygiene.yml create mode 100755 scripts/ci/check_runner_ephemerality.sh diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index fe0f57b7d..9359bb066 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -111,7 +111,18 @@ runs: - name: Setup Rust cache uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: - cache-all-crates: true + # false is rust-cache's own default. With true, cleanup.ts returns + # *before* pruning ~/.cargo/registry/src, and config.ts archives the + # whole registry — so every cache carried the unpacked source tree of + # every dependency, not just "a few extra crates". + # + # No coverage is lost: getPackages runs `cargo metadata --all-features`, + # a strict superset of any single lane's feature closure, and -sys crates + # are explicitly exempted from pruning (their src timestamps would + # otherwise trigger rebuilds). Anything pruned is re-unpacked from the + # .crate files still in registry/cache, whose mtimes crates.io + # normalises, so cargo fingerprints stay valid. + cache-all-crates: false cache-on-failure: true shared-key: ${{ inputs.cache-shared-key }} save-if: ${{ inputs.cache-save-if }} diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 30d889763..ffb8fd972 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -100,7 +100,9 @@ jobs: - name: Setup Rust cache uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: - cache-all-crates: true + # Same reasoning as the setup composite: true archives every + # dependency's unpacked source tree. + cache-all-crates: false cache-on-failure: true shared-key: rustfs-cargo-deny save-if: ${{ github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/cache-warm.yml b/.github/workflows/cache-warm.yml index d69a89c79..9435a992f 100644 --- a/.github/workflows/cache-warm.yml +++ b/.github/workflows/cache-warm.yml @@ -107,10 +107,6 @@ jobs: cache-save-if: 'true' install-build-packaging-tools: 'false' - # --all-targets covers the test binaries nextest builds, including - # e2e_test, which test-and-lint's own run excludes. The second build adds - # the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses - # and that no lint lane enables. # rustfs/backlog#1601 gate. sccache can only cache compilation units whose # --emit includes link, so it covers workspace rlibs and nothing else: # clippy is metadata-only, and the ~100 test binaries, the rustfs bin and @@ -138,6 +134,10 @@ jobs: retention-days: 30 if-no-files-found: error + # --all-targets covers the test binaries nextest builds, including + # e2e_test, which test-and-lint's own run excludes. The second build adds + # the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses + # and that no lint lane enables. - name: Build ci-dev superset env: # Same limit ci.yml puts on its nextest step: this builds the same @@ -148,6 +148,21 @@ jobs: cargo build --workspace --all-targets cargo build -p rustfs --bins --features e2e-test-hooks + # Runs before rust-cache's post step, so these are the sizes it is about + # to archive. Reported so the cache-all-crates decision stays evidence-led: + # registry/src is what that flag prunes, registry/cache is what the pruned + # sources are re-unpacked from. See rustfs/backlog#1600. + - name: Report cache input sizes + if: always() + run: | + { + echo "### Cache input sizes (ci-dev)" + echo '```' + du -sh ~/.cargo/registry/src ~/.cargo/registry/cache ~/.cargo/registry/index \ + ~/.cargo/git target 2>/dev/null || true + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + # Readers: test-and-lint-rio-v2, build-rustfs-debug-binary-rio-v2. warm-ci-feat-rio: name: Warm ci-feat-rio diff --git a/.github/workflows/runner-hygiene.yml b/.github/workflows/runner-hygiene.yml new file mode 100644 index 000000000..c231b5706 --- /dev/null +++ b/.github/workflows/runner-hygiene.yml @@ -0,0 +1,81 @@ +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Asserts that the self-hosted runners are still ephemeral — one job per pod. +# +# This repository is public and its pull_request jobs run on those runners, +# executing the PR's own build.rs, proc-macros and tests. The only thing keeping +# that code from reaching a later job is that each ARC pod handles exactly one +# job and is then destroyed. That guarantee lives in the ARC scale-set +# configuration, outside this repository, where it can be changed without any PR +# — so it is asserted here from the outside, against real run data, instead of +# being assumed. +# +# Monthly rather than per-PR: the property changes only when someone +# reconfigures the scale set, and the check costs a few dozen API calls. +# See docs/ci/runners.md and rustfs/backlog#1602. + +name: Runner Hygiene + +on: + schedule: + - cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron) + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: runner-hygiene + cancel-in-progress: false + +jobs: + check-ephemerality: + name: Check runner ephemerality + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + # Exit 2 (inconclusive / broken) is deliberately not a pass: a window + # where every sm-* job was still queued would otherwise look identical to + # a clean bill of health. + - name: Assert one job per self-hosted runner + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: ./scripts/ci/check_runner_ephemerality.sh 40 + + alert-on-failure: + name: Alert on scheduled failure + needs: [check-ephemerality] + # Same ci-8 mechanism as coverage.yml, audit.yml and the nightly lanes: + # scheduled runs file a tracking issue, manual dispatch stays quiet so + # debugging never produces a spurious alert. + if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/scripts/ci/check_runner_ephemerality.sh b/scripts/ci/check_runner_ephemerality.sh new file mode 100755 index 000000000..bd2021a19 --- /dev/null +++ b/scripts/ci/check_runner_ephemerality.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Assert that self-hosted runners are ephemeral: one job per runner, ever. +# +# WHY THIS IS CHECKED CONTINUOUSLY RATHER THAN ONCE +# +# rustfs/rustfs is public, and pull_request jobs run on self-hosted runners — +# they execute the PR's own build.rs, proc-macros and tests, which is arbitrary +# code. What keeps that from reaching the release build is that each runner is a +# fresh ARC pod that handles exactly one job and is destroyed. Nothing in this +# repository enforces it: it is a property of the ARC scale-set configuration, +# which lives outside the repo and can be changed without any PR. So it is +# asserted from the outside, on real data. +# +# A repeated runner_name means a runner served two jobs, i.e. state survived +# between them, i.e. a poisoned PR job could leave something behind for whatever +# runs next — including, while the release build shares the sm-standard-2 pool, +# a release build. +# +# Usage: scripts/ci/check_runner_ephemerality.sh [run-count] +# run-count defaults to 40. MIN_SAMPLE (default 10) is the number of observed +# assignments below which the window is reported INCONCLUSIVE rather than OK. +# +# Exit codes: 0 ephemeral, 1 a runner served two jobs, 2 inconclusive or broken. +set -euo pipefail + +REPO="${GITHUB_REPOSITORY:-rustfs/rustfs}" +LIMIT="${1:-40}" +# Below this many observed assignments the window says nothing useful: with the +# pool saturated, most sm-* jobs sit queued with runner_name still null, and a +# handful of samples would pass by coincidence rather than by evidence. +MIN_SAMPLE="${MIN_SAMPLE:-10}" + +command -v gh >/dev/null || { echo "ERROR: gh CLI not found" >&2; exit 2; } + +runs="$(gh run list --repo "$REPO" --limit "$LIMIT" --json databaseId --jq '.[].databaseId')" +[ -n "$runs" ] || { echo "ERROR: no runs returned for $REPO" >&2; exit 2; } + +names="$( + for run in $runs; do + gh api "repos/${REPO}/actions/runs/${run}/jobs" --paginate \ + --jq '.jobs[] | select(.runner_name != null) | select(.runner_name | startswith("sm-")) | .runner_name' 2>/dev/null || true + done +)" + +total="$(printf '%s\n' "$names" | grep -c . || true)" +if [ "${total:-0}" -eq 0 ]; then + echo "ERROR: no self-hosted (sm-*) runner names found across $LIMIT runs." >&2 + echo " Either the label scheme changed or the API shape did — this check" >&2 + echo " must not silently pass by finding nothing." >&2 + exit 2 +fi + +if [ "$total" -lt "$MIN_SAMPLE" ]; then + echo "INCONCLUSIVE: only ${total} self-hosted job assignments across ${LIMIT} runs" >&2 + echo " (need ${MIN_SAMPLE}). Most sm-* jobs are probably still queued, so" >&2 + echo " runner_name is null. Re-run with a larger window when the pool drains." >&2 + exit 2 +fi + +dupes="$(printf '%s\n' "$names" | sort | uniq -d)" + +if [ -n "$dupes" ]; then + echo "ERROR: self-hosted runners served more than one job — not ephemeral:" >&2 + printf '%s\n' "$dupes" | while read -r name; do + count="$(printf '%s\n' "$names" | grep -c "^${name}$")" + echo " ${name}: ${count} jobs" >&2 + done + echo "" >&2 + echo "State can survive between jobs on those runners. Because this is a" >&2 + echo "public repository whose pull_request jobs run PR-authored code, and" >&2 + echo "because build.yml's release legs share the sm-standard-2 pool, a" >&2 + echo "poisoned PR job could reach a release build. Check the ARC scale-set" >&2 + echo "configuration (rustfs/backlog#1602)." >&2 + exit 1 +fi + +echo "OK: ${total} self-hosted job assignments across ${LIMIT} runs, all on distinct runners"