diff --git a/.github/workflows/architecture-migration-rules.yml b/.github/workflows/architecture-migration-rules.yml index a8e144119..9153b69e0 100644 --- a/.github/workflows/architecture-migration-rules.yml +++ b/.github/workflows/architecture-migration-rules.yml @@ -37,6 +37,7 @@ jobs: name: Cancel Closed PR Runs if: github.event_name == 'pull_request' && github.event.action == 'closed' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Explain cancellation run run: echo "PR closed; this run only cancels older runs in the same concurrency group." @@ -45,6 +46,7 @@ jobs: name: Architecture Migration Rules if: github.event_name != 'pull_request' || github.event.action != 'closed' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 408c8684f..bdb4f9425 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -39,7 +39,12 @@ on: - 'scripts/security/check_preview_release_workflow.sh' - 'scripts/security/check_workflow_pins.sh' schedule: - - cron: '0 3 * * 0' # Weekly on Sunday 03:00 UTC (staggered after the midnight ci/build crons) + # Daily, not weekly. This schedule exists to catch RustSec advisories + # published against an unchanged dependency tree; at weekly cadence a new + # advisory could sit unnoticed for seven days. The check list is unchanged — + # splitting it into a light daily advisories-only run and a weekly full run + # would create runs where sources/bans/licenses go unverified. + - cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons) workflow_dispatch: permissions: @@ -59,6 +64,7 @@ jobs: name: Cancel Closed PR Runs if: github.event_name == 'pull_request' && github.event.action == 'closed' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Explain cancellation run run: echo "PR closed; this run only cancels older runs in the same concurrency group." @@ -120,12 +126,16 @@ jobs: - name: Check setup cache-save-if is explicit run: ./scripts/security/check_cache_save_if.sh + - name: Check every job declares a timeout + run: ./scripts/security/check_job_timeouts.sh + - name: Check preview release workflow policy run: ./scripts/security/check_preview_release_workflow.sh dependency-review: name: Dependency Review runs-on: ubuntu-latest + timeout-minutes: 30 if: github.event_name == 'pull_request' && github.event.action != 'closed' permissions: contents: read @@ -145,3 +155,26 @@ jobs: # conscious re-review of the license/provenance claim (backlog#1181). allow-dependencies-licenses: pkg:cargo/rustfs-uring@0.1.0 comment-summary-in-pr: always + + alert-on-failure: + name: Alert on scheduled failure + # dependency-review is deliberately excluded: it only runs on pull_request, + # so it can never contribute a failure to a scheduled run. + needs: [cargo-deny, workflow-pin-report] + # A scheduled cargo-deny failure usually means the dependency tree just + # matched a newly published advisory — the single most important signal this + # workflow produces, and until now it was only visible to whoever happened to + # open the Actions tab. Same ci-8 mechanism coverage.yml and + # e2e-replication-nightly.yml already use. + if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 696532a90..63c555f63 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -83,6 +83,7 @@ jobs: build-check: name: Build Strategy Check runs-on: ubuntu-latest + timeout-minutes: 30 outputs: should_build: ${{ steps.check.outputs.should_build }} build_type: ${{ steps.check.outputs.build_type }} @@ -164,6 +165,7 @@ jobs: name: Prepare Platform Matrix needs: build-check runs-on: ubuntu-latest + timeout-minutes: 30 outputs: matrix: ${{ steps.select.outputs.matrix }} selected: ${{ steps.select.outputs.selected }} @@ -171,10 +173,14 @@ jobs: - name: Select target platforms id: select shell: bash + env: + # via env, not interpolation: a dispatch input is free-form text and + # would otherwise be pasted into the script for bash to evaluate. + RAW_PLATFORMS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.platforms || 'all' }} run: | set -euo pipefail - selected="${{ github.event_name == 'workflow_dispatch' && github.event.inputs.platforms || 'all' }}" + selected="$RAW_PLATFORMS" selected="$(echo "${selected}" | tr -d '[:space:]')" if [[ -z "${selected}" ]]; then selected="all" @@ -710,9 +716,14 @@ jobs: needs: [ build-check, build-rustfs ] if: always() && needs.build-check.outputs.should_build == 'true' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Build completion summary shell: bash + env: + # dispatch input via env: free-form text must not be pasted into the + # script for bash to evaluate. + INPUT_BUILD_DOCKER: ${{ github.event.inputs.build_docker }} run: | BUILD_TYPE="${{ needs.build-check.outputs.build_type }}" VERSION="${{ needs.build-check.outputs.version }}" @@ -754,7 +765,7 @@ jobs: echo "🐳 Docker Images:" if [[ "$BUILD_TYPE" == "preview" ]]; then echo "⏭️ Preview tags do not publish Docker images" - elif [[ "${{ github.event.inputs.build_docker }}" == "false" ]]; then + elif [[ "$INPUT_BUILD_DOCKER" == "false" ]]; then echo "⏭️ Docker image build was skipped (binary only build)" elif [[ "$BUILD_STATUS" == "success" ]]; then echo "🔄 Docker images will be built and pushed automatically via workflow_run event" @@ -768,6 +779,7 @@ jobs: needs: [ build-check, build-rustfs ] if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease') runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: write outputs: @@ -827,6 +839,7 @@ jobs: needs: [ build-check, build-rustfs, create-release ] if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease') runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: write actions: read @@ -918,6 +931,7 @@ jobs: needs: [ build-check, publish-release ] if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease') runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Update latest.json env: @@ -977,6 +991,7 @@ jobs: needs: [ build-check, create-release, upload-release-assets ] if: startsWith(github.ref, 'refs/tags/') && (needs.build-check.outputs.build_type == 'preview' || needs.build-check.outputs.build_type == 'release' || needs.build-check.outputs.build_type == 'prerelease') runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: write steps: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5567744d..e2a346cd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -81,6 +81,7 @@ jobs: name: Cancel Closed PR Runs if: github.event_name == 'pull_request' && github.event.action == 'closed' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Explain cancellation run run: echo "PR closed; this run only cancels older runs in the same concurrency group." @@ -89,6 +90,7 @@ jobs: name: Typos if: github.event_name != 'pull_request' || github.event.action != 'closed' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - name: Typos check with custom config file diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml index 622958a14..ef557656d 100644 --- a/.github/workflows/cla.yml +++ b/.github/workflows/cla.yml @@ -37,6 +37,7 @@ jobs: name: Cancel Closed PR Runs if: github.event_name == 'pull_request_target' && github.event.action == 'closed' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Explain cancellation run run: echo "PR closed; this run only cancels older runs in the same concurrency group." @@ -44,6 +45,7 @@ jobs: cla: if: ${{ (github.event_name != 'issue_comment' || github.event.issue.pull_request) && (github.event_name != 'pull_request_target' || github.event.action != 'closed') }} runs-on: ubuntu-latest + timeout-minutes: 30 steps: - name: Report CLA result for merge queue if: github.event_name == 'merge_group' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 75ef45a59..ebb955da8 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -85,6 +85,7 @@ jobs: github.event.workflow_run.head_branch != 'main' && !contains(github.event.workflow_run.head_branch, '-preview')) runs-on: ubuntu-latest + timeout-minutes: 30 outputs: should_build: ${{ steps.check.outputs.should_build }} should_push: ${{ steps.check.outputs.should_push }} @@ -102,6 +103,12 @@ jobs: - name: Check build conditions id: check + env: + # dispatch inputs via env, not `${{ }}` interpolation: they are + # free-form strings and would otherwise be evaluated by bash. + INPUT_VERSION: ${{ github.event.inputs.version }} + INPUT_PUSH_IMAGES: ${{ github.event.inputs.push_images }} + INPUT_FORCE_REBUILD: ${{ github.event.inputs.force_rebuild }} run: | should_build=false should_push=false @@ -202,9 +209,9 @@ jobs: elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then # Manual trigger - input_version="${{ github.event.inputs.version }}" + input_version="$INPUT_VERSION" version="${input_version}" - should_push="${{ github.event.inputs.push_images }}" + should_push="$INPUT_PUSH_IMAGES" should_build=true # Get short SHA @@ -212,7 +219,7 @@ jobs: echo "🎯 Manual Docker build triggered:" echo " 📋 Requested version: $input_version" - echo " 🔧 Force rebuild: ${{ github.event.inputs.force_rebuild }}" + echo " 🔧 Force rebuild: $INPUT_FORCE_REBUILD" echo " 🚀 Push images: $should_push" case "$input_version" in @@ -439,6 +446,7 @@ jobs: needs: [ build-check, build-docker ] if: needs.build-check.outputs.should_build == 'true' && needs.build-check.outputs.should_push == 'true' runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read security-events: write @@ -493,6 +501,7 @@ jobs: needs: [ build-check, build-docker ] if: always() && needs.build-check.outputs.should_build == 'true' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Docker build completion summary run: | diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 713bc1694..3fe5ab0de 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -59,6 +59,7 @@ jobs: name: Cancel Closed PR Runs if: github.event_name == 'pull_request' && github.event.action == 'closed' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Explain cancellation run run: echo "PR closed; this run only cancels older runs in the same concurrency group." diff --git a/.github/workflows/helm-package.yml b/.github/workflows/helm-package.yml index 7e313c9b6..63c279caf 100644 --- a/.github/workflows/helm-package.yml +++ b/.github/workflows/helm-package.yml @@ -32,6 +32,7 @@ permissions: jobs: build-helm-package: runs-on: ubuntu-latest + timeout-minutes: 30 if: | (github.event_name == 'workflow_dispatch' && !contains(github.event.inputs.version, '-preview')) || ( @@ -50,15 +51,23 @@ jobs: - name: Checkout helm chart repo uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + # Both inputs reach the shell through env rather than `${{ }}` + # interpolation. A git ref name may contain `$(...)` — anything without a + # space is a legal tag — and interpolation pastes it into the script + # verbatim, where bash would run it. Reading "$RAW_INPUT" instead makes it + # data. - name: Normalize release version id: version + env: + RAW_INPUT: ${{ github.event.inputs.version }} + RAW_BRANCH: ${{ github.event.workflow_run.head_branch }} run: | set -eux if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - RAW="${{ github.event.inputs.version }}" + RAW="$RAW_INPUT" else - RAW="${{ github.event.workflow_run.head_branch }}" + RAW="$RAW_BRANCH" fi case "$RAW" in @@ -73,10 +82,13 @@ jobs: ./scripts/helm_chart_version.sh "$RAW_TAG" - name: Replace chart version and app version + env: + CHART_VERSION: ${{ steps.version.outputs.chart_version }} + APP_VERSION: ${{ steps.version.outputs.app_version }} run: | set -eux - sed -i -E 's/^version:.*/version: "${{ steps.version.outputs.chart_version }}"/' helm/rustfs/Chart.yaml - sed -i -E 's/^appVersion:.*/appVersion: "${{ steps.version.outputs.app_version }}"/' helm/rustfs/Chart.yaml + sed -i -E "s/^version:.*/version: \"${CHART_VERSION}\"/" helm/rustfs/Chart.yaml + sed -i -E "s/^appVersion:.*/appVersion: \"${APP_VERSION}\"/" helm/rustfs/Chart.yaml - name: Set up Helm uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 @@ -101,6 +113,7 @@ jobs: publish-helm-package: runs-on: ubuntu-latest + timeout-minutes: 30 needs: [ build-helm-package ] if: needs.build-helm-package.result == 'success' @@ -123,11 +136,19 @@ jobs: - name: Generate index run: helm repo index . --url https://charts.rustfs.com + # app_version is derived from the triggering tag name, and this job holds + # the cross-repository push token with rustfs/helm already checked out — + # the worst place in the repo to paste an attacker-influenced string into + # a shell line. Passed through env so bash treats it as data. - name: Push helm package and index file + env: + GIT_USERNAME: ${{ secrets.USERNAME }} + GIT_EMAIL: ${{ secrets.EMAIL_ADDRESS }} + APP_VERSION: ${{ needs.build-helm-package.outputs.app_version }} run: | set -eux - git config --global user.name "${{ secrets.USERNAME }}" - git config --global user.email "${{ secrets.EMAIL_ADDRESS }}" + git config --global user.name "${GIT_USERNAME}" + git config --global user.email "${GIT_EMAIL}" git add . - git commit -m "Update rustfs helm package with ${{ needs.build-helm-package.outputs.app_version }}." || echo "No changes to commit" + git commit -m "Update rustfs helm package with ${APP_VERSION}." || echo "No changes to commit" git push origin main diff --git a/.github/workflows/issue-translator.yml b/.github/workflows/issue-translator.yml index 996ee5061..1bb98a956 100644 --- a/.github/workflows/issue-translator.yml +++ b/.github/workflows/issue-translator.yml @@ -26,6 +26,7 @@ permissions: jobs: build: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: usthe/issues-translate-action@b41f55ddc81d7d54bd542a4f289fe28ec081898e # v2.7 with: diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index dc20702eb..85829ef47 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -46,6 +46,7 @@ jobs: name: Cancel Closed PR Runs if: github.event_name == 'pull_request' && github.event.action == 'closed' runs-on: ubuntu-latest + timeout-minutes: 10 steps: - name: Explain cancellation run run: echo "PR closed; this run only cancels older runs in the same concurrency group." diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml index b9848e397..7a831303c 100644 --- a/.github/workflows/performance-ab.yml +++ b/.github/workflows/performance-ab.yml @@ -162,13 +162,15 @@ jobs: - name: Decide exemption id: exempt + env: + INPUT_ALLOW_REGRESSION: ${{ github.event.inputs.allow_regression }} run: | allow="false" if [[ "${{ github.event_name }}" == "pull_request" ]] \ && ${{ contains(github.event.pull_request.labels.*.name, 'perf-deliberate-tradeoff') }}; then allow="true" fi - if [[ "${{ github.event.inputs.allow_regression }}" == "true" ]]; then + if [[ "$INPUT_ALLOW_REGRESSION" == "true" ]]; then allow="true" fi echo "allow_regression=$allow" >> "$GITHUB_OUTPUT" @@ -224,6 +226,8 @@ jobs: - name: Run warp A/B and gate id: ab + env: + INPUT_DURATION: ${{ github.event.inputs.duration }} run: | set -euo pipefail # Budget note: with perf-3's cached baseline the nightly does no source @@ -234,7 +238,7 @@ jobs: # budget, which the rig's previous 60s health poll undershot (the first # two nightly failures). perf-6 recalibrates these once the noise study # lands. - duration="${{ github.event.inputs.duration || '12s' }}" + duration="${INPUT_DURATION:-12s}" baseline_sha="${{ steps.commits.outputs.baseline_sha }}" candidate_sha="${{ steps.commits.outputs.candidate_sha }}" baseline_hit="${{ steps.baseline_cache.outputs.cache-hit }}" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 30520387b..3152e720d 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -20,6 +20,7 @@ on: jobs: stale: runs-on: ubuntu-latest + timeout-minutes: 30 steps: - uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9 with: diff --git a/.github/workflows/star-history.yml b/.github/workflows/star-history.yml index f7457e6de..82fdb303b 100644 --- a/.github/workflows/star-history.yml +++ b/.github/workflows/star-history.yml @@ -15,6 +15,7 @@ concurrency: jobs: update: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: overtrue/repo-visuals-action@72f34d24769ff5d341956da2f23952594ef2f1e2 # v1.3.0 with: diff --git a/scripts/security/check_job_timeouts.sh b/scripts/security/check_job_timeouts.sh new file mode 100755 index 000000000..f9c6fdf09 --- /dev/null +++ b/scripts/security/check_job_timeouts.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Every job that occupies a runner must declare timeout-minutes. +# +# GitHub's default is 360 minutes. This repository has a history of runners +# stalling intermittently (#5394) and of a plain `git checkout` taking 9m57s +# under node-level I/O contention, so an undeclared timeout means one wedged job +# can hold a runner for six hours out of a pool of roughly 15-21. +# +# Only jobs with `runs-on` are checked: a job that calls a reusable workflow has +# no runner of its own and cannot declare a timeout. +# +# Usage: scripts/security/check_job_timeouts.sh +set -euo pipefail + +cd "$(dirname "$0")/../.." + +status=0 + +for file in .github/workflows/*.yml .github/workflows/*.yaml; do + [ -e "$file" ] || continue + + awk -v file="$file" ' + function flush() { + if (job != "" && has_runs_on && !has_timeout) { + printf "%s:%d: job `%s` has runs-on but no timeout-minutes\n", file, job_line, job > "/dev/stderr" + bad++ + } + job = ""; has_runs_on = 0; has_timeout = 0 + } + /^jobs:[[:space:]]*$/ { in_jobs = 1; next } + { + line = $0 + sub(/[[:space:]]+$/, "", line) + if (line ~ /^[[:space:]]*#/ || line ~ /^[[:space:]]*$/) next + + # A non-indented key ends the jobs mapping. + if (in_jobs && line !~ /^[[:space:]]/) { flush(); in_jobs = 0 } + if (!in_jobs) next + + if (line ~ /^ [a-zA-Z0-9_-]+:[[:space:]]*$/) { + flush() + job = line + sub(/^[[:space:]]*/, "", job) + sub(/:.*$/, "", job) + job_line = NR + next + } + if (job == "") next + if (line ~ /^ runs-on:/) has_runs_on = 1 + if (line ~ /^ timeout-minutes:/) has_timeout = 1 + } + END { flush(); exit (bad > 0) } + ' "$file" || status=1 +done + +if [ "$status" -ne 0 ]; then + echo "" >&2 + echo "Add timeout-minutes to each job above. Rough budgets used in this repo:" >&2 + echo " 10 echo-only and guard-script jobs" >&2 + echo " 30 jobs that call the GitHub API, upload assets, or push over the network" >&2 + echo " 90+ anything using ./.github/actions/setup (cold cache restore alone is 11-21 min)" >&2 + exit 1 +fi + +echo "OK: every job with runs-on declares timeout-minutes"