mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
Zhengchao An
4310b55238
Scheduled workflow failures previously went unnoticed (15 consecutive red weekly s3-tests sweeps, silent perf nightly failures). Add a reusable composite action that opens a tracking issue titled "[scheduled-failure] <workflow name>" — or appends a comment to the existing open one (dedupe key = workflow name) — with the run URL, run attempt, and failed job names, labeled "infrastructure". Wire it into the scheduled paths of e2e-s3tests, mint, fuzz (nightly) and performance-ab via a final alert-on-failure job gated by "always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')", with issues:write scoped to that job only. e2e-s3tests and mint previously had no permissions key; they now get workflow-level contents:read, reducing every other job's token. Add a dispatch-only drill workflow that forces a job failure and runs the action through the exact consumer wiring, for end-to-end verification. The ci-7 e2e nightly does not exist yet; it is recorded as a pending consumer in the action README. Refs: rustfs/backlog#1149 (ci-8), rustfs/backlog#1155
254 lines
11 KiB
YAML
254 lines
11 KiB
YAML
# Copyright 2024 RustFS Team
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
name: Fuzz
|
|
|
|
on:
|
|
pull_request:
|
|
types: [ opened, synchronize, reopened, closed ]
|
|
# PR trigger is intentionally narrow: only changes to the fuzz harness
|
|
# itself gate a PR. Broad crate paths (ecstore/filemeta/utils/policy/…)
|
|
# are covered by the nightly `schedule` run below, which fuzzes against
|
|
# whatever landed on main. Widening these paths previously queued a
|
|
# ~45min fuzz-build on nearly every PR and is why this workflow was
|
|
# disabled; do not re-add crate paths here.
|
|
paths:
|
|
- "fuzz/**"
|
|
- "scripts/fuzz/**"
|
|
- ".github/workflows/fuzz.yml"
|
|
schedule:
|
|
- cron: "0 2 * * *"
|
|
workflow_dispatch:
|
|
inputs:
|
|
profile:
|
|
description: "Which fuzz profile to run"
|
|
required: false
|
|
default: "both"
|
|
type: choice
|
|
options:
|
|
- smoke
|
|
- nightly
|
|
- both
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu
|
|
# Fuzz targets build without the dial9 feature, so tokio_unstable is not needed.
|
|
RUSTFLAGS: "-C target-feature=-crt-static"
|
|
|
|
jobs:
|
|
cancel-closed-pr-runs:
|
|
name: Cancel Closed PR Runs
|
|
if: github.event_name == 'pull_request' && github.event.action == 'closed'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Explain cancellation run
|
|
run: echo "PR closed; this run only cancels older runs in the same concurrency group."
|
|
|
|
# ──────────────────────────────────────────────────────────────
|
|
# Phase 1: Build all fuzz harness binaries once.
|
|
# ──────────────────────────────────────────────────────────────
|
|
fuzz-build:
|
|
name: Build Fuzz Harness
|
|
if: >
|
|
(github.event_name == 'pull_request' && github.event.action != 'closed') ||
|
|
github.event_name == 'schedule' ||
|
|
github.event_name == 'workflow_dispatch'
|
|
runs-on: sm-standard-4
|
|
timeout-minutes: 45
|
|
env:
|
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Setup Rust environment
|
|
uses: ./.github/actions/setup
|
|
with:
|
|
rust-version: nightly
|
|
cache-shared-key: fuzz-${{ hashFiles('fuzz/Cargo.lock') }}
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
cache-save-if: ${{ github.ref == 'refs/heads/main' || github.event_name == 'schedule' }}
|
|
|
|
- name: Install cargo-fuzz
|
|
uses: taiki-e/install-action@bffeee26d4db9be238a4ea78d8826604ebcb594d # v2
|
|
with:
|
|
tool: cargo-fuzz
|
|
|
|
- name: Build all fuzz targets
|
|
run: BUILD_ONLY=1 ./scripts/fuzz/run.sh
|
|
|
|
- name: Stage prebuilt fuzz binaries
|
|
run: |
|
|
binary_root="fuzz/prebuilt/${CARGO_BUILD_TARGET}/release"
|
|
mkdir -p "${binary_root}"
|
|
# Keep this list in sync with the smoke/nightly matrices and
|
|
# scripts/fuzz/run.sh default target set (all 5 buildable bins in
|
|
# fuzz/Cargo.toml). The *_storage_api.rs files are `mod` submodules
|
|
# of their parent targets, not standalone bins.
|
|
for target in archive_extract bucket_validation local_metadata path_containment policy_ingress; do
|
|
install -Dm755 "fuzz/target/${CARGO_BUILD_TARGET}/release/${target}" "${binary_root}/${target}"
|
|
done
|
|
|
|
- name: Upload prebuilt fuzz binaries
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: fuzz-prebuilt-binaries-${{ github.run_number }}
|
|
path: |
|
|
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/archive_extract
|
|
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/bucket_validation
|
|
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/local_metadata
|
|
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/path_containment
|
|
fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release/policy_ingress
|
|
if-no-files-found: error
|
|
retention-days: 1
|
|
compression-level: 0
|
|
|
|
# ──────────────────────────────────────────────────────────────
|
|
# Phase 2 (PR): Run smoke targets in parallel via matrix.
|
|
# ──────────────────────────────────────────────────────────────
|
|
pr-fuzz-smoke:
|
|
name: "Smoke / ${{ matrix.target }}"
|
|
needs: fuzz-build
|
|
if: >
|
|
(github.event_name == 'pull_request' && github.event.action != 'closed') ||
|
|
(github.event_name == 'workflow_dispatch' &&
|
|
(github.event.inputs.profile == 'smoke' || github.event.inputs.profile == 'both'))
|
|
runs-on: sm-standard-4
|
|
timeout-minutes: 30
|
|
strategy:
|
|
fail-fast: false
|
|
# One job per target runs in parallel, so wall-clock time is per-target
|
|
# (~60s fuzz + download/chmod overhead), not the sum across the matrix.
|
|
matrix:
|
|
target: [archive_extract, bucket_validation, local_metadata, path_containment, policy_ingress]
|
|
env:
|
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Download prebuilt fuzz binaries
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
|
with:
|
|
name: fuzz-prebuilt-binaries-${{ github.run_number }}
|
|
path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release
|
|
|
|
- name: Restore executable bit
|
|
run: chmod +x "fuzz/prebuilt/${CARGO_BUILD_TARGET}/release/${{ matrix.target }}"
|
|
|
|
- name: Run ${{ matrix.target }}
|
|
env:
|
|
FUZZ_TARGET: ${{ matrix.target }}
|
|
MAX_TOTAL_TIME: 60
|
|
SKIP_BUILD: 1
|
|
USE_PREBUILT_BINARY: 1
|
|
PREBUILT_BINARY_DIR: ${{ github.workspace }}/fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release
|
|
run: ./scripts/fuzz/run.sh
|
|
|
|
- name: Upload fuzz smoke artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: fuzz-smoke-${{ matrix.target }}-${{ github.run_number }}
|
|
path: |
|
|
fuzz/artifacts/**
|
|
fuzz/corpus/${{ matrix.target }}/**
|
|
if-no-files-found: ignore
|
|
retention-days: 7
|
|
|
|
# ──────────────────────────────────────────────────────────────
|
|
# Phase 2 (Nightly): Run all targets in parallel via matrix.
|
|
# ──────────────────────────────────────────────────────────────
|
|
nightly-fuzz-corpus:
|
|
name: "Nightly / ${{ matrix.target }}"
|
|
needs: fuzz-build
|
|
# TODO(ci-8): when the schedule-failure-issue composite action lands,
|
|
# add a step here (or a dependent job) that opens/updates a GitHub issue
|
|
# on nightly failure. ci-8 is the single alerting mechanism for all
|
|
# scheduled workflows; do not self-roll alerting in this workflow.
|
|
if: >
|
|
github.event_name == 'schedule' ||
|
|
(github.event_name == 'workflow_dispatch' &&
|
|
(github.event.inputs.profile == 'nightly' || github.event.inputs.profile == 'both'))
|
|
runs-on: sm-standard-4
|
|
timeout-minutes: 60
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
target: [archive_extract, bucket_validation, local_metadata, path_containment, policy_ingress]
|
|
env:
|
|
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
|
|
- name: Download prebuilt fuzz binaries
|
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
|
with:
|
|
name: fuzz-prebuilt-binaries-${{ github.run_number }}
|
|
path: fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release
|
|
|
|
- name: Restore executable bit
|
|
run: chmod +x "fuzz/prebuilt/${CARGO_BUILD_TARGET}/release/${{ matrix.target }}"
|
|
|
|
- name: Run ${{ matrix.target }}
|
|
env:
|
|
FUZZ_TARGET: ${{ matrix.target }}
|
|
MAX_TOTAL_TIME: 300
|
|
SKIP_BUILD: 1
|
|
USE_PREBUILT_BINARY: 1
|
|
PREBUILT_BINARY_DIR: ${{ github.workspace }}/fuzz/prebuilt/${{ env.CARGO_BUILD_TARGET }}/release
|
|
run: ./scripts/fuzz/run.sh
|
|
|
|
- name: Upload nightly fuzz artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
|
with:
|
|
name: fuzz-nightly-${{ matrix.target }}-${{ github.run_number }}
|
|
path: |
|
|
fuzz/artifacts/**
|
|
fuzz/corpus/${{ matrix.target }}/**
|
|
if-no-files-found: ignore
|
|
retention-days: 30
|
|
|
|
# ──────────────────────────────────────────────────────────────
|
|
# Nightly alerting: open/update a tracking issue on failure.
|
|
# ──────────────────────────────────────────────────────────────
|
|
alert-on-failure:
|
|
name: Alert on scheduled failure
|
|
needs: [fuzz-build, nightly-fuzz-corpus]
|
|
# `always()` is required: without it this job is skipped when a needed
|
|
# job fails. Alerts only for scheduled (nightly) runs (backlog#1149
|
|
# ci-8); PR and manual dispatch failures are already watched by a human.
|
|
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
issues: write
|
|
steps:
|
|
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
|
- name: Open or update failure-tracking issue
|
|
uses: ./.github/actions/schedule-failure-issue
|
|
with:
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|