gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-23 04:39:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

ci: detect incomplete and stale scheduled validations (#6357)

Browse Source
This commit is contained in:
Zhengchao An
2026-08-23 01:40:28 +08:00
committed by GitHub
parent 87235ffd28
commit ddc4120c82
20 changed files with 1123 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/actions/schedule-failure-issue/action.yml
+53 -12
View File
@@ -14,9 +14,10 @@
name: "Schedule Failure Issue"
description: >-
Open (or update) a tracking issue when a scheduled workflow run fails.
Open (or update) a tracking issue when a scheduled workflow run fails or
does not complete normally.
Dedupes by workflow name: if an open issue titled
"[scheduled-failure] <workflow name>" already exists, the failure is
"[scheduled-failure] <workflow name>" already exists, the result is
appended as a comment; otherwise a new issue is created. This is the
single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).
@@ -38,6 +39,30 @@ inputs:
Set to an empty string to skip labeling.
required: false
default: "infrastructure"
source-run-id:
description: "Run ID to report. Defaults to the current workflow run."
required: false
default: ${{ github.run_id }}
source-run-attempt:
description: "Run attempt to report. Defaults to the current attempt."
required: false
default: ${{ github.run_attempt }}
source-event:
description: "Trigger event of the run being reported."
required: false
default: ${{ github.event_name }}
source-ref-name:
description: "Ref name of the run being reported."
required: false
default: ${{ github.ref_name }}
source-sha:
description: "Commit SHA of the run being reported."
required: false
default: ${{ github.sha }}
details-file:
description: "Optional Markdown file appended to the issue body."
required: false
default: ""
runs:
using: "composite"
@@ -48,17 +73,22 @@ runs:
GH_TOKEN: ${{ inputs.github-token }}
WORKFLOW_NAME: ${{ inputs.workflow-name }}
ISSUE_LABEL: ${{ inputs.label }}
SOURCE_RUN_ID: ${{ inputs.source-run-id }}
SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
SOURCE_EVENT: ${{ inputs.source-event }}
SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
SOURCE_SHA: ${{ inputs.source-sha }}
DETAILS_FILE: ${{ inputs.details-file }}
run: |
set -euo pipefail
title="[scheduled-failure] ${WORKFLOW_NAME}"
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"
# Failed job names for this run attempt. The alert job runs while the
# run as a whole is still in progress, so inspect the jobs that have
# already completed with a non-success conclusion.
# Inspect the reported run attempt. It can be the current in-workflow
# failure or a completed run observed by the external watchdog.
failed_jobs="$(gh api \
"repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
"repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
--paginate \
--jq '.jobs[]
| select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
@@ -67,15 +97,26 @@ runs:
failed_jobs="- (failed job not recorded yet — see the run page)"
fi
details=""
if [ -n "${DETAILS_FILE}" ]; then
if [ -f "${DETAILS_FILE}" ]; then
details="$(cat "${DETAILS_FILE}")"
else
details="Details file was not available: \`${DETAILS_FILE}\`"
fi
fi
body="$(cat <<EOF
Scheduled run of **${WORKFLOW_NAME}** failed.
Run of **${WORKFLOW_NAME}** did not complete successfully.
- Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
- Event: \`${GITHUB_EVENT_NAME}\`
- Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
- Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
- Event: \`${SOURCE_EVENT}\`
- Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`
Failed jobs:
Non-success jobs:
${failed_jobs}
${details}
EOF
)"
.github/scheduled-validations.json
+14
View File
@@ -0,0 +1,14 @@
[
{ "workflow": ".github/workflows/audit.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/build.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
{ "workflow": ".github/workflows/minio-interop.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
{ "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792 }
]
.github/workflows/audit.yml
+1 -1
View File
@@ -46,7 +46,7 @@ on:
# advisory could sit unnoticed for seven days. The check list is unchanged —
# splitting it into a light daily advisories-only run and a weekly full run
# would create runs where sources/bans/licenses go unverified.
- cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
- cron: '23 3 * * *' # Daily 03:23 UTC
workflow_dispatch:
permissions:
.github/workflows/build.yml
+21 -1
View File
@@ -52,7 +52,7 @@ on:
- ".dockerignore"
- "flake.lock"
schedule:
- cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
- cron: "13 1 * * 0" # Weekly on Sunday 01:13 UTC
workflow_dispatch:
inputs:
build_docker:
@@ -1032,3 +1032,23 @@ jobs:
echo "🎉 Released $TAG successfully!"
echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
alert-on-failure:
name: Alert on scheduled failure
needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
if: >-
always() && github.event_name == 'schedule' &&
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
.github/workflows/ci-docs-only.yml
+4 -1
View File
@@ -126,7 +126,10 @@ jobs:
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
run: python3 ./scripts/check_test_wiring.py
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
.github/workflows/ci.yml
+39 -2
View File
@@ -59,7 +59,7 @@ on:
merge_group:
types: [ checks_requested ]
schedule:
- cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
- cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC
workflow_dispatch:
permissions:
@@ -161,7 +161,10 @@ jobs:
run: ./scripts/check_embedded_secrets.sh
- name: Check test wiring
run: python3 ./scripts/check_test_wiring.py
run: |
python3 ./scripts/check_test_wiring.py --self-test
python3 ./scripts/check_scheduled_validation_freshness.py --self-test
python3 ./scripts/check_test_wiring.py
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
@@ -1032,3 +1035,37 @@ jobs:
path: artifacts/s3tests-single/**
if-no-files-found: ignore
retention-days: 3
alert-on-failure:
name: Alert on scheduled failure
needs:
- typos
- quick-checks
- test-and-lint
- test-ilm-integration-serial
- test-and-lint-rio-v2
- test-and-lint-protocols
- build-rustfs-debug-binary
- build-rustfs-debug-binary-rio-v2
- uring-integration
- e2e-tests
- e2e-full
- e2e-tests-rio-v2
- s3-implemented-tests
- s3-lifecycle-behavior-tests
if: >-
always() && github.event_name == 'schedule' &&
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
.github/workflows/coverage.yml
+1 -1
View File
@@ -37,7 +37,7 @@ on:
# build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
# (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
# e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
- cron: "0 7 * * 0"
- cron: "43 7 * * 0"
# Only alert-on-failure needs more than read access; it declares its own
# job-level `issues: write`.
.github/workflows/e2e-replication-nightly.yml
+1 -1
View File
@@ -40,7 +40,7 @@ on:
schedule:
# 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
# stale (01:30) and performance-ab (06:00).
- cron: "0 4 * * *"
- cron: "29 4 * * *"
# Only alert-on-failure needs more than read access; it declares its own
# job-level `issues: write`.
.github/workflows/e2e-s3tests.yml
+1 -1
View File
@@ -93,7 +93,7 @@ on:
schedule:
# Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
# single-node and the 4-node distributed topologies (matrix below).
- cron: "0 2 * * 0"
- cron: "19 2 * * 0"
env:
# main user
.github/workflows/fuzz.yml
+1 -1
View File
@@ -30,7 +30,7 @@ on:
- "Cargo.lock"
- ".github/workflows/fuzz.yml"
schedule:
- cron: "0 2 * * *"
- cron: "17 2 * * *"
workflow_dispatch:
inputs:
profile:
.github/workflows/minio-interop.yml
+18
View File
@@ -121,3 +121,21 @@ jobs:
cargo nextest run --run-ignored ignored-only --no-tests=fail \
-p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
-E "$INTEROP_FILTER"
alert-on-failure:
name: Alert on scheduled failure
needs: [minio-interop]
if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
.github/workflows/mint.yml
+1 -1
View File
@@ -76,7 +76,7 @@ on:
schedule:
# Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to
# 3h) has finished, so the two never contend for the same runner pool.
- cron: "0 6 * * 0"
- cron: "41 6 * * 0"
env:
S3_ACCESS_KEY: rustfsadmin-ci
.github/workflows/nightly-gnu.yml
+21 -1
View File
@@ -16,7 +16,7 @@ name: Nightly GNU Build
on:
schedule:
- cron: "0 0 * * *"
- cron: "7 0 * * *"
timezone: "Asia/Shanghai"
workflow_dispatch:
@@ -194,3 +194,23 @@ jobs:
- name: Run HA leader failover live checks (three-node Raft cluster in Docker)
run: bash scripts/test/vault_ha_kms_live.sh
alert-on-failure:
name: Alert on scheduled failure
needs: [build, kms-vault-lane, kms-vault-ha-failover]
if: >-
always() && github.event_name == 'schedule' &&
(contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update failure-tracking issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
.github/workflows/performance-ab.yml
+1 -1
View File
@@ -33,7 +33,7 @@ name: Performance A/B
on:
schedule:
- cron: "0 6 * * *" # 06:00 UTC nightly, against main
- cron: "31 6 * * *" # 06:31 UTC nightly, against main
workflow_dispatch:
inputs:
duration:
.github/workflows/runner-hygiene.yml
+1 -1
View File
@@ -30,7 +30,7 @@ name: Runner Hygiene
on:
schedule:
- cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
- cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC
workflow_dispatch:
permissions:
.github/workflows/scheduled-validation-freshness.yml
+57
View File
@@ -0,0 +1,57 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Scheduled Validation Freshness
on:
schedule:
- cron: "47 23 * * *"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: scheduled-validation-freshness
cancel-in-progress: false
jobs:
check-freshness:
name: Check scheduled validation freshness
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
actions: read
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Check latest scheduled runs
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
set +e
python3 scripts/check_scheduled_validation_freshness.py \
--report "${RUNNER_TEMP}/scheduled-validation-freshness.md"
status=$?
cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}"
exit "${status}"
- name: Open or update freshness issue
if: failure()
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
details-file: ${{ runner.temp }}/scheduled-validation-freshness.md
.github/workflows/scheduled-validation-watchdog.yml
+63
View File
@@ -0,0 +1,63 @@
# Copyright 2024 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Scheduled Validation Watchdog
on:
workflow_run:
workflows:
- "Security Audit"
- "Build and Release"
- "Continuous Integration"
- "coverage"
- "e2e-nightly"
- "e2e-s3tests"
- "Fuzz"
- "mint"
- "minio-interop"
- "Nightly GNU Build"
- "Performance A/B"
- "Runner Hygiene"
types: [completed]
permissions:
contents: read
jobs:
alert-on-incomplete-run:
name: Alert on incomplete scheduled run
if: >-
github.event.workflow_run.event == 'schedule' &&
github.event.workflow_run.conclusion != 'success' &&
github.event.workflow_run.conclusion != 'failure'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
actions: read
contents: read
issues: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
with:
persist-credentials: false
- name: Open or update incomplete-run issue
uses: ./.github/actions/schedule-failure-issue
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
workflow-name: ${{ github.event.workflow_run.name }}
source-run-id: ${{ github.event.workflow_run.id }}
source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
source-event: ${{ github.event.workflow_run.event }}
source-ref-name: ${{ github.event.workflow_run.head_branch }}
source-sha: ${{ github.event.workflow_run.head_sha }}