From 8c0eaf225ddc8dc82dc815523288cda8b990d905 Mon Sep 17 00:00:00 2001 From: overtrue Date: Sat, 22 Aug 2026 08:10:16 +0800 Subject: [PATCH] ci: detect incomplete scheduled validation runs --- .../actions/schedule-failure-issue/action.yml | 49 ++++++-- .github/workflows/build.yml | 20 ++++ .github/workflows/ci.yml | 34 ++++++ .github/workflows/nightly-gnu.yml | 20 ++++ .../scheduled-validation-watchdog.yml | 62 ++++++++++ scripts/check_test_wiring.py | 113 +++++++++++++++++- 6 files changed, 285 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/scheduled-validation-watchdog.yml diff --git a/.github/actions/schedule-failure-issue/action.yml b/.github/actions/schedule-failure-issue/action.yml index 60e938690..34f1839be 100644 --- a/.github/actions/schedule-failure-issue/action.yml +++ b/.github/actions/schedule-failure-issue/action.yml @@ -14,9 +14,10 @@ name: "Schedule Failure Issue" description: >- - Open (or update) a tracking issue when a scheduled workflow run fails. + Open (or update) a tracking issue when a scheduled workflow run fails or + does not complete normally. Dedupes by workflow name: if an open issue titled - "[scheduled-failure] " already exists, the failure is + "[scheduled-failure] " already exists, the result is appended as a comment; otherwise a new issue is created. This is the single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8). @@ -38,6 +39,26 @@ inputs: Set to an empty string to skip labeling. required: false default: "infrastructure" + source-run-id: + description: "Run ID to report. Defaults to the current workflow run." + required: false + default: ${{ github.run_id }} + source-run-attempt: + description: "Run attempt to report. Defaults to the current attempt." + required: false + default: ${{ github.run_attempt }} + source-event: + description: "Trigger event of the run being reported." + required: false + default: ${{ github.event_name }} + source-ref-name: + description: "Ref name of the run being reported." + required: false + default: ${{ github.ref_name }} + source-sha: + description: "Commit SHA of the run being reported." + required: false + default: ${{ github.sha }} runs: using: "composite" @@ -48,17 +69,21 @@ runs: GH_TOKEN: ${{ inputs.github-token }} WORKFLOW_NAME: ${{ inputs.workflow-name }} ISSUE_LABEL: ${{ inputs.label }} + SOURCE_RUN_ID: ${{ inputs.source-run-id }} + SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }} + SOURCE_EVENT: ${{ inputs.source-event }} + SOURCE_REF_NAME: ${{ inputs.source-ref-name }} + SOURCE_SHA: ${{ inputs.source-sha }} run: | set -euo pipefail title="[scheduled-failure] ${WORKFLOW_NAME}" - run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}" - # Failed job names for this run attempt. The alert job runs while the - # run as a whole is still in progress, so inspect the jobs that have - # already completed with a non-success conclusion. + # Inspect the reported run attempt. It can be the current in-workflow + # failure or a completed run observed by the external watchdog. failed_jobs="$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \ --paginate \ --jq '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled") @@ -68,13 +93,13 @@ runs: fi body="$(cat <- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a06e4667..f239a39d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1032,3 +1032,37 @@ jobs: path: artifacts/s3tests-single/** if-no-files-found: ignore retention-days: 3 + + alert-on-failure: + name: Alert on scheduled failure + needs: + - typos + - quick-checks + - test-and-lint + - test-ilm-integration-serial + - test-and-lint-rio-v2 + - test-and-lint-protocols + - build-rustfs-debug-binary + - build-rustfs-debug-binary-rio-v2 + - uring-integration + - e2e-tests + - e2e-full + - e2e-tests-rio-v2 + - s3-implemented-tests + - s3-lifecycle-behavior-tests + if: >- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/nightly-gnu.yml b/.github/workflows/nightly-gnu.yml index 1f7c2d488..c78092ef0 100644 --- a/.github/workflows/nightly-gnu.yml +++ b/.github/workflows/nightly-gnu.yml @@ -194,3 +194,23 @@ jobs: - name: Run HA leader failover live checks (three-node Raft cluster in Docker) run: bash scripts/test/vault_ha_kms_live.sh + + alert-on-failure: + name: Alert on scheduled failure + needs: [build, kms-vault-lane, kms-vault-ha-failover] + if: >- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/scheduled-validation-watchdog.yml b/.github/workflows/scheduled-validation-watchdog.yml new file mode 100644 index 000000000..95f486d20 --- /dev/null +++ b/.github/workflows/scheduled-validation-watchdog.yml @@ -0,0 +1,62 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Scheduled Validation Watchdog + +on: + workflow_run: + workflows: + - "Security Audit" + - "Build and Release" + - "Continuous Integration" + - "coverage" + - "e2e-nightly" + - "e2e-s3tests" + - "Fuzz" + - "mint" + - "Nightly GNU Build" + - "Performance A/B" + - "Runner Hygiene" + types: [completed] + +permissions: + contents: read + +jobs: + alert-on-incomplete-run: + name: Alert on incomplete scheduled run + if: >- + github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.conclusion != 'success' && + github.event.workflow_run.conclusion != 'failure' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update incomplete-run issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + workflow-name: ${{ github.event.workflow_run.name }} + source-run-id: ${{ github.event.workflow_run.id }} + source-run-attempt: ${{ github.event.workflow_run.run_attempt }} + source-event: ${{ github.event.workflow_run.event }} + source-ref-name: ${{ github.event.workflow_run.head_branch }} + source-sha: ${{ github.event.workflow_run.head_sha }} diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 4edd41f0d..2a18a7f0d 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -15,6 +15,19 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] +SCHEDULED_ALERT_WORKFLOWS = ( + ".github/workflows/audit.yml", + ".github/workflows/build.yml", + ".github/workflows/ci.yml", + ".github/workflows/coverage.yml", + ".github/workflows/e2e-replication-nightly.yml", + ".github/workflows/e2e-s3tests.yml", + ".github/workflows/fuzz.yml", + ".github/workflows/mint.yml", + ".github/workflows/nightly-gnu.yml", + ".github/workflows/performance-ab.yml", + ".github/workflows/runner-hygiene.yml", +) def words(value: str) -> set[str]: @@ -252,6 +265,69 @@ def check_profile_definitions(root: Path) -> list[str]: return errors +def check_scheduled_alerts(root: Path) -> list[str]: + errors: list[str] = [] + for relative in SCHEDULED_ALERT_WORKFLOWS: + path = root / relative + try: + lines = path.read_text().splitlines() + except FileNotFoundError: + errors.append(f"{relative}: missing scheduled validation workflow") + continue + + try: + start = lines.index(" alert-on-failure:") + 1 + except ValueError: + errors.append(f"{relative}: missing alert-on-failure job") + continue + end = next( + (index for index in range(start, len(lines)) if re.fullmatch(r" [A-Za-z0-9_-]+:", lines[index])), + len(lines), + ) + job = "\n".join(line.split("#", 1)[0] for line in lines[start:end]) + required = ( + "always()", + "github.event_name == 'schedule'", + "contains(needs.*.result, 'failure')", + ) + missing = [token for token in required if token not in job] + if missing: + errors.append(f"{relative}: alert-on-failure missing {', '.join(missing)}") + + watchdog_path = root / ".github/workflows/scheduled-validation-watchdog.yml" + try: + watchdog = "\n".join( + line.split("#", 1)[0] for line in watchdog_path.read_text().splitlines() + ) + except FileNotFoundError: + errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing completion watchdog") + return errors + for relative in SCHEDULED_ALERT_WORKFLOWS: + path = root / relative + if not path.is_file(): + continue + source = path.read_text() + match = re.search(r"^name:\s*[\"']?([^\"'\n]+)", source, re.MULTILINE) + if not match: + errors.append(f"{relative}: missing workflow name") + elif f'- "{match.group(1).strip()}"' not in watchdog: + errors.append(f"{relative}: missing from scheduled completion watchdog") + required = ( + "github.event.workflow_run.event == 'schedule'", + "github.event.workflow_run.conclusion != 'success'", + "github.event.workflow_run.conclusion != 'failure'", + "workflow-name: ${{ github.event.workflow_run.name }}", + "source-run-id: ${{ github.event.workflow_run.id }}", + "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}", + ) + missing = [token for token in required if token not in watchdog] + if missing: + errors.append( + ".github/workflows/scheduled-validation-watchdog.yml: missing " + ", ".join(missing) + ) + return errors + + def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]: try: expected_digest = profile_selection(root, profile) @@ -281,6 +357,7 @@ def validate(root: Path) -> list[str]: errors.extend(check_runner_selection(root)) errors.extend(check_s3_tests_runner(root)) errors.extend(check_profile_definitions(root)) + errors.extend(check_scheduled_alerts(root)) return errors @@ -413,6 +490,40 @@ class SelfTests(unittest.TestCase): with mock.patch.object(sys, "platform", "linux"): self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1) + def test_scheduled_alerts_require_completion_watchdog(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + alert = ( + " alert-on-failure:\n" + " if: always() && github.event_name == 'schedule' && " + "contains(needs.*.result, 'failure')\n" + ) + names: list[str] = [] + for relative in SCHEDULED_ALERT_WORKFLOWS: + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + names.append(path.stem) + path.write_text(f'name: "{path.stem}"\n{alert}') + watchdog = root / ".github/workflows/scheduled-validation-watchdog.yml" + watchdog.write_text( + "\n".join(f'- "{name}"' for name in names) + + "\ngithub.event.workflow_run.event == 'schedule'\n" + + "github.event.workflow_run.conclusion != 'success'\n" + + "github.event.workflow_run.conclusion != 'failure'\n" + + "workflow-name: ${{ github.event.workflow_run.name }}\n" + + "source-run-id: ${{ github.event.workflow_run.id }}\n" + + "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}\n" + ) + self.assertEqual(check_scheduled_alerts(root), []) + + first = root / SCHEDULED_ALERT_WORKFLOWS[0] + first.write_text(first.read_text().replace("contains(needs.*.result, 'failure')", "false")) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first.read_text().replace("false", "contains(needs.*.result, 'failure')")) + + watchdog.write_text(watchdog.read_text().replace(f'- "{names[0]}"\n', "")) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + def main() -> int: if sys.argv[1:] == ["--self-test"]: suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) @@ -436,7 +547,7 @@ def main() -> int: for error in errors: print(f"ERROR: {error}", file=sys.stderr) return 1 - print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired") + print("OK: e2e modules, runner selection, fuzz matrices, profiles, and scheduled alerts are wired") return 0