From ddc4120c82bf82627e711deee1f1ce302e141635 Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sun, 23 Aug 2026 01:40:28 +0800 Subject: [PATCH] ci: detect incomplete and stale scheduled validations (#6357) --- .config/make/tests.mak | 1 + .../actions/schedule-failure-issue/action.yml | 65 +- .github/scheduled-validations.json | 14 + .github/workflows/audit.yml | 2 +- .github/workflows/build.yml | 22 +- .github/workflows/ci-docs-only.yml | 5 +- .github/workflows/ci.yml | 41 +- .github/workflows/coverage.yml | 2 +- .github/workflows/e2e-replication-nightly.yml | 2 +- .github/workflows/e2e-s3tests.yml | 2 +- .github/workflows/fuzz.yml | 2 +- .github/workflows/minio-interop.yml | 18 + .github/workflows/mint.yml | 2 +- .github/workflows/nightly-gnu.yml | 22 +- .github/workflows/performance-ab.yml | 2 +- .github/workflows/runner-hygiene.yml | 2 +- .../scheduled-validation-freshness.yml | 57 ++ .../scheduled-validation-watchdog.yml | 63 ++ .../check_scheduled_validation_freshness.py | 263 ++++++++ scripts/check_test_wiring.py | 562 +++++++++++++++++- 20 files changed, 1123 insertions(+), 26 deletions(-) create mode 100644 .github/scheduled-validations.json create mode 100644 .github/workflows/scheduled-validation-freshness.yml create mode 100644 .github/workflows/scheduled-validation-watchdog.yml create mode 100644 scripts/check_scheduled_validation_freshness.py diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 1dec7db3e..626df9b84 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -36,6 +36,7 @@ script-tests: ## Run shell script tests ./scripts/test_manual_transition_runbooks.sh ./scripts/check_embedded_secrets.sh --self-test python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_scheduled_validation_freshness.py --self-test python3 ./scripts/s3-tests/test_report_compat.py bash -n ./scripts/validate_object_data_cache_cold_stampede.sh python3 ./scripts/check_object_data_cache_follower_samples.py --self-test diff --git a/.github/actions/schedule-failure-issue/action.yml b/.github/actions/schedule-failure-issue/action.yml index 60e938690..d505387e4 100644 --- a/.github/actions/schedule-failure-issue/action.yml +++ b/.github/actions/schedule-failure-issue/action.yml @@ -14,9 +14,10 @@ name: "Schedule Failure Issue" description: >- - Open (or update) a tracking issue when a scheduled workflow run fails. + Open (or update) a tracking issue when a scheduled workflow run fails or + does not complete normally. Dedupes by workflow name: if an open issue titled - "[scheduled-failure] " already exists, the failure is + "[scheduled-failure] " already exists, the result is appended as a comment; otherwise a new issue is created. This is the single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8). @@ -38,6 +39,30 @@ inputs: Set to an empty string to skip labeling. required: false default: "infrastructure" + source-run-id: + description: "Run ID to report. Defaults to the current workflow run." + required: false + default: ${{ github.run_id }} + source-run-attempt: + description: "Run attempt to report. Defaults to the current attempt." + required: false + default: ${{ github.run_attempt }} + source-event: + description: "Trigger event of the run being reported." + required: false + default: ${{ github.event_name }} + source-ref-name: + description: "Ref name of the run being reported." + required: false + default: ${{ github.ref_name }} + source-sha: + description: "Commit SHA of the run being reported." + required: false + default: ${{ github.sha }} + details-file: + description: "Optional Markdown file appended to the issue body." + required: false + default: "" runs: using: "composite" @@ -48,17 +73,22 @@ runs: GH_TOKEN: ${{ inputs.github-token }} WORKFLOW_NAME: ${{ inputs.workflow-name }} ISSUE_LABEL: ${{ inputs.label }} + SOURCE_RUN_ID: ${{ inputs.source-run-id }} + SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }} + SOURCE_EVENT: ${{ inputs.source-event }} + SOURCE_REF_NAME: ${{ inputs.source-ref-name }} + SOURCE_SHA: ${{ inputs.source-sha }} + DETAILS_FILE: ${{ inputs.details-file }} run: | set -euo pipefail title="[scheduled-failure] ${WORKFLOW_NAME}" - run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}" - # Failed job names for this run attempt. The alert job runs while the - # run as a whole is still in progress, so inspect the jobs that have - # already completed with a non-success conclusion. + # Inspect the reported run attempt. It can be the current in-workflow + # failure or a completed run observed by the external watchdog. failed_jobs="$(gh api \ - "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \ --paginate \ --jq '.jobs[] | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled") @@ -67,15 +97,26 @@ runs: failed_jobs="- (failed job not recorded yet — see the run page)" fi + details="" + if [ -n "${DETAILS_FILE}" ]; then + if [ -f "${DETAILS_FILE}" ]; then + details="$(cat "${DETAILS_FILE}")" + else + details="Details file was not available: \`${DETAILS_FILE}\`" + fi + fi + body="$(cat <- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index 85ce9b32f..de78e7f7f 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -126,7 +126,10 @@ jobs: run: ./scripts/check_embedded_secrets.sh - name: Check test wiring - run: python3 ./scripts/check_test_wiring.py + run: | + python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a06e4667..ae74ec40a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ on: merge_group: types: [ checks_requested ] schedule: - - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC + - cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC workflow_dispatch: permissions: @@ -161,7 +161,10 @@ jobs: run: ./scripts/check_embedded_secrets.sh - name: Check test wiring - run: python3 ./scripts/check_test_wiring.py + run: | + python3 ./scripts/check_test_wiring.py --self-test + python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed run: ./scripts/check_no_planning_docs.sh @@ -1032,3 +1035,37 @@ jobs: path: artifacts/s3tests-single/** if-no-files-found: ignore retention-days: 3 + + alert-on-failure: + name: Alert on scheduled failure + needs: + - typos + - quick-checks + - test-and-lint + - test-ilm-integration-serial + - test-and-lint-rio-v2 + - test-and-lint-protocols + - build-rustfs-debug-binary + - build-rustfs-debug-binary-rio-v2 + - uring-integration + - e2e-tests + - e2e-full + - e2e-tests-rio-v2 + - s3-implemented-tests + - s3-lifecycle-behavior-tests + if: >- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index baa3c87a7..ece00c2eb 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -37,7 +37,7 @@ on: # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17), # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes. - - cron: "0 7 * * 0" + - cron: "43 7 * * 0" # Only alert-on-failure needs more than read access; it declares its own # job-level `issues: write`. diff --git a/.github/workflows/e2e-replication-nightly.yml b/.github/workflows/e2e-replication-nightly.yml index 837d9f79f..ef312d180 100644 --- a/.github/workflows/e2e-replication-nightly.yml +++ b/.github/workflows/e2e-replication-nightly.yml @@ -40,7 +40,7 @@ on: schedule: # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00), # stale (01:30) and performance-ab (06:00). - - cron: "0 4 * * *" + - cron: "29 4 * * *" # Only alert-on-failure needs more than read access; it declares its own # job-level `issues: write`. diff --git a/.github/workflows/e2e-s3tests.yml b/.github/workflows/e2e-s3tests.yml index 1e3ad90b6..693fa8450 100644 --- a/.github/workflows/e2e-s3tests.yml +++ b/.github/workflows/e2e-s3tests.yml @@ -93,7 +93,7 @@ on: schedule: # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the # single-node and the 4-node distributed topologies (matrix below). - - cron: "0 2 * * 0" + - cron: "19 2 * * 0" env: # main user diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index d41a1107f..6aa780fc6 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -30,7 +30,7 @@ on: - "Cargo.lock" - ".github/workflows/fuzz.yml" schedule: - - cron: "0 2 * * *" + - cron: "17 2 * * *" workflow_dispatch: inputs: profile: diff --git a/.github/workflows/minio-interop.yml b/.github/workflows/minio-interop.yml index 3ee33e9bf..5f105ac41 100644 --- a/.github/workflows/minio-interop.yml +++ b/.github/workflows/minio-interop.yml @@ -121,3 +121,21 @@ jobs: cargo nextest run --run-ignored ignored-only --no-tests=fail \ -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \ -E "$INTEROP_FILTER" + + alert-on-failure: + name: Alert on scheduled failure + needs: [minio-interop] + if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/mint.yml b/.github/workflows/mint.yml index dd9acc6f8..b9a7e354a 100644 --- a/.github/workflows/mint.yml +++ b/.github/workflows/mint.yml @@ -76,7 +76,7 @@ on: schedule: # Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to # 3h) has finished, so the two never contend for the same runner pool. - - cron: "0 6 * * 0" + - cron: "41 6 * * 0" env: S3_ACCESS_KEY: rustfsadmin-ci diff --git a/.github/workflows/nightly-gnu.yml b/.github/workflows/nightly-gnu.yml index 1f7c2d488..946761e54 100644 --- a/.github/workflows/nightly-gnu.yml +++ b/.github/workflows/nightly-gnu.yml @@ -16,7 +16,7 @@ name: Nightly GNU Build on: schedule: - - cron: "0 0 * * *" + - cron: "7 0 * * *" timezone: "Asia/Shanghai" workflow_dispatch: @@ -194,3 +194,23 @@ jobs: - name: Run HA leader failover live checks (three-node Raft cluster in Docker) run: bash scripts/test/vault_ha_kms_live.sh + + alert-on-failure: + name: Alert on scheduled failure + needs: [build, kms-vault-lane, kms-vault-ha-failover] + if: >- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml index ac8e8d7c5..a7ecf1d4b 100644 --- a/.github/workflows/performance-ab.yml +++ b/.github/workflows/performance-ab.yml @@ -33,7 +33,7 @@ name: Performance A/B on: schedule: - - cron: "0 6 * * *" # 06:00 UTC nightly, against main + - cron: "31 6 * * *" # 06:31 UTC nightly, against main workflow_dispatch: inputs: duration: diff --git a/.github/workflows/runner-hygiene.yml b/.github/workflows/runner-hygiene.yml index c231b5706..bdebf2c77 100644 --- a/.github/workflows/runner-hygiene.yml +++ b/.github/workflows/runner-hygiene.yml @@ -30,7 +30,7 @@ name: Runner Hygiene on: schedule: - - cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron) + - cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC workflow_dispatch: permissions: diff --git a/.github/workflows/scheduled-validation-freshness.yml b/.github/workflows/scheduled-validation-freshness.yml new file mode 100644 index 000000000..eef340869 --- /dev/null +++ b/.github/workflows/scheduled-validation-freshness.yml @@ -0,0 +1,57 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Scheduled Validation Freshness + +on: + schedule: + - cron: "47 23 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: scheduled-validation-freshness + cancel-in-progress: false + +jobs: + check-freshness: + name: Check scheduled validation freshness + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Check latest scheduled runs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set +e + python3 scripts/check_scheduled_validation_freshness.py \ + --report "${RUNNER_TEMP}/scheduled-validation-freshness.md" + status=$? + cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}" + exit "${status}" + - name: Open or update freshness issue + if: failure() + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + details-file: ${{ runner.temp }}/scheduled-validation-freshness.md diff --git a/.github/workflows/scheduled-validation-watchdog.yml b/.github/workflows/scheduled-validation-watchdog.yml new file mode 100644 index 000000000..e778ec640 --- /dev/null +++ b/.github/workflows/scheduled-validation-watchdog.yml @@ -0,0 +1,63 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Scheduled Validation Watchdog + +on: + workflow_run: + workflows: + - "Security Audit" + - "Build and Release" + - "Continuous Integration" + - "coverage" + - "e2e-nightly" + - "e2e-s3tests" + - "Fuzz" + - "mint" + - "minio-interop" + - "Nightly GNU Build" + - "Performance A/B" + - "Runner Hygiene" + types: [completed] + +permissions: + contents: read + +jobs: + alert-on-incomplete-run: + name: Alert on incomplete scheduled run + if: >- + github.event.workflow_run.event == 'schedule' && + github.event.workflow_run.conclusion != 'success' && + github.event.workflow_run.conclusion != 'failure' + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update incomplete-run issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + workflow-name: ${{ github.event.workflow_run.name }} + source-run-id: ${{ github.event.workflow_run.id }} + source-run-attempt: ${{ github.event.workflow_run.run_attempt }} + source-event: ${{ github.event.workflow_run.event }} + source-ref-name: ${{ github.event.workflow_run.head_branch }} + source-sha: ${{ github.event.workflow_run.head_sha }} diff --git a/scripts/check_scheduled_validation_freshness.py b/scripts/check_scheduled_validation_freshness.py new file mode 100644 index 000000000..d9bdf0597 --- /dev/null +++ b/scripts/check_scheduled_validation_freshness.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""Fail when a critical scheduled validation has not started recently.""" + +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +import json +import os +from pathlib import Path +import re +import sys +import tempfile +import unittest +from unittest import mock +from urllib.parse import quote, urlencode +from urllib.request import Request, urlopen + + +ROOT = Path(__file__).resolve().parents[1] + + +def load_validations(path: Path) -> list[tuple[str, int]]: + data = json.loads(path.read_text()) + if not isinstance(data, list) or not data: + raise ValueError("scheduled validation config must be a non-empty list") + + validations: list[tuple[str, int]] = [] + seen: set[str] = set() + for item in data: + if not isinstance(item, dict): + raise ValueError("scheduled validation entries must be objects") + workflow = item.get("workflow") + max_age_hours = item.get("max_age_hours") + if not isinstance(workflow, str) or not re.fullmatch( + r"\.github/workflows/[a-z0-9-]+\.yml", workflow + ): + raise ValueError(f"invalid scheduled validation workflow: {workflow!r}") + if workflow in seen: + raise ValueError(f"duplicate scheduled validation workflow: {workflow}") + if ( + not isinstance(max_age_hours, int) + or isinstance(max_age_hours, bool) + or max_age_hours <= 0 + ): + raise ValueError(f"invalid max_age_hours for {workflow}: {max_age_hours!r}") + seen.add(workflow) + validations.append((workflow, max_age_hours)) + return validations + + +def parse_timestamp(value: object) -> datetime: + if not isinstance(value, str): + raise ValueError(f"invalid run timestamp: {value!r}") + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + raise ValueError(f"run timestamp has no timezone: {value!r}") + return parsed.astimezone(timezone.utc) + + +def stale_reason( + run: dict[str, object] | None, now: datetime, max_age_hours: int +) -> str | None: + if run is None: + return "no scheduled run has been recorded" + created_at = parse_timestamp(run.get("created_at")) + age = now - created_at + if age > timedelta(hours=max_age_hours): + return f"last scheduled run is {age.total_seconds() / 3600:.1f}h old" + return None + + +def fetch_latest_scheduled_run( + repository: str, workflow: str, token: str, api_url: str +) -> dict[str, object] | None: + owner, repo = repository.split("/", 1) + workflow_name = Path(workflow).name + endpoint = ( + f"{api_url.rstrip('/')}/repos/{quote(owner, safe='')}/{quote(repo, safe='')}" + f"/actions/workflows/{quote(workflow_name, safe='')}/runs?" + + urlencode({"event": "schedule", "per_page": 1}) + ) + request = Request( + endpoint, + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + }, + ) + with urlopen(request, timeout=30) as response: + payload = json.load(response) + runs = payload.get("workflow_runs") + if not isinstance(runs, list): + raise ValueError(f"GitHub returned no workflow_runs list for {workflow}") + if not runs: + return None + if not isinstance(runs[0], dict): + raise ValueError(f"GitHub returned an invalid workflow run for {workflow}") + return runs[0] + + +def write_report(path: Path, failures: list[tuple[str, int, str, str]]) -> None: + lines = ["## Scheduled validation freshness"] + if not failures: + lines.append("") + lines.append("All critical scheduled validations have a recent scheduled run.") + else: + lines.extend( + [ + "", + "The following critical validations are stale or could not be inspected:", + "", + "| Workflow | Limit | Result | Last run |", + "| --- | ---: | --- | --- |", + ] + ) + for workflow, max_age_hours, reason, run_url in failures: + link = f"[open]({run_url})" if run_url else "—" + lines.append(f"| `{workflow}` | {max_age_hours}h | {reason} | {link} |") + path.write_text("\n".join(lines) + "\n") + + +def check_freshness( + config: Path, report: Path, repository: str, token: str, api_url: str +) -> int: + now = datetime.now(timezone.utc) + failures: list[tuple[str, int, str, str]] = [] + for workflow, max_age_hours in load_validations(config): + try: + run = fetch_latest_scheduled_run(repository, workflow, token, api_url) + reason = stale_reason(run, now, max_age_hours) + if reason is not None: + run_url = str(run.get("html_url", "")) if run else "" + failures.append((workflow, max_age_hours, reason, run_url)) + except Exception as error: + failures.append( + (workflow, max_age_hours, f"inspection failed: {error}", "") + ) + write_report(report, failures) + return 1 if failures else 0 + + +class SelfTests(unittest.TestCase): + NOW = datetime(2026, 8, 22, 12, tzinfo=timezone.utc) + + def test_freshness_boundaries(self) -> None: + at_limit = {"created_at": "2026-08-21T00:00:00Z"} + past_limit = {"created_at": "2026-08-20T23:59:59Z"} + self.assertIsNone(stale_reason(at_limit, self.NOW, 36)) + self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36)) + self.assertIsNotNone(stale_reason(None, self.NOW, 36)) + + def test_config_rejects_duplicate_and_invalid_entries(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "validations.json" + path.write_text( + json.dumps( + [ + {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}, + {"workflow": ".github/workflows/ci.yml", "max_age_hours": 0}, + ] + ) + ) + with self.assertRaises(ValueError): + load_validations(path) + path.write_text( + json.dumps( + [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 0}] + ) + ) + with self.assertRaises(ValueError): + load_validations(path) + + def test_check_reports_missing_runs(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + config = root / "validations.json" + report = root / "report.md" + config.write_text( + json.dumps( + [ + {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}, + {"workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36}, + {"workflow": ".github/workflows/mint.yml", "max_age_hours": 36}, + ] + ) + ) + with mock.patch( + __name__ + ".fetch_latest_scheduled_run", + side_effect=[ + {"created_at": "2999-01-01T00:00:00Z"}, + None, + RuntimeError("API unavailable"), + ], + ): + self.assertEqual( + check_freshness( + config, + report, + "rustfs/rustfs", + "token", + "https://api.github.test", + ), + 1, + ) + contents = report.read_text() + self.assertIn(".github/workflows/fuzz.yml", contents) + self.assertIn("inspection failed: API unavailable", contents) + self.assertNotIn(".github/workflows/ci.yml`", contents) + + config.write_text( + json.dumps( + [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}] + ) + ) + with mock.patch( + __name__ + ".fetch_latest_scheduled_run", + return_value={"created_at": "2999-01-01T00:00:00Z"}, + ): + self.assertEqual( + check_freshness( + config, + report, + "rustfs/rustfs", + "token", + "https://api.github.test", + ), + 0, + ) + self.assertIn("All critical scheduled validations", report.read_text()) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--config", type=Path, default=ROOT / ".github/scheduled-validations.json" + ) + parser.add_argument("--report", type=Path) + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + + if args.self_test: + load_validations(args.config) + suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) + return ( + 0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1 + ) + if args.report is None: + parser.error("--report is required unless --self-test is used") + + repository = os.environ.get("GITHUB_REPOSITORY", "") + token = os.environ.get("GH_TOKEN", "") + api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com") + if not re.fullmatch(r"[^/\s]+/[^/\s]+", repository): + parser.error("GITHUB_REPOSITORY must be owner/repository") + if not token: + parser.error("GH_TOKEN is required") + return check_freshness(args.config, args.report, repository, token, api_url) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/check_test_wiring.py b/scripts/check_test_wiring.py index 4edd41f0d..aa97dc944 100755 --- a/scripts/check_test_wiring.py +++ b/scripts/check_test_wiring.py @@ -10,11 +10,17 @@ import sys import tempfile import tomllib import unittest +from datetime import datetime, timezone from unittest import mock from pathlib import Path +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError ROOT = Path(__file__).resolve().parents[1] +SCHEDULED_ALERT_WORKFLOWS = tuple( + item["workflow"] + for item in json.loads((ROOT / ".github/scheduled-validations.json").read_text()) +) def words(value: str) -> set[str]: @@ -252,6 +258,292 @@ def check_profile_definitions(root: Path) -> list[str]: return errors +def yaml_block(lines: list[str], key: str, indent: int) -> list[str] | None: + try: + start = lines.index(f"{' ' * indent}{key}:") + 1 + except ValueError: + return None + end = next( + ( + index + for index in range(start, len(lines)) + if lines[index].strip() + and not lines[index].lstrip().startswith("#") + and len(lines[index]) - len(lines[index].lstrip()) <= indent + ), + len(lines), + ) + return lines[start:end] + + +def workflow_step_block(job_lines: list[str], action: str) -> tuple[int, list[str]] | None: + uses_index = next( + ( + index + for index, line in enumerate(job_lines) + if ( + line.split("#", 1)[0].strip() == f"- uses: {action}" + and len(line) - len(line.lstrip()) == 6 + ) + or ( + line.split("#", 1)[0].strip() == f"uses: {action}" + and len(line) - len(line.lstrip()) == 8 + ) + ), + None, + ) + if uses_index is None: + return None + start = next( + ( + index + for index in range(uses_index, -1, -1) + if job_lines[index].lstrip().startswith("- ") + ), + uses_index, + ) + indent = len(job_lines[start]) - len(job_lines[start].lstrip()) + end = next( + ( + index + for index in range(start + 1, len(job_lines)) + if len(job_lines[index]) - len(job_lines[index].lstrip()) == indent + and job_lines[index].lstrip().startswith("- ") + ), + len(job_lines), + ) + return start, job_lines[start:end] + + +def alert_step_errors( + job_lines: list[str], + expected_action_if: str | None, + required_permissions: tuple[str, ...], + required_action_tokens: tuple[str, ...], +) -> list[str]: + checkout = workflow_step_block(job_lines, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0") + action = workflow_step_block(job_lines, "./.github/actions/schedule-failure-issue") + errors: list[str] = [] + permissions = yaml_block(job_lines, "permissions", 4) + permission_text = "\n".join(line.split("#", 1)[0] for line in permissions or []) + missing_permissions = [token for token in required_permissions if token not in permission_text] + if missing_permissions: + errors.append("alert job permissions missing " + ", ".join(missing_permissions)) + if checkout is None: + errors.append("checkout step is missing") + if action is None: + errors.append("local alert action step is missing") + if checkout is None or action is None: + return errors + + if checkout[0] >= action[0]: + errors.append("checkout must run before the local alert action") + checkout_ifs = [line.strip() for line in checkout[1] if line.strip().startswith("if:")] + if checkout_ifs: + errors.append("checkout step must not be conditional") + action_ifs = [line.strip() for line in action[1] if line.strip().startswith("if:")] + expected_ifs = [] if expected_action_if is None else [expected_action_if] + if action_ifs != expected_ifs: + errors.append("alert action has an invalid step condition") + action_text = "\n".join(line.split("#", 1)[0] for line in action[1]) + missing_action_tokens = [token for token in required_action_tokens if token not in action_text] + if missing_action_tokens: + errors.append("alert action inputs missing " + ", ".join(missing_action_tokens)) + return errors + + +def schedule_utc_slots(hour: int, minute: int, timezone_name: str | None) -> set[tuple[int, int]]: + if timezone_name is None: + return {(hour, minute)} + zone = ZoneInfo(timezone_name) + return { + (utc.hour, utc.minute) + for year in (2025, 2026) + for month in range(1, 13) + for utc in [datetime(year, month, 1, hour, minute, tzinfo=zone).astimezone(timezone.utc)] + } + + +def check_scheduled_alerts(root: Path) -> list[str]: + errors: list[str] = [] + schedule_slots: dict[tuple[int, int], list[str]] = {} + for relative in SCHEDULED_ALERT_WORKFLOWS: + path = root / relative + try: + lines = path.read_text().splitlines() + except FileNotFoundError: + errors.append(f"{relative}: missing scheduled validation workflow") + continue + + on_block = yaml_block(lines, "on", 0) + schedule_block = yaml_block(on_block or [], "schedule", 2) + schedule_lines = schedule_block or [] + cron_indices = [index for index, line in enumerate(schedule_lines) if re.match(r"^\s*-\s+cron:", line)] + if not cron_indices: + errors.append(f"{relative}: missing simple numeric schedule") + else: + for position, cron_index in enumerate(cron_indices): + cron_line = schedule_lines[cron_index] + schedule = re.match(r"^\s*-\s+cron:\s*[\"']?(\d+)\s+(\d+)\s+", cron_line) + if not schedule: + errors.append(f"{relative}: missing simple numeric schedule") + continue + minute, hour = map(int, schedule.groups()) + if minute == 0: + errors.append(f"{relative}: scheduled validation must avoid minute zero") + entry_end = cron_indices[position + 1] if position + 1 < len(cron_indices) else len(schedule_lines) + entry = "\n".join(schedule_lines[cron_index + 1 : entry_end]) + timezone_match = re.search(r"^\s*timezone:\s*[\"']?([^\"'\s]+)", entry, re.MULTILINE) + timezone_name = timezone_match.group(1) if timezone_match else None + try: + utc_slots = schedule_utc_slots(hour, minute, timezone_name) + except ZoneInfoNotFoundError: + errors.append(f"{relative}: unknown schedule timezone {timezone_name}") + continue + for slot in utc_slots: + schedule_slots.setdefault(slot, []).append(relative) + + job_lines = yaml_block(lines, "alert-on-failure", 2) + if job_lines is None: + errors.append(f"{relative}: missing alert-on-failure job") + continue + job = "\n".join(line.split("#", 1)[0] for line in job_lines) + required = ( + "always()", + "github.event_name == 'schedule'", + "contains(needs.*.result, 'failure')", + "issues: write", + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: ./.github/actions/schedule-failure-issue", + "github-token: ${{ secrets.GITHUB_TOKEN }}", + ) + missing = [token for token in required if token not in job] + if missing: + errors.append(f"{relative}: alert-on-failure missing {', '.join(missing)}") + else: + errors.extend( + f"{relative}: {error}" + for error in alert_step_errors(job_lines, None, ("issues: write",), ("github-token: ${{ secrets.GITHUB_TOKEN }}",)) + ) + + for (hour, minute), workflows in schedule_slots.items(): + if len(workflows) > 1: + errors.append( + f"scheduled validations share {hour:02d}:{minute:02d} UTC: {', '.join(workflows)}" + ) + + watchdog_path = root / ".github/workflows/scheduled-validation-watchdog.yml" + try: + watchdog_lines = watchdog_path.read_text().splitlines() + except FileNotFoundError: + errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing completion watchdog") + return errors + watchdog_on = yaml_block(watchdog_lines, "on", 0) + watchdog_run = yaml_block(watchdog_on or [], "workflow_run", 2) + watchdog_workflows = yaml_block(watchdog_run or [], "workflows", 4) + if watchdog_workflows is None: + errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing workflow_run workflows") + return errors + watchdog_sources = "\n".join(line.split("#", 1)[0] for line in watchdog_workflows) + for relative in SCHEDULED_ALERT_WORKFLOWS: + path = root / relative + if not path.is_file(): + continue + source = path.read_text() + match = re.search(r"^name:\s*[\"']?([^\"'\n]+)", source, re.MULTILINE) + if not match: + errors.append(f"{relative}: missing workflow name") + elif f'- "{match.group(1).strip()}"' not in watchdog_sources: + errors.append(f"{relative}: missing from scheduled completion watchdog") + watchdog_job_lines = yaml_block(watchdog_lines, "alert-on-incomplete-run", 2) + if watchdog_job_lines is None: + errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing alert-on-incomplete-run job") + return errors + watchdog_job = "\n".join(line.split("#", 1)[0] for line in watchdog_job_lines) + required = ( + "github.event.workflow_run.event == 'schedule'", + "github.event.workflow_run.conclusion != 'success'", + "github.event.workflow_run.conclusion != 'failure'", + "actions: read", + "issues: write", + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: ./.github/actions/schedule-failure-issue", + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "workflow-name: ${{ github.event.workflow_run.name }}", + "source-run-id: ${{ github.event.workflow_run.id }}", + "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}", + "source-event: ${{ github.event.workflow_run.event }}", + "source-ref-name: ${{ github.event.workflow_run.head_branch }}", + "source-sha: ${{ github.event.workflow_run.head_sha }}", + ) + missing = [token for token in required if token not in watchdog_job] + if missing: + errors.append( + ".github/workflows/scheduled-validation-watchdog.yml: missing " + ", ".join(missing) + ) + else: + errors.extend( + ".github/workflows/scheduled-validation-watchdog.yml: " + error + for error in alert_step_errors( + watchdog_job_lines, + None, + ("actions: read", "issues: write"), + ( + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "workflow-name: ${{ github.event.workflow_run.name }}", + "source-run-id: ${{ github.event.workflow_run.id }}", + "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}", + "source-event: ${{ github.event.workflow_run.event }}", + "source-ref-name: ${{ github.event.workflow_run.head_branch }}", + "source-sha: ${{ github.event.workflow_run.head_sha }}", + ), + ) + ) + + freshness_path = root / ".github/workflows/scheduled-validation-freshness.yml" + try: + freshness_lines = freshness_path.read_text().splitlines() + except FileNotFoundError: + errors.append(".github/workflows/scheduled-validation-freshness.yml: missing freshness check") + return errors + freshness_job_lines = yaml_block(freshness_lines, "check-freshness", 2) + if freshness_job_lines is None: + errors.append(".github/workflows/scheduled-validation-freshness.yml: missing check-freshness job") + return errors + freshness_job = "\n".join(line.split("#", 1)[0] for line in freshness_job_lines) + required = ( + "python3 scripts/check_scheduled_validation_freshness.py", + "actions: read", + "issues: write", + "if: failure()", + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: ./.github/actions/schedule-failure-issue", + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "details-file: ${{ runner.temp }}/scheduled-validation-freshness.md", + ) + missing = [token for token in required if token not in freshness_job] + if missing: + errors.append( + ".github/workflows/scheduled-validation-freshness.yml: missing " + ", ".join(missing) + ) + else: + errors.extend( + ".github/workflows/scheduled-validation-freshness.yml: " + error + for error in alert_step_errors( + freshness_job_lines, + "if: failure()", + ("actions: read", "issues: write"), + ( + "github-token: ${{ secrets.GITHUB_TOKEN }}", + "details-file: ${{ runner.temp }}/scheduled-validation-freshness.md", + ), + ) + ) + if not (root / "scripts/check_scheduled_validation_freshness.py").is_file(): + errors.append("scripts/check_scheduled_validation_freshness.py: missing freshness checker") + return errors + + def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]: try: expected_digest = profile_selection(root, profile) @@ -281,6 +573,7 @@ def validate(root: Path) -> list[str]: errors.extend(check_runner_selection(root)) errors.extend(check_s3_tests_runner(root)) errors.extend(check_profile_definitions(root)) + errors.extend(check_scheduled_alerts(root)) return errors @@ -363,6 +656,7 @@ class SelfTests(unittest.TestCase): mock.patch(__name__ + ".check_fuzz_targets", return_value=[]), mock.patch(__name__ + ".check_runner_selection", return_value=[]), mock.patch(__name__ + ".check_profile_definitions", return_value=[]), + mock.patch(__name__ + ".check_scheduled_alerts", return_value=[]), ): self.assertEqual(len(validate(root)), 1) @@ -413,6 +707,272 @@ class SelfTests(unittest.TestCase): with mock.patch.object(sys, "platform", "linux"): self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1) + def test_scheduled_alerts_require_completion_watchdog(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + alert = ( + " alert-on-failure:\n" + " if: always() && github.event_name == 'schedule' && " + "contains(needs.*.result, 'failure')\n" + " permissions:\n" + " issues: write\n" + " steps:\n" + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " - uses: ./.github/actions/schedule-failure-issue\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + ) + names: list[str] = [] + for index, relative in enumerate(SCHEDULED_ALERT_WORKFLOWS, start=1): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + names.append(path.stem) + path.write_text( + f'name: "{path.stem}"\n' + f'on:\n schedule:\n - cron: "{index} {index} * * *"\n' + f'jobs:\n{alert}' + ) + watchdog = root / ".github/workflows/scheduled-validation-watchdog.yml" + watchdog.write_text( + "on:\n workflow_run:\n workflows:\n" + + "\n".join(f' - "{name}"' for name in names) + + "\njobs:\n" + + " alert-on-incomplete-run:\n" + + " github.event.workflow_run.event == 'schedule'\n" + + " github.event.workflow_run.conclusion != 'success'\n" + + " github.event.workflow_run.conclusion != 'failure'\n" + + " permissions:\n" + + " actions: read\n" + + " issues: write\n" + + " steps:\n" + + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + + " - uses: ./.github/actions/schedule-failure-issue\n" + + " with:\n" + + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + + " workflow-name: ${{ github.event.workflow_run.name }}\n" + + " source-run-id: ${{ github.event.workflow_run.id }}\n" + + " source-run-attempt: ${{ github.event.workflow_run.run_attempt }}\n" + + " source-event: ${{ github.event.workflow_run.event }}\n" + + " source-ref-name: ${{ github.event.workflow_run.head_branch }}\n" + + " source-sha: ${{ github.event.workflow_run.head_sha }}\n" + ) + freshness = root / ".github/workflows/scheduled-validation-freshness.yml" + freshness.write_text( + "jobs:\n" + " check-freshness:\n" + " permissions:\n" + " actions: read\n" + " issues: write\n" + " steps:\n" + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " - run: python3 scripts/check_scheduled_validation_freshness.py\n" + " - uses: ./.github/actions/schedule-failure-issue\n" + " if: failure()\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + " details-file: ${{ runner.temp }}/scheduled-validation-freshness.md\n" + ) + checker = root / "scripts/check_scheduled_validation_freshness.py" + checker.parent.mkdir() + checker.write_text("") + self.assertEqual(check_scheduled_alerts(root), []) + + first = root / SCHEDULED_ALERT_WORKFLOWS[0] + mutations = ( + ("contains(needs.*.result, 'failure')", "false"), + ("issues: write", "issues: read"), + ( + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: actions/checkout@missing", + ), + ( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ( + " - uses: ./.github/actions/schedule-failure-issue\n", + " - uses: ./.github/actions/schedule-failure-issue\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"), + ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"), + ) + for required, replacement in mutations: + original = first.read_text() + first.write_text(original.replace(required, replacement)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(original) + + first_original = first.read_text() + real_steps = ( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " - uses: ./.github/actions/schedule-failure-issue\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + ) + first.write_text( + first_original.replace( + real_steps, + " - run: |\n" + " : <<'MARKER'\n" + " uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " MARKER\n" + " - run: |\n" + " : <<'MARKER'\n" + " uses: ./.github/actions/schedule-failure-issue\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + " MARKER\n", + ) + ) + self.assertTrue(check_scheduled_alerts(root)) + first.write_text( + first_original.replace( + real_steps, + " - uses: ./.github/actions/schedule-failure-issue\n" + " with:\n" + " github-token: ${{ secrets.GITHUB_TOKEN }}\n" + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + + watchdog_mutations = ( + ("actions: read", "actions: none"), + ("issues: write", "issues: read"), + ( + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: actions/checkout@missing", + ), + ( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ( + " - uses: ./.github/actions/schedule-failure-issue\n", + " - uses: ./.github/actions/schedule-failure-issue\n" + " if: github.event_name == 'workflow_dispatch'\n", + ), + ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"), + ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"), + ("source-event: ${{ github.event.workflow_run.event }}", "source-event: watchdog"), + ( + "source-ref-name: ${{ github.event.workflow_run.head_branch }}", + "source-ref-name: main", + ), + ("source-sha: ${{ github.event.workflow_run.head_sha }}", "source-sha: missing"), + ) + for required, replacement in watchdog_mutations: + original = watchdog.read_text() + watchdog.write_text(original.replace(required, replacement)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(original) + + watchdog_original = watchdog.read_text() + watchdog.write_text( + watchdog_original.replace("issues: write", "issues: read") + + " decoy:\n permissions:\n issues: write\n" + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(watchdog_original) + + first_original = first.read_text() + first.write_text( + first_original.replace(' schedule:\n - cron: "1 1 * * *"\n', "") + + ' decoy:\n strategy:\n matrix:\n cron:\n - "1 1 * * *"\n' + + ' runs-on: ubuntu-latest\n steps:\n - run: true\n' + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + + first.write_text( + first_original.replace( + ' - cron: "1 1 * * *"\n', + ' - cron: "1 1 * * *"\n - cron: "0 5 * * *"\n', + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text( + first_original.replace( + ' - cron: "1 1 * * *"\n', + ' - cron: "1 1 * * *"\n - cron: "2 2 * * *"\n', + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + + watchdog.write_text( + watchdog_original.replace(f' - "{names[0]}"\n', "") + + f' decoy:\n strategy:\n matrix:\n workflow:\n - "{names[0]}"\n' + + ' runs-on: ubuntu-latest\n steps:\n - run: true\n' + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(watchdog_original) + + watchdog.write_text(watchdog_original.replace(f' - "{names[0]}"\n', "")) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + watchdog.write_text(watchdog_original) + original = first.read_text() + first.write_text(re.sub(r'- cron: "\d+ \d+', '- cron: "0 0', original, count=1)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(original) + + second = root / SCHEDULED_ALERT_WORKFLOWS[1] + second_original = second.read_text() + second.write_text(re.sub(r'- cron: "\d+ \d+', '- cron: "1 1', second_original, count=1)) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + second.write_text(second_original) + + first.write_text( + first_original.replace( + ' - cron: "1 1 * * *"\n', + ' - cron: "7 0 * * *"\n timezone: "Asia/Shanghai"\n', + ) + ) + second.write_text( + second_original.replace( + ' - cron: "2 2 * * *"\n', + ' - cron: "2 2 * * *"\n - cron: "7 16 * * *"\n', + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + first.write_text(first_original) + second.write_text(second_original) + + freshness_original = freshness.read_text() + freshness.write_text(freshness_original.replace("details-file:", "report-file:")) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing") + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace( + "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0", + "uses: actions/checkout@missing", + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace( + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n", + " - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n" + " if: github.event_name == 'workflow_dispatch'\n", + ) + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace("if: failure()", "if: github.event_name == 'workflow_dispatch'") + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + freshness.write_text( + freshness_original.replace("issues: write", "issues: read") + + " decoy:\n permissions:\n issues: write\n" + ) + self.assertEqual(len(check_scheduled_alerts(root)), 1) + def main() -> int: if sys.argv[1:] == ["--self-test"]: suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests) @@ -436,7 +996,7 @@ def main() -> int: for error in errors: print(f"ERROR: {error}", file=sys.stderr) return 1 - print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired") + print("OK: e2e modules, runner selection, fuzz matrices, profiles, and scheduled alerts are wired") return 0