ci: detect incomplete and stale scheduled validations (#6357)

2026-08-22 20:36:38 +00:00 · 2026-08-23 01:40:28 +08:00
parent 87235ffd28
commit ddc4120c82
20 changed files with 1123 additions and 26 deletions
@@ -36,6 +36,7 @@ script-tests: ## Run shell script tests
 	./scripts/test_manual_transition_runbooks.sh
 	./scripts/check_embedded_secrets.sh --self-test
 	python3 ./scripts/check_test_wiring.py --self-test
+	python3 ./scripts/check_scheduled_validation_freshness.py --self-test
 	python3 ./scripts/s3-tests/test_report_compat.py
 	bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
 	python3 ./scripts/check_object_data_cache_follower_samples.py --self-test
@@ -14,9 +14,10 @@

 name: "Schedule Failure Issue"
 description: >-
-  Open (or update) a tracking issue when a scheduled workflow run fails.
+  Open (or update) a tracking issue when a scheduled workflow run fails or
+  does not complete normally.
  Dedupes by workflow name: if an open issue titled
-  "[scheduled-failure] <workflow name>" already exists, the failure is
+  "[scheduled-failure] <workflow name>" already exists, the result is
  appended as a comment; otherwise a new issue is created. This is the
  single alerting mechanism for all scheduled pipelines (backlog#1149 ci-8).

@@ -38,6 +39,30 @@ inputs:
      Set to an empty string to skip labeling.
    required: false
    default: "infrastructure"
+  source-run-id:
+    description: "Run ID to report. Defaults to the current workflow run."
+    required: false
+    default: ${{ github.run_id }}
+  source-run-attempt:
+    description: "Run attempt to report. Defaults to the current attempt."
+    required: false
+    default: ${{ github.run_attempt }}
+  source-event:
+    description: "Trigger event of the run being reported."
+    required: false
+    default: ${{ github.event_name }}
+  source-ref-name:
+    description: "Ref name of the run being reported."
+    required: false
+    default: ${{ github.ref_name }}
+  source-sha:
+    description: "Commit SHA of the run being reported."
+    required: false
+    default: ${{ github.sha }}
+  details-file:
+    description: "Optional Markdown file appended to the issue body."
+    required: false
+    default: ""

 runs:
  using: "composite"
@@ -48,17 +73,22 @@ runs:
        GH_TOKEN: ${{ inputs.github-token }}
        WORKFLOW_NAME: ${{ inputs.workflow-name }}
        ISSUE_LABEL: ${{ inputs.label }}
+        SOURCE_RUN_ID: ${{ inputs.source-run-id }}
+        SOURCE_RUN_ATTEMPT: ${{ inputs.source-run-attempt }}
+        SOURCE_EVENT: ${{ inputs.source-event }}
+        SOURCE_REF_NAME: ${{ inputs.source-ref-name }}
+        SOURCE_SHA: ${{ inputs.source-sha }}
+        DETAILS_FILE: ${{ inputs.details-file }}
      run: |
        set -euo pipefail

        title="[scheduled-failure] ${WORKFLOW_NAME}"
-        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+        run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}"

-        # Failed job names for this run attempt. The alert job runs while the
-        # run as a whole is still in progress, so inspect the jobs that have
-        # already completed with a non-success conclusion.
+        # Inspect the reported run attempt. It can be the current in-workflow
+        # failure or a completed run observed by the external watchdog.
        failed_jobs="$(gh api \
-          "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}/jobs" \
+          "repos/${GITHUB_REPOSITORY}/actions/runs/${SOURCE_RUN_ID}/attempts/${SOURCE_RUN_ATTEMPT}/jobs" \
          --paginate \
          --jq '.jobs[]
                | select(.conclusion == "failure" or .conclusion == "timed_out" or .conclusion == "cancelled")
@@ -67,15 +97,26 @@ runs:
          failed_jobs="- (failed job not recorded yet — see the run page)"
        fi

+        details=""
+        if [ -n "${DETAILS_FILE}" ]; then
+          if [ -f "${DETAILS_FILE}" ]; then
+            details="$(cat "${DETAILS_FILE}")"
+          else
+            details="Details file was not available: \`${DETAILS_FILE}\`"
+          fi
+        fi
+
        body="$(cat <<EOF
-        Scheduled run of **${WORKFLOW_NAME}** failed.
+        Run of **${WORKFLOW_NAME}** did not complete successfully.

-        - Run: ${run_url} (attempt ${GITHUB_RUN_ATTEMPT})
-        - Event: \`${GITHUB_EVENT_NAME}\`
-        - Ref: \`${GITHUB_REF_NAME}\` @ \`${GITHUB_SHA}\`
+        - Run: ${run_url} (attempt ${SOURCE_RUN_ATTEMPT})
+        - Event: \`${SOURCE_EVENT}\`
+        - Ref: \`${SOURCE_REF_NAME}\` @ \`${SOURCE_SHA}\`

-        Failed jobs:
+        Non-success jobs:
        ${failed_jobs}
+
+        ${details}
        EOF
        )"

@@ -0,0 +1,14 @@
+[
+  { "workflow": ".github/workflows/audit.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/build.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/ci.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/coverage.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/e2e-replication-nightly.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
+  { "workflow": ".github/workflows/minio-interop.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
+  { "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792 }
+]
@@ -46,7 +46,7 @@ on:
    # advisory could sit unnoticed for seven days. The check list is unchanged —
    # splitting it into a light daily advisories-only run and a weekly full run
    # would create runs where sources/bans/licenses go unverified.
-    - cron: '0 3 * * *' # Daily 03:00 UTC (staggered after the midnight ci/build crons)
+    - cron: '23 3 * * *' # Daily 03:23 UTC
  workflow_dispatch:

 permissions:
@@ -52,7 +52,7 @@ on:
      - ".dockerignore"
      - "flake.lock"
  schedule:
-    - cron: "0 1 * * 0" # Weekly on Sunday 01:00 UTC (staggered after the ci.yml midnight cron)
+    - cron: "13 1 * * 0" # Weekly on Sunday 01:13 UTC
  workflow_dispatch:
    inputs:
      build_docker:
@@ -1032,3 +1032,23 @@ jobs:

          echo "🎉 Released $TAG successfully!"
          echo "📄 Release URL: ${{ needs.create-release.outputs.release_url }}"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build-check, prepare-platform-matrix, build-rustfs, build-summary]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -126,7 +126,10 @@ jobs:
        run: ./scripts/check_embedded_secrets.sh

      - name: Check test wiring
-        run: python3 ./scripts/check_test_wiring.py
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/check_test_wiring.py

      - name: Check no planning docs committed
        run: ./scripts/check_no_planning_docs.sh
@@ -59,7 +59,7 @@ on:
  merge_group:
    types: [ checks_requested ]
  schedule:
-    - cron: "0 0 * * 0" # Weekly on Sunday at midnight UTC
+    - cron: "11 0 * * 0" # Weekly on Sunday 00:11 UTC
  workflow_dispatch:

 permissions:
@@ -161,7 +161,10 @@ jobs:
        run: ./scripts/check_embedded_secrets.sh

      - name: Check test wiring
-        run: python3 ./scripts/check_test_wiring.py
+        run: |
+          python3 ./scripts/check_test_wiring.py --self-test
+          python3 ./scripts/check_scheduled_validation_freshness.py --self-test
+          python3 ./scripts/check_test_wiring.py

      - name: Check no planning docs committed
        run: ./scripts/check_no_planning_docs.sh
@@ -1032,3 +1035,37 @@ jobs:
          path: artifacts/s3tests-single/**
          if-no-files-found: ignore
          retention-days: 3
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs:
+      - typos
+      - quick-checks
+      - test-and-lint
+      - test-ilm-integration-serial
+      - test-and-lint-rio-v2
+      - test-and-lint-protocols
+      - build-rustfs-debug-binary
+      - build-rustfs-debug-binary-rio-v2
+      - uring-integration
+      - e2e-tests
+      - e2e-full
+      - e2e-tests-rio-v2
+      - s3-implemented-tests
+      - s3-lifecycle-behavior-tests
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -37,7 +37,7 @@ on:
    # build (01:00), e2e-s3tests (02:00), audit (03:00), nix-flake-update
    # (05:00), mint (06:00), and the daily fuzz (02:00), minio-interop (03:17),
    # e2e-replication-nightly (04:00) and performance-ab (06:00) lanes.
-    - cron: "0 7 * * 0"
+    - cron: "43 7 * * 0"

 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
@@ -40,7 +40,7 @@ on:
  schedule:
    # 04:00 UTC nightly — staggered clear of fuzz/e2e-s3tests (02:00),
    # stale (01:30) and performance-ab (06:00).
-    - cron: "0 4 * * *"
+    - cron: "29 4 * * *"

 # Only alert-on-failure needs more than read access; it declares its own
 # job-level `issues: write`.
@@ -93,7 +93,7 @@ on:
  schedule:
    # Weekly full sweep (Sunday 02:00 UTC): full suite, run against BOTH the
    # single-node and the 4-node distributed topologies (matrix below).
-    - cron: "0 2 * * 0"
+    - cron: "19 2 * * 0"

 env:
  # main user
@@ -30,7 +30,7 @@ on:
      - "Cargo.lock"
      - ".github/workflows/fuzz.yml"
  schedule:
-    - cron: "0 2 * * *"
+    - cron: "17 2 * * *"
  workflow_dispatch:
    inputs:
      profile:
@@ -121,3 +121,21 @@ jobs:
          cargo nextest run --run-ignored ignored-only --no-tests=fail \
            -p "$INTEROP_PACKAGE" --features "$INTEROP_FEATURES" \
            -E "$INTEROP_FILTER"
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [minio-interop]
+    if: always() && github.event_name == 'schedule' && contains(needs.*.result, 'failure')
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -76,7 +76,7 @@ on:
  schedule:
    # Weekly, after the Sunday s3-tests full sweep (starts 02:00 UTC, up to
    # 3h) has finished, so the two never contend for the same runner pool.
-    - cron: "0 6 * * 0"
+    - cron: "41 6 * * 0"

 env:
  S3_ACCESS_KEY: rustfsadmin-ci
@@ -16,7 +16,7 @@ name: Nightly GNU Build

 on:
  schedule:
-    - cron: "0 0 * * *"
+    - cron: "7 0 * * *"
      timezone: "Asia/Shanghai"
  workflow_dispatch:

@@ -194,3 +194,23 @@ jobs:

      - name: Run HA leader failover live checks (three-node Raft cluster in Docker)
        run: bash scripts/test/vault_ha_kms_live.sh
+
+  alert-on-failure:
+    name: Alert on scheduled failure
+    needs: [build, kms-vault-lane, kms-vault-ha-failover]
+    if: >-
+      always() && github.event_name == 'schedule' &&
+      (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled'))
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update failure-tracking issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
@@ -33,7 +33,7 @@ name: Performance A/B

 on:
  schedule:
-    - cron: "0 6 * * *" # 06:00 UTC nightly, against main
+    - cron: "31 6 * * *" # 06:31 UTC nightly, against main
  workflow_dispatch:
    inputs:
      duration:
@@ -30,7 +30,7 @@ name: Runner Hygiene

 on:
  schedule:
-    - cron: "0 6 1 * *" # Monthly, 1st at 06:00 UTC (after the daily audit cron)
+    - cron: "37 6 1 * *" # Monthly, 1st at 06:37 UTC
  workflow_dispatch:

 permissions:
@@ -0,0 +1,57 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Validation Freshness
+
+on:
+  schedule:
+    - cron: "47 23 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: scheduled-validation-freshness
+  cancel-in-progress: false
+
+jobs:
+  check-freshness:
+    name: Check scheduled validation freshness
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Check latest scheduled runs
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set +e
+          python3 scripts/check_scheduled_validation_freshness.py \
+            --report "${RUNNER_TEMP}/scheduled-validation-freshness.md"
+          status=$?
+          cat "${RUNNER_TEMP}/scheduled-validation-freshness.md" >> "${GITHUB_STEP_SUMMARY}"
+          exit "${status}"
+      - name: Open or update freshness issue
+        if: failure()
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          details-file: ${{ runner.temp }}/scheduled-validation-freshness.md
@@ -0,0 +1,63 @@
+# Copyright 2024 RustFS Team
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Scheduled Validation Watchdog
+
+on:
+  workflow_run:
+    workflows:
+      - "Security Audit"
+      - "Build and Release"
+      - "Continuous Integration"
+      - "coverage"
+      - "e2e-nightly"
+      - "e2e-s3tests"
+      - "Fuzz"
+      - "mint"
+      - "minio-interop"
+      - "Nightly GNU Build"
+      - "Performance A/B"
+      - "Runner Hygiene"
+    types: [completed]
+
+permissions:
+  contents: read
+
+jobs:
+  alert-on-incomplete-run:
+    name: Alert on incomplete scheduled run
+    if: >-
+      github.event.workflow_run.event == 'schedule' &&
+      github.event.workflow_run.conclusion != 'success' &&
+      github.event.workflow_run.conclusion != 'failure'
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    permissions:
+      actions: read
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
+        with:
+          persist-credentials: false
+      - name: Open or update incomplete-run issue
+        uses: ./.github/actions/schedule-failure-issue
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          workflow-name: ${{ github.event.workflow_run.name }}
+          source-run-id: ${{ github.event.workflow_run.id }}
+          source-run-attempt: ${{ github.event.workflow_run.run_attempt }}
+          source-event: ${{ github.event.workflow_run.event }}
+          source-ref-name: ${{ github.event.workflow_run.head_branch }}
+          source-sha: ${{ github.event.workflow_run.head_sha }}
@@ -0,0 +1,263 @@
+#!/usr/bin/env python3
+"""Fail when a critical scheduled validation has not started recently."""
+
+from __future__ import annotations
+
+import argparse
+from datetime import datetime, timedelta, timezone
+import json
+import os
+from pathlib import Path
+import re
+import sys
+import tempfile
+import unittest
+from unittest import mock
+from urllib.parse import quote, urlencode
+from urllib.request import Request, urlopen
+
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def load_validations(path: Path) -> list[tuple[str, int]]:
+    data = json.loads(path.read_text())
+    if not isinstance(data, list) or not data:
+        raise ValueError("scheduled validation config must be a non-empty list")
+
+    validations: list[tuple[str, int]] = []
+    seen: set[str] = set()
+    for item in data:
+        if not isinstance(item, dict):
+            raise ValueError("scheduled validation entries must be objects")
+        workflow = item.get("workflow")
+        max_age_hours = item.get("max_age_hours")
+        if not isinstance(workflow, str) or not re.fullmatch(
+            r"\.github/workflows/[a-z0-9-]+\.yml", workflow
+        ):
+            raise ValueError(f"invalid scheduled validation workflow: {workflow!r}")
+        if workflow in seen:
+            raise ValueError(f"duplicate scheduled validation workflow: {workflow}")
+        if (
+            not isinstance(max_age_hours, int)
+            or isinstance(max_age_hours, bool)
+            or max_age_hours <= 0
+        ):
+            raise ValueError(f"invalid max_age_hours for {workflow}: {max_age_hours!r}")
+        seen.add(workflow)
+        validations.append((workflow, max_age_hours))
+    return validations
+
+
+def parse_timestamp(value: object) -> datetime:
+    if not isinstance(value, str):
+        raise ValueError(f"invalid run timestamp: {value!r}")
+    parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    if parsed.tzinfo is None:
+        raise ValueError(f"run timestamp has no timezone: {value!r}")
+    return parsed.astimezone(timezone.utc)
+
+
+def stale_reason(
+    run: dict[str, object] | None, now: datetime, max_age_hours: int
+) -> str | None:
+    if run is None:
+        return "no scheduled run has been recorded"
+    created_at = parse_timestamp(run.get("created_at"))
+    age = now - created_at
+    if age > timedelta(hours=max_age_hours):
+        return f"last scheduled run is {age.total_seconds() / 3600:.1f}h old"
+    return None
+
+
+def fetch_latest_scheduled_run(
+    repository: str, workflow: str, token: str, api_url: str
+) -> dict[str, object] | None:
+    owner, repo = repository.split("/", 1)
+    workflow_name = Path(workflow).name
+    endpoint = (
+        f"{api_url.rstrip('/')}/repos/{quote(owner, safe='')}/{quote(repo, safe='')}"
+        f"/actions/workflows/{quote(workflow_name, safe='')}/runs?"
+        + urlencode({"event": "schedule", "per_page": 1})
+    )
+    request = Request(
+        endpoint,
+        headers={
+            "Accept": "application/vnd.github+json",
+            "Authorization": f"Bearer {token}",
+            "X-GitHub-Api-Version": "2022-11-28",
+        },
+    )
+    with urlopen(request, timeout=30) as response:
+        payload = json.load(response)
+    runs = payload.get("workflow_runs")
+    if not isinstance(runs, list):
+        raise ValueError(f"GitHub returned no workflow_runs list for {workflow}")
+    if not runs:
+        return None
+    if not isinstance(runs[0], dict):
+        raise ValueError(f"GitHub returned an invalid workflow run for {workflow}")
+    return runs[0]
+
+
+def write_report(path: Path, failures: list[tuple[str, int, str, str]]) -> None:
+    lines = ["## Scheduled validation freshness"]
+    if not failures:
+        lines.append("")
+        lines.append("All critical scheduled validations have a recent scheduled run.")
+    else:
+        lines.extend(
+            [
+                "",
+                "The following critical validations are stale or could not be inspected:",
+                "",
+                "| Workflow | Limit | Result | Last run |",
+                "| --- | ---: | --- | --- |",
+            ]
+        )
+        for workflow, max_age_hours, reason, run_url in failures:
+            link = f"[open]({run_url})" if run_url else "—"
+            lines.append(f"| `{workflow}` | {max_age_hours}h | {reason} | {link} |")
+    path.write_text("\n".join(lines) + "\n")
+
+
+def check_freshness(
+    config: Path, report: Path, repository: str, token: str, api_url: str
+) -> int:
+    now = datetime.now(timezone.utc)
+    failures: list[tuple[str, int, str, str]] = []
+    for workflow, max_age_hours in load_validations(config):
+        try:
+            run = fetch_latest_scheduled_run(repository, workflow, token, api_url)
+            reason = stale_reason(run, now, max_age_hours)
+            if reason is not None:
+                run_url = str(run.get("html_url", "")) if run else ""
+                failures.append((workflow, max_age_hours, reason, run_url))
+        except Exception as error:
+            failures.append(
+                (workflow, max_age_hours, f"inspection failed: {error}", "")
+            )
+    write_report(report, failures)
+    return 1 if failures else 0
+
+
+class SelfTests(unittest.TestCase):
+    NOW = datetime(2026, 8, 22, 12, tzinfo=timezone.utc)
+
+    def test_freshness_boundaries(self) -> None:
+        at_limit = {"created_at": "2026-08-21T00:00:00Z"}
+        past_limit = {"created_at": "2026-08-20T23:59:59Z"}
+        self.assertIsNone(stale_reason(at_limit, self.NOW, 36))
+        self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36))
+        self.assertIsNotNone(stale_reason(None, self.NOW, 36))
+
+    def test_config_rejects_duplicate_and_invalid_entries(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "validations.json"
+            path.write_text(
+                json.dumps(
+                    [
+                        {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36},
+                        {"workflow": ".github/workflows/ci.yml", "max_age_hours": 0},
+                    ]
+                )
+            )
+            with self.assertRaises(ValueError):
+                load_validations(path)
+            path.write_text(
+                json.dumps(
+                    [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 0}]
+                )
+            )
+            with self.assertRaises(ValueError):
+                load_validations(path)
+
+    def test_check_reports_missing_runs(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            config = root / "validations.json"
+            report = root / "report.md"
+            config.write_text(
+                json.dumps(
+                    [
+                        {"workflow": ".github/workflows/ci.yml", "max_age_hours": 36},
+                        {"workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36},
+                        {"workflow": ".github/workflows/mint.yml", "max_age_hours": 36},
+                    ]
+                )
+            )
+            with mock.patch(
+                __name__ + ".fetch_latest_scheduled_run",
+                side_effect=[
+                    {"created_at": "2999-01-01T00:00:00Z"},
+                    None,
+                    RuntimeError("API unavailable"),
+                ],
+            ):
+                self.assertEqual(
+                    check_freshness(
+                        config,
+                        report,
+                        "rustfs/rustfs",
+                        "token",
+                        "https://api.github.test",
+                    ),
+                    1,
+                )
+            contents = report.read_text()
+            self.assertIn(".github/workflows/fuzz.yml", contents)
+            self.assertIn("inspection failed: API unavailable", contents)
+            self.assertNotIn(".github/workflows/ci.yml`", contents)
+
+            config.write_text(
+                json.dumps(
+                    [{"workflow": ".github/workflows/ci.yml", "max_age_hours": 36}]
+                )
+            )
+            with mock.patch(
+                __name__ + ".fetch_latest_scheduled_run",
+                return_value={"created_at": "2999-01-01T00:00:00Z"},
+            ):
+                self.assertEqual(
+                    check_freshness(
+                        config,
+                        report,
+                        "rustfs/rustfs",
+                        "token",
+                        "https://api.github.test",
+                    ),
+                    0,
+                )
+            self.assertIn("All critical scheduled validations", report.read_text())
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--config", type=Path, default=ROOT / ".github/scheduled-validations.json"
+    )
+    parser.add_argument("--report", type=Path)
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+
+    if args.self_test:
+        load_validations(args.config)
+        suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
+        return (
+            0 if unittest.TextTestRunner(verbosity=2).run(suite).wasSuccessful() else 1
+        )
+    if args.report is None:
+        parser.error("--report is required unless --self-test is used")
+
+    repository = os.environ.get("GITHUB_REPOSITORY", "")
+    token = os.environ.get("GH_TOKEN", "")
+    api_url = os.environ.get("GITHUB_API_URL", "https://api.github.com")
+    if not re.fullmatch(r"[^/\s]+/[^/\s]+", repository):
+        parser.error("GITHUB_REPOSITORY must be owner/repository")
+    if not token:
+        parser.error("GH_TOKEN is required")
+    return check_freshness(args.config, args.report, repository, token, api_url)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
@@ -10,11 +10,17 @@ import sys
 import tempfile
 import tomllib
 import unittest
+from datetime import datetime, timezone
 from unittest import mock
 from pathlib import Path
+from zoneinfo import ZoneInfo, ZoneInfoNotFoundError


 ROOT = Path(__file__).resolve().parents[1]
+SCHEDULED_ALERT_WORKFLOWS = tuple(
+    item["workflow"]
+    for item in json.loads((ROOT / ".github/scheduled-validations.json").read_text())
+)


 def words(value: str) -> set[str]:
@@ -252,6 +258,292 @@ def check_profile_definitions(root: Path) -> list[str]:
    return errors


+def yaml_block(lines: list[str], key: str, indent: int) -> list[str] | None:
+    try:
+        start = lines.index(f"{' ' * indent}{key}:") + 1
+    except ValueError:
+        return None
+    end = next(
+        (
+            index
+            for index in range(start, len(lines))
+            if lines[index].strip()
+            and not lines[index].lstrip().startswith("#")
+            and len(lines[index]) - len(lines[index].lstrip()) <= indent
+        ),
+        len(lines),
+    )
+    return lines[start:end]
+
+
+def workflow_step_block(job_lines: list[str], action: str) -> tuple[int, list[str]] | None:
+    uses_index = next(
+        (
+            index
+            for index, line in enumerate(job_lines)
+            if (
+                line.split("#", 1)[0].strip() == f"- uses: {action}"
+                and len(line) - len(line.lstrip()) == 6
+            )
+            or (
+                line.split("#", 1)[0].strip() == f"uses: {action}"
+                and len(line) - len(line.lstrip()) == 8
+            )
+        ),
+        None,
+    )
+    if uses_index is None:
+        return None
+    start = next(
+        (
+            index
+            for index in range(uses_index, -1, -1)
+            if job_lines[index].lstrip().startswith("- ")
+        ),
+        uses_index,
+    )
+    indent = len(job_lines[start]) - len(job_lines[start].lstrip())
+    end = next(
+        (
+            index
+            for index in range(start + 1, len(job_lines))
+            if len(job_lines[index]) - len(job_lines[index].lstrip()) == indent
+            and job_lines[index].lstrip().startswith("- ")
+        ),
+        len(job_lines),
+    )
+    return start, job_lines[start:end]
+
+
+def alert_step_errors(
+    job_lines: list[str],
+    expected_action_if: str | None,
+    required_permissions: tuple[str, ...],
+    required_action_tokens: tuple[str, ...],
+) -> list[str]:
+    checkout = workflow_step_block(job_lines, "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0")
+    action = workflow_step_block(job_lines, "./.github/actions/schedule-failure-issue")
+    errors: list[str] = []
+    permissions = yaml_block(job_lines, "permissions", 4)
+    permission_text = "\n".join(line.split("#", 1)[0] for line in permissions or [])
+    missing_permissions = [token for token in required_permissions if token not in permission_text]
+    if missing_permissions:
+        errors.append("alert job permissions missing " + ", ".join(missing_permissions))
+    if checkout is None:
+        errors.append("checkout step is missing")
+    if action is None:
+        errors.append("local alert action step is missing")
+    if checkout is None or action is None:
+        return errors
+
+    if checkout[0] >= action[0]:
+        errors.append("checkout must run before the local alert action")
+    checkout_ifs = [line.strip() for line in checkout[1] if line.strip().startswith("if:")]
+    if checkout_ifs:
+        errors.append("checkout step must not be conditional")
+    action_ifs = [line.strip() for line in action[1] if line.strip().startswith("if:")]
+    expected_ifs = [] if expected_action_if is None else [expected_action_if]
+    if action_ifs != expected_ifs:
+        errors.append("alert action has an invalid step condition")
+    action_text = "\n".join(line.split("#", 1)[0] for line in action[1])
+    missing_action_tokens = [token for token in required_action_tokens if token not in action_text]
+    if missing_action_tokens:
+        errors.append("alert action inputs missing " + ", ".join(missing_action_tokens))
+    return errors
+
+
+def schedule_utc_slots(hour: int, minute: int, timezone_name: str | None) -> set[tuple[int, int]]:
+    if timezone_name is None:
+        return {(hour, minute)}
+    zone = ZoneInfo(timezone_name)
+    return {
+        (utc.hour, utc.minute)
+        for year in (2025, 2026)
+        for month in range(1, 13)
+        for utc in [datetime(year, month, 1, hour, minute, tzinfo=zone).astimezone(timezone.utc)]
+    }
+
+
+def check_scheduled_alerts(root: Path) -> list[str]:
+    errors: list[str] = []
+    schedule_slots: dict[tuple[int, int], list[str]] = {}
+    for relative in SCHEDULED_ALERT_WORKFLOWS:
+        path = root / relative
+        try:
+            lines = path.read_text().splitlines()
+        except FileNotFoundError:
+            errors.append(f"{relative}: missing scheduled validation workflow")
+            continue
+
+        on_block = yaml_block(lines, "on", 0)
+        schedule_block = yaml_block(on_block or [], "schedule", 2)
+        schedule_lines = schedule_block or []
+        cron_indices = [index for index, line in enumerate(schedule_lines) if re.match(r"^\s*-\s+cron:", line)]
+        if not cron_indices:
+            errors.append(f"{relative}: missing simple numeric schedule")
+        else:
+            for position, cron_index in enumerate(cron_indices):
+                cron_line = schedule_lines[cron_index]
+                schedule = re.match(r"^\s*-\s+cron:\s*[\"']?(\d+)\s+(\d+)\s+", cron_line)
+                if not schedule:
+                    errors.append(f"{relative}: missing simple numeric schedule")
+                    continue
+                minute, hour = map(int, schedule.groups())
+                if minute == 0:
+                    errors.append(f"{relative}: scheduled validation must avoid minute zero")
+                entry_end = cron_indices[position + 1] if position + 1 < len(cron_indices) else len(schedule_lines)
+                entry = "\n".join(schedule_lines[cron_index + 1 : entry_end])
+                timezone_match = re.search(r"^\s*timezone:\s*[\"']?([^\"'\s]+)", entry, re.MULTILINE)
+                timezone_name = timezone_match.group(1) if timezone_match else None
+                try:
+                    utc_slots = schedule_utc_slots(hour, minute, timezone_name)
+                except ZoneInfoNotFoundError:
+                    errors.append(f"{relative}: unknown schedule timezone {timezone_name}")
+                    continue
+                for slot in utc_slots:
+                    schedule_slots.setdefault(slot, []).append(relative)
+
+        job_lines = yaml_block(lines, "alert-on-failure", 2)
+        if job_lines is None:
+            errors.append(f"{relative}: missing alert-on-failure job")
+            continue
+        job = "\n".join(line.split("#", 1)[0] for line in job_lines)
+        required = (
+            "always()",
+            "github.event_name == 'schedule'",
+            "contains(needs.*.result, 'failure')",
+            "issues: write",
+            "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+            "uses: ./.github/actions/schedule-failure-issue",
+            "github-token: ${{ secrets.GITHUB_TOKEN }}",
+        )
+        missing = [token for token in required if token not in job]
+        if missing:
+            errors.append(f"{relative}: alert-on-failure missing {', '.join(missing)}")
+        else:
+            errors.extend(
+                f"{relative}: {error}"
+                for error in alert_step_errors(job_lines, None, ("issues: write",), ("github-token: ${{ secrets.GITHUB_TOKEN }}",))
+            )
+
+    for (hour, minute), workflows in schedule_slots.items():
+        if len(workflows) > 1:
+            errors.append(
+                f"scheduled validations share {hour:02d}:{minute:02d} UTC: {', '.join(workflows)}"
+            )
+
+    watchdog_path = root / ".github/workflows/scheduled-validation-watchdog.yml"
+    try:
+        watchdog_lines = watchdog_path.read_text().splitlines()
+    except FileNotFoundError:
+        errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing completion watchdog")
+        return errors
+    watchdog_on = yaml_block(watchdog_lines, "on", 0)
+    watchdog_run = yaml_block(watchdog_on or [], "workflow_run", 2)
+    watchdog_workflows = yaml_block(watchdog_run or [], "workflows", 4)
+    if watchdog_workflows is None:
+        errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing workflow_run workflows")
+        return errors
+    watchdog_sources = "\n".join(line.split("#", 1)[0] for line in watchdog_workflows)
+    for relative in SCHEDULED_ALERT_WORKFLOWS:
+        path = root / relative
+        if not path.is_file():
+            continue
+        source = path.read_text()
+        match = re.search(r"^name:\s*[\"']?([^\"'\n]+)", source, re.MULTILINE)
+        if not match:
+            errors.append(f"{relative}: missing workflow name")
+        elif f'- "{match.group(1).strip()}"' not in watchdog_sources:
+            errors.append(f"{relative}: missing from scheduled completion watchdog")
+    watchdog_job_lines = yaml_block(watchdog_lines, "alert-on-incomplete-run", 2)
+    if watchdog_job_lines is None:
+        errors.append(".github/workflows/scheduled-validation-watchdog.yml: missing alert-on-incomplete-run job")
+        return errors
+    watchdog_job = "\n".join(line.split("#", 1)[0] for line in watchdog_job_lines)
+    required = (
+        "github.event.workflow_run.event == 'schedule'",
+        "github.event.workflow_run.conclusion != 'success'",
+        "github.event.workflow_run.conclusion != 'failure'",
+        "actions: read",
+        "issues: write",
+        "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+        "uses: ./.github/actions/schedule-failure-issue",
+        "github-token: ${{ secrets.GITHUB_TOKEN }}",
+        "workflow-name: ${{ github.event.workflow_run.name }}",
+        "source-run-id: ${{ github.event.workflow_run.id }}",
+        "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}",
+        "source-event: ${{ github.event.workflow_run.event }}",
+        "source-ref-name: ${{ github.event.workflow_run.head_branch }}",
+        "source-sha: ${{ github.event.workflow_run.head_sha }}",
+    )
+    missing = [token for token in required if token not in watchdog_job]
+    if missing:
+        errors.append(
+            ".github/workflows/scheduled-validation-watchdog.yml: missing " + ", ".join(missing)
+        )
+    else:
+        errors.extend(
+            ".github/workflows/scheduled-validation-watchdog.yml: " + error
+            for error in alert_step_errors(
+                watchdog_job_lines,
+                None,
+                ("actions: read", "issues: write"),
+                (
+                    "github-token: ${{ secrets.GITHUB_TOKEN }}",
+                    "workflow-name: ${{ github.event.workflow_run.name }}",
+                    "source-run-id: ${{ github.event.workflow_run.id }}",
+                    "source-run-attempt: ${{ github.event.workflow_run.run_attempt }}",
+                    "source-event: ${{ github.event.workflow_run.event }}",
+                    "source-ref-name: ${{ github.event.workflow_run.head_branch }}",
+                    "source-sha: ${{ github.event.workflow_run.head_sha }}",
+                ),
+            )
+        )
+
+    freshness_path = root / ".github/workflows/scheduled-validation-freshness.yml"
+    try:
+        freshness_lines = freshness_path.read_text().splitlines()
+    except FileNotFoundError:
+        errors.append(".github/workflows/scheduled-validation-freshness.yml: missing freshness check")
+        return errors
+    freshness_job_lines = yaml_block(freshness_lines, "check-freshness", 2)
+    if freshness_job_lines is None:
+        errors.append(".github/workflows/scheduled-validation-freshness.yml: missing check-freshness job")
+        return errors
+    freshness_job = "\n".join(line.split("#", 1)[0] for line in freshness_job_lines)
+    required = (
+        "python3 scripts/check_scheduled_validation_freshness.py",
+        "actions: read",
+        "issues: write",
+        "if: failure()",
+        "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+        "uses: ./.github/actions/schedule-failure-issue",
+        "github-token: ${{ secrets.GITHUB_TOKEN }}",
+        "details-file: ${{ runner.temp }}/scheduled-validation-freshness.md",
+    )
+    missing = [token for token in required if token not in freshness_job]
+    if missing:
+        errors.append(
+            ".github/workflows/scheduled-validation-freshness.yml: missing " + ", ".join(missing)
+        )
+    else:
+        errors.extend(
+            ".github/workflows/scheduled-validation-freshness.yml: " + error
+            for error in alert_step_errors(
+                freshness_job_lines,
+                "if: failure()",
+                ("actions: read", "issues: write"),
+                (
+                    "github-token: ${{ secrets.GITHUB_TOKEN }}",
+                    "details-file: ${{ runner.temp }}/scheduled-validation-freshness.md",
+                ),
+            )
+        )
+    if not (root / "scripts/check_scheduled_validation_freshness.py").is_file():
+        errors.append("scripts/check_scheduled_validation_freshness.py: missing freshness checker")
+    return errors
+
+
 def check_profile_listing(root: Path, profile: str, listing: Path) -> list[str]:
    try:
        expected_digest = profile_selection(root, profile)
@@ -281,6 +573,7 @@ def validate(root: Path) -> list[str]:
    errors.extend(check_runner_selection(root))
    errors.extend(check_s3_tests_runner(root))
    errors.extend(check_profile_definitions(root))
+    errors.extend(check_scheduled_alerts(root))
    return errors


@@ -363,6 +656,7 @@ class SelfTests(unittest.TestCase):
                mock.patch(__name__ + ".check_fuzz_targets", return_value=[]),
                mock.patch(__name__ + ".check_runner_selection", return_value=[]),
                mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
+                mock.patch(__name__ + ".check_scheduled_alerts", return_value=[]),
            ):
                self.assertEqual(len(validate(root)), 1)

@@ -413,6 +707,272 @@ class SelfTests(unittest.TestCase):
            with mock.patch.object(sys, "platform", "linux"):
                self.assertEqual(len(check_profile_listing(root, "e2e-full", listing)), 1)

+    def test_scheduled_alerts_require_completion_watchdog(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            alert = (
+                "  alert-on-failure:\n"
+                "    if: always() && github.event_name == 'schedule' && "
+                "contains(needs.*.result, 'failure')\n"
+                "    permissions:\n"
+                "      issues: write\n"
+                "    steps:\n"
+                "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                "      - uses: ./.github/actions/schedule-failure-issue\n"
+                "        with:\n"
+                "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+            )
+            names: list[str] = []
+            for index, relative in enumerate(SCHEDULED_ALERT_WORKFLOWS, start=1):
+                path = root / relative
+                path.parent.mkdir(parents=True, exist_ok=True)
+                names.append(path.stem)
+                path.write_text(
+                    f'name: "{path.stem}"\n'
+                    f'on:\n  schedule:\n    - cron: "{index} {index} * * *"\n'
+                    f'jobs:\n{alert}'
+                )
+            watchdog = root / ".github/workflows/scheduled-validation-watchdog.yml"
+            watchdog.write_text(
+                "on:\n  workflow_run:\n    workflows:\n"
+                + "\n".join(f'      - "{name}"' for name in names)
+                + "\njobs:\n"
+                + "  alert-on-incomplete-run:\n"
+                + "    github.event.workflow_run.event == 'schedule'\n"
+                + "    github.event.workflow_run.conclusion != 'success'\n"
+                + "    github.event.workflow_run.conclusion != 'failure'\n"
+                + "    permissions:\n"
+                + "      actions: read\n"
+                + "      issues: write\n"
+                + "    steps:\n"
+                + "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                + "      - uses: ./.github/actions/schedule-failure-issue\n"
+                + "        with:\n"
+                + "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+                + "          workflow-name: ${{ github.event.workflow_run.name }}\n"
+                + "          source-run-id: ${{ github.event.workflow_run.id }}\n"
+                + "          source-run-attempt: ${{ github.event.workflow_run.run_attempt }}\n"
+                + "          source-event: ${{ github.event.workflow_run.event }}\n"
+                + "          source-ref-name: ${{ github.event.workflow_run.head_branch }}\n"
+                + "          source-sha: ${{ github.event.workflow_run.head_sha }}\n"
+            )
+            freshness = root / ".github/workflows/scheduled-validation-freshness.yml"
+            freshness.write_text(
+                "jobs:\n"
+                "  check-freshness:\n"
+                "    permissions:\n"
+                "      actions: read\n"
+                "      issues: write\n"
+                "    steps:\n"
+                "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                "      - run: python3 scripts/check_scheduled_validation_freshness.py\n"
+                "      - uses: ./.github/actions/schedule-failure-issue\n"
+                "        if: failure()\n"
+                "        with:\n"
+                "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+                "          details-file: ${{ runner.temp }}/scheduled-validation-freshness.md\n"
+            )
+            checker = root / "scripts/check_scheduled_validation_freshness.py"
+            checker.parent.mkdir()
+            checker.write_text("")
+            self.assertEqual(check_scheduled_alerts(root), [])
+
+            first = root / SCHEDULED_ALERT_WORKFLOWS[0]
+            mutations = (
+                ("contains(needs.*.result, 'failure')", "false"),
+                ("issues: write", "issues: read"),
+                (
+                    "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+                    "uses: actions/checkout@missing",
+                ),
+                (
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n",
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                    "        if: github.event_name == 'workflow_dispatch'\n",
+                ),
+                (
+                    "      - uses: ./.github/actions/schedule-failure-issue\n",
+                    "      - uses: ./.github/actions/schedule-failure-issue\n"
+                    "        if: github.event_name == 'workflow_dispatch'\n",
+                ),
+                ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"),
+                ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"),
+            )
+            for required, replacement in mutations:
+                original = first.read_text()
+                first.write_text(original.replace(required, replacement))
+                self.assertEqual(len(check_scheduled_alerts(root)), 1)
+                first.write_text(original)
+
+            first_original = first.read_text()
+            real_steps = (
+                "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                "      - uses: ./.github/actions/schedule-failure-issue\n"
+                "        with:\n"
+                "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+            )
+            first.write_text(
+                first_original.replace(
+                    real_steps,
+                    "      - run: |\n"
+                    "          : <<'MARKER'\n"
+                    "          uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                    "          MARKER\n"
+                    "      - run: |\n"
+                    "          : <<'MARKER'\n"
+                    "          uses: ./.github/actions/schedule-failure-issue\n"
+                    "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+                    "          MARKER\n",
+                )
+            )
+            self.assertTrue(check_scheduled_alerts(root))
+            first.write_text(
+                first_original.replace(
+                    real_steps,
+                    "      - uses: ./.github/actions/schedule-failure-issue\n"
+                    "        with:\n"
+                    "          github-token: ${{ secrets.GITHUB_TOKEN }}\n"
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n",
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(first_original)
+
+            watchdog_mutations = (
+                ("actions: read", "actions: none"),
+                ("issues: write", "issues: read"),
+                (
+                    "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+                    "uses: actions/checkout@missing",
+                ),
+                (
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n",
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                    "        if: github.event_name == 'workflow_dispatch'\n",
+                ),
+                (
+                    "      - uses: ./.github/actions/schedule-failure-issue\n",
+                    "      - uses: ./.github/actions/schedule-failure-issue\n"
+                    "        if: github.event_name == 'workflow_dispatch'\n",
+                ),
+                ("uses: ./.github/actions/schedule-failure-issue", "uses: actions/checkout@v7"),
+                ("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing"),
+                ("source-event: ${{ github.event.workflow_run.event }}", "source-event: watchdog"),
+                (
+                    "source-ref-name: ${{ github.event.workflow_run.head_branch }}",
+                    "source-ref-name: main",
+                ),
+                ("source-sha: ${{ github.event.workflow_run.head_sha }}", "source-sha: missing"),
+            )
+            for required, replacement in watchdog_mutations:
+                original = watchdog.read_text()
+                watchdog.write_text(original.replace(required, replacement))
+                self.assertEqual(len(check_scheduled_alerts(root)), 1)
+                watchdog.write_text(original)
+
+            watchdog_original = watchdog.read_text()
+            watchdog.write_text(
+                watchdog_original.replace("issues: write", "issues: read")
+                + "  decoy:\n    permissions:\n      issues: write\n"
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            watchdog.write_text(watchdog_original)
+
+            first_original = first.read_text()
+            first.write_text(
+                first_original.replace('  schedule:\n    - cron: "1 1 * * *"\n', "")
+                + '  decoy:\n    strategy:\n      matrix:\n        cron:\n          - "1 1 * * *"\n'
+                + '    runs-on: ubuntu-latest\n    steps:\n      - run: true\n'
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(first_original)
+
+            first.write_text(
+                first_original.replace(
+                    '    - cron: "1 1 * * *"\n',
+                    '    - cron: "1 1 * * *"\n    - cron: "0 5 * * *"\n',
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(
+                first_original.replace(
+                    '    - cron: "1 1 * * *"\n',
+                    '    - cron: "1 1 * * *"\n    - cron: "2 2 * * *"\n',
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(first_original)
+
+            watchdog.write_text(
+                watchdog_original.replace(f'      - "{names[0]}"\n', "")
+                + f'  decoy:\n    strategy:\n      matrix:\n        workflow:\n          - "{names[0]}"\n'
+                + '    runs-on: ubuntu-latest\n    steps:\n      - run: true\n'
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            watchdog.write_text(watchdog_original)
+
+            watchdog.write_text(watchdog_original.replace(f'      - "{names[0]}"\n', ""))
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            watchdog.write_text(watchdog_original)
+            original = first.read_text()
+            first.write_text(re.sub(r'- cron: "\d+ \d+', '- cron: "0 0', original, count=1))
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(original)
+
+            second = root / SCHEDULED_ALERT_WORKFLOWS[1]
+            second_original = second.read_text()
+            second.write_text(re.sub(r'- cron: "\d+ \d+', '- cron: "1 1', second_original, count=1))
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            second.write_text(second_original)
+
+            first.write_text(
+                first_original.replace(
+                    '    - cron: "1 1 * * *"\n',
+                    '    - cron: "7 0 * * *"\n      timezone: "Asia/Shanghai"\n',
+                )
+            )
+            second.write_text(
+                second_original.replace(
+                    '    - cron: "2 2 * * *"\n',
+                    '    - cron: "2 2 * * *"\n    - cron: "7 16 * * *"\n',
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            first.write_text(first_original)
+            second.write_text(second_original)
+
+            freshness_original = freshness.read_text()
+            freshness.write_text(freshness_original.replace("details-file:", "report-file:"))
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            freshness.write_text(
+                freshness_original.replace("github-token: ${{ secrets.GITHUB_TOKEN }}", "github-token: missing")
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            freshness.write_text(
+                freshness_original.replace(
+                    "uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0",
+                    "uses: actions/checkout@missing",
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            freshness.write_text(
+                freshness_original.replace(
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n",
+                    "      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0\n"
+                    "        if: github.event_name == 'workflow_dispatch'\n",
+                )
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            freshness.write_text(
+                freshness_original.replace("if: failure()", "if: github.event_name == 'workflow_dispatch'")
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+            freshness.write_text(
+                freshness_original.replace("issues: write", "issues: read")
+                + "  decoy:\n    permissions:\n      issues: write\n"
+            )
+            self.assertEqual(len(check_scheduled_alerts(root)), 1)
+
 def main() -> int:
    if sys.argv[1:] == ["--self-test"]:
        suite = unittest.defaultTestLoader.loadTestsFromTestCase(SelfTests)
@@ -436,7 +996,7 @@ def main() -> int:
        for error in errors:
            print(f"ERROR: {error}", file=sys.stderr)
        return 1
-    print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired")
+    print("OK: e2e modules, runner selection, fuzz matrices, profiles, and scheduled alerts are wired")
    return 0