ci: stop daily freshness false alarms for dormant scheduled workflows (#6608)

.github/scheduled-validations.json

@@ -7,8 +7,11 @@
   { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
   { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
   { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
-  { "workflow": ".github/workflows/minio-interop.yml", "max_age_hours": 36 },
   { "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
   { "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
-  { "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792 }
+  {
+    "workflow": ".github/workflows/runner-hygiene.yml",
+    "max_age_hours": 792,
+    "never_ran_grace_until": "2026-09-02T06:37:00Z"
+  }
 ]

.github/workflows/minio-interop.yml

@@ -37,6 +37,11 @@
 # banner. Re-enabling is a UI action; anyone doing so should first check that the
 # workflow still matches the current CI layout. See rustfs/backlog#1603.
 #
+# While disabled, this workflow is deliberately absent from
+# .github/scheduled-validations.json — a disabled workflow can never satisfy the
+# freshness check. Whoever re-enables it must re-add the entry in the same
+# change so the freshness gate covers it again.
+#
 name: minio-interop
 
 on:

docs/testing/ci-gates.md

@@ -85,12 +85,12 @@ schedules.
 | Cadence (UTC unless noted) | Workflow / validation | Budget | Verdict and artifacts | Reproduction |
 |---|---|---:|---|---|
 | Daily 02:17 | Fuzz: five nightly corpus targets | 60 min build; 60 min per target | Gate; corpus/crash artifacts, scheduled failure alert | `MAX_TOTAL_TIME=<seconds> ./scripts/fuzz/run.sh` |
-| Daily 03:17 | MinIO interop (EC + SSE read parity) | 40 min | Gate; scheduled failure alert | Dispatch `minio-interop.yml` or follow its pinned Docker fixture steps |
+| Dormant (cron 03:17 once re-enabled) | MinIO interop (EC + SSE read parity) | 40 min | Manually disabled in the Actions settings (backlog#1603) and therefore outside the freshness list; re-add it to `.github/scheduled-validations.json` when re-enabling | Follow the pinned Docker fixture steps in `minio-interop.yml` |
 | Daily 04:29 | Replication / cluster-fault / protocol e2e | 45 / 90 / 90 min | Three independent gates; JUnit, membership, and server logs | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` |
 | Daily 06:31 | Warp performance A/B | 180 min | Regression budget gate; A/B summaries and server logs | `bash scripts/run_hotpath_warp_abba.sh --help` |
 | Daily 00:07 Asia/Shanghai (16:07 UTC previous day) | Nightly GNU build and Vault lanes | 150 / 90 / 60 min | Build, live Vault, and HA failover gates | Use the commands and pinned Vault images in `nightly-gnu.yml` |
 | Daily 03:23 | Security Audit | 20 / 5 min, plus 30 min on PR dependency review | Cargo Deny and workflow-pin gates; scheduled failure alert | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |
-| Daily 23:47 | Scheduled Validation Freshness | 10 min | Fails when a critical schedule was never created or is stale | Dispatch `scheduled-validation-freshness.yml` |
+| Daily 23:47 | Scheduled Validation Freshness | 10 min | Fails when a critical schedule was never created or is stale; an entry may carry `never_ran_grace_until` to cover the window before a newly enabled cron's first slot | Dispatch `scheduled-validation-freshness.yml` |
 | Sunday 00:11 | Full `Continuous Integration` matrix | Per-job budgets above | Weekly variant coverage, including dormant rio-v2 binary/e2e lanes | Dispatch `ci.yml` |
 | Sunday 01:13 | Seven-platform build matrix | 150 min per platform | Build/package integrity; scheduled failure alert | Dispatch `build.yml` with an exact platform set |
 | Sunday 02:19 | Ceph s3-tests full sweep: single and real four-node, four shards each | 180 min per shard | Compatibility gate; report, JUnit, exact node IDs, and server logs | `scripts/s3-tests/run.sh` against an existing single or distributed target |

scripts/check_scheduled_validation_freshness.py

@@ -20,12 +20,12 @@ from urllib.request import Request, urlopen
 ROOT = Path(__file__).resolve().parents[1]
 
 
-def load_validations(path: Path) -> list[tuple[str, int]]:
+def load_validations(path: Path) -> list[tuple[str, int, datetime | None]]:
     data = json.loads(path.read_text())
     if not isinstance(data, list) or not data:
         raise ValueError("scheduled validation config must be a non-empty list")
 
-    validations: list[tuple[str, int]] = []
+    validations: list[tuple[str, int, datetime | None]] = []
     seen: set[str] = set()
     for item in data:
         if not isinstance(item, dict):
@@ -44,8 +44,21 @@ def load_validations(path: Path) -> list[tuple[str, int]]:
             or max_age_hours <= 0
         ):
             raise ValueError(f"invalid max_age_hours for {workflow}: {max_age_hours!r}")
+        grace_raw = item.get("never_ran_grace_until")
+        grace: datetime | None = None
+        if grace_raw is not None:
+            if not isinstance(grace_raw, str):
+                raise ValueError(
+                    f"invalid never_ran_grace_until for {workflow}: {grace_raw!r}"
+                )
+            try:
+                grace = parse_timestamp(grace_raw)
+            except ValueError as error:
+                raise ValueError(
+                    f"invalid never_ran_grace_until for {workflow}: {error}"
+                ) from error
         seen.add(workflow)
-        validations.append((workflow, max_age_hours))
+        validations.append((workflow, max_age_hours, grace))
     return validations
 
 
@@ -59,9 +72,18 @@ def parse_timestamp(value: object) -> datetime:
 
 
 def stale_reason(
-    run: dict[str, object] | None, now: datetime, max_age_hours: int
+    run: dict[str, object] | None,
+    now: datetime,
+    max_age_hours: int,
+    never_ran_grace_until: datetime | None = None,
 ) -> str | None:
     if run is None:
+        # The grace deadline only covers a workflow whose first scheduled slot
+        # has not arrived yet (for example a monthly cron enabled mid-month).
+        # A recorded-but-old run proves the schedule used to fire and stopped,
+        # so the grace never masks that case.
+        if never_ran_grace_until is not None and now <= never_ran_grace_until:
+            return None
         return "no scheduled run has been recorded"
     created_at = parse_timestamp(run.get("created_at"))
     age = now - created_at
@@ -126,10 +148,10 @@ def check_freshness(
 ) -> int:
     now = datetime.now(timezone.utc)
     failures: list[tuple[str, int, str, str]] = []
-    for workflow, max_age_hours in load_validations(config):
+    for workflow, max_age_hours, never_ran_grace_until in load_validations(config):
         try:
             run = fetch_latest_scheduled_run(repository, workflow, token, api_url)
-            reason = stale_reason(run, now, max_age_hours)
+            reason = stale_reason(run, now, max_age_hours, never_ran_grace_until)
             if reason is not None:
                 run_url = str(run.get("html_url", "")) if run else ""
                 failures.append((workflow, max_age_hours, reason, run_url))
@@ -151,6 +173,15 @@ class SelfTests(unittest.TestCase):
         self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36))
         self.assertIsNotNone(stale_reason(None, self.NOW, 36))
 
+    def test_never_ran_grace_only_covers_missing_runs(self) -> None:
+        future_grace = self.NOW + timedelta(hours=1)
+        past_grace = self.NOW - timedelta(seconds=1)
+        self.assertIsNone(stale_reason(None, self.NOW, 36, future_grace))
+        self.assertIsNone(stale_reason(None, self.NOW, 36, self.NOW))
+        self.assertIsNotNone(stale_reason(None, self.NOW, 36, past_grace))
+        stale_run = {"created_at": "2026-08-20T23:59:59Z"}
+        self.assertIsNotNone(stale_reason(stale_run, self.NOW, 36, future_grace))
+
     def test_config_rejects_duplicate_and_invalid_entries(self) -> None:
         with tempfile.TemporaryDirectory() as tmp:
             path = Path(tmp) / "validations.json"
@@ -171,6 +202,41 @@ class SelfTests(unittest.TestCase):
             )
             with self.assertRaises(ValueError):
                 load_validations(path)
+            for bad_grace in (36, "not-a-timestamp", "2026-09-01T00:00:00"):
+                path.write_text(
+                    json.dumps(
+                        [
+                            {
+                                "workflow": ".github/workflows/ci.yml",
+                                "max_age_hours": 36,
+                                "never_ran_grace_until": bad_grace,
+                            }
+                        ]
+                    )
+                )
+                with self.assertRaises(ValueError):
+                    load_validations(path)
+            path.write_text(
+                json.dumps(
+                    [
+                        {
+                            "workflow": ".github/workflows/ci.yml",
+                            "max_age_hours": 36,
+                            "never_ran_grace_until": "2026-09-02T06:37:00Z",
+                        }
+                    ]
+                )
+            )
+            self.assertEqual(
+                load_validations(path),
+                [
+                    (
+                        ".github/workflows/ci.yml",
+                        36,
+                        datetime(2026, 9, 2, 6, 37, tzinfo=timezone.utc),
+                    )
+                ],
+            )
 
     def test_check_reports_missing_runs(self) -> None:
         with tempfile.TemporaryDirectory() as tmp: