diff --git a/.github/scheduled-validations.json b/.github/scheduled-validations.json index aa38b6c6c..f9a66f3d5 100644 --- a/.github/scheduled-validations.json +++ b/.github/scheduled-validations.json @@ -7,8 +7,11 @@ { "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 }, { "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 }, { "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 }, - { "workflow": ".github/workflows/minio-interop.yml", "max_age_hours": 36 }, { "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 }, { "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 }, - { "workflow": ".github/workflows/runner-hygiene.yml", "max_age_hours": 792 } + { + "workflow": ".github/workflows/runner-hygiene.yml", + "max_age_hours": 792, + "never_ran_grace_until": "2026-09-02T06:37:00Z" + } ] diff --git a/.github/workflows/minio-interop.yml b/.github/workflows/minio-interop.yml index 5f105ac41..8e1ebb7cd 100644 --- a/.github/workflows/minio-interop.yml +++ b/.github/workflows/minio-interop.yml @@ -37,6 +37,11 @@ # banner. Re-enabling is a UI action; anyone doing so should first check that the # workflow still matches the current CI layout. See rustfs/backlog#1603. # +# While disabled, this workflow is deliberately absent from +# .github/scheduled-validations.json — a disabled workflow can never satisfy the +# freshness check. Whoever re-enables it must re-add the entry in the same +# change so the freshness gate covers it again. +# name: minio-interop on: diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index d87ce1668..75b5ed02c 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -85,12 +85,12 @@ schedules. | Cadence (UTC unless noted) | Workflow / validation | Budget | Verdict and artifacts | Reproduction | |---|---|---:|---|---| | Daily 02:17 | Fuzz: five nightly corpus targets | 60 min build; 60 min per target | Gate; corpus/crash artifacts, scheduled failure alert | `MAX_TOTAL_TIME= ./scripts/fuzz/run.sh` | -| Daily 03:17 | MinIO interop (EC + SSE read parity) | 40 min | Gate; scheduled failure alert | Dispatch `minio-interop.yml` or follow its pinned Docker fixture steps | +| Dormant (cron 03:17 once re-enabled) | MinIO interop (EC + SSE read parity) | 40 min | Manually disabled in the Actions settings (backlog#1603) and therefore outside the freshness list; re-add it to `.github/scheduled-validations.json` when re-enabling | Follow the pinned Docker fixture steps in `minio-interop.yml` | | Daily 04:29 | Replication / cluster-fault / protocol e2e | 45 / 90 / 90 min | Three independent gates; JUnit, membership, and server logs | `cargo nextest run --profile e2e-repl-nightly -p e2e_test`; `--profile e2e-nightly`; `-j 1 --profile e2e-protocols` | | Daily 06:31 | Warp performance A/B | 180 min | Regression budget gate; A/B summaries and server logs | `bash scripts/run_hotpath_warp_abba.sh --help` | | Daily 00:07 Asia/Shanghai (16:07 UTC previous day) | Nightly GNU build and Vault lanes | 150 / 90 / 60 min | Build, live Vault, and HA failover gates | Use the commands and pinned Vault images in `nightly-gnu.yml` | | Daily 03:23 | Security Audit | 20 / 5 min, plus 30 min on PR dependency review | Cargo Deny and workflow-pin gates; scheduled failure alert | `cargo deny check`; `scripts/security/check_workflow_pins.sh` | -| Daily 23:47 | Scheduled Validation Freshness | 10 min | Fails when a critical schedule was never created or is stale | Dispatch `scheduled-validation-freshness.yml` | +| Daily 23:47 | Scheduled Validation Freshness | 10 min | Fails when a critical schedule was never created or is stale; an entry may carry `never_ran_grace_until` to cover the window before a newly enabled cron's first slot | Dispatch `scheduled-validation-freshness.yml` | | Sunday 00:11 | Full `Continuous Integration` matrix | Per-job budgets above | Weekly variant coverage, including dormant rio-v2 binary/e2e lanes | Dispatch `ci.yml` | | Sunday 01:13 | Seven-platform build matrix | 150 min per platform | Build/package integrity; scheduled failure alert | Dispatch `build.yml` with an exact platform set | | Sunday 02:19 | Ceph s3-tests full sweep: single and real four-node, four shards each | 180 min per shard | Compatibility gate; report, JUnit, exact node IDs, and server logs | `scripts/s3-tests/run.sh` against an existing single or distributed target | diff --git a/scripts/check_scheduled_validation_freshness.py b/scripts/check_scheduled_validation_freshness.py index d9bdf0597..a5812c7a3 100644 --- a/scripts/check_scheduled_validation_freshness.py +++ b/scripts/check_scheduled_validation_freshness.py @@ -20,12 +20,12 @@ from urllib.request import Request, urlopen ROOT = Path(__file__).resolve().parents[1] -def load_validations(path: Path) -> list[tuple[str, int]]: +def load_validations(path: Path) -> list[tuple[str, int, datetime | None]]: data = json.loads(path.read_text()) if not isinstance(data, list) or not data: raise ValueError("scheduled validation config must be a non-empty list") - validations: list[tuple[str, int]] = [] + validations: list[tuple[str, int, datetime | None]] = [] seen: set[str] = set() for item in data: if not isinstance(item, dict): @@ -44,8 +44,21 @@ def load_validations(path: Path) -> list[tuple[str, int]]: or max_age_hours <= 0 ): raise ValueError(f"invalid max_age_hours for {workflow}: {max_age_hours!r}") + grace_raw = item.get("never_ran_grace_until") + grace: datetime | None = None + if grace_raw is not None: + if not isinstance(grace_raw, str): + raise ValueError( + f"invalid never_ran_grace_until for {workflow}: {grace_raw!r}" + ) + try: + grace = parse_timestamp(grace_raw) + except ValueError as error: + raise ValueError( + f"invalid never_ran_grace_until for {workflow}: {error}" + ) from error seen.add(workflow) - validations.append((workflow, max_age_hours)) + validations.append((workflow, max_age_hours, grace)) return validations @@ -59,9 +72,18 @@ def parse_timestamp(value: object) -> datetime: def stale_reason( - run: dict[str, object] | None, now: datetime, max_age_hours: int + run: dict[str, object] | None, + now: datetime, + max_age_hours: int, + never_ran_grace_until: datetime | None = None, ) -> str | None: if run is None: + # The grace deadline only covers a workflow whose first scheduled slot + # has not arrived yet (for example a monthly cron enabled mid-month). + # A recorded-but-old run proves the schedule used to fire and stopped, + # so the grace never masks that case. + if never_ran_grace_until is not None and now <= never_ran_grace_until: + return None return "no scheduled run has been recorded" created_at = parse_timestamp(run.get("created_at")) age = now - created_at @@ -126,10 +148,10 @@ def check_freshness( ) -> int: now = datetime.now(timezone.utc) failures: list[tuple[str, int, str, str]] = [] - for workflow, max_age_hours in load_validations(config): + for workflow, max_age_hours, never_ran_grace_until in load_validations(config): try: run = fetch_latest_scheduled_run(repository, workflow, token, api_url) - reason = stale_reason(run, now, max_age_hours) + reason = stale_reason(run, now, max_age_hours, never_ran_grace_until) if reason is not None: run_url = str(run.get("html_url", "")) if run else "" failures.append((workflow, max_age_hours, reason, run_url)) @@ -151,6 +173,15 @@ class SelfTests(unittest.TestCase): self.assertIsNotNone(stale_reason(past_limit, self.NOW, 36)) self.assertIsNotNone(stale_reason(None, self.NOW, 36)) + def test_never_ran_grace_only_covers_missing_runs(self) -> None: + future_grace = self.NOW + timedelta(hours=1) + past_grace = self.NOW - timedelta(seconds=1) + self.assertIsNone(stale_reason(None, self.NOW, 36, future_grace)) + self.assertIsNone(stale_reason(None, self.NOW, 36, self.NOW)) + self.assertIsNotNone(stale_reason(None, self.NOW, 36, past_grace)) + stale_run = {"created_at": "2026-08-20T23:59:59Z"} + self.assertIsNotNone(stale_reason(stale_run, self.NOW, 36, future_grace)) + def test_config_rejects_duplicate_and_invalid_entries(self) -> None: with tempfile.TemporaryDirectory() as tmp: path = Path(tmp) / "validations.json" @@ -171,6 +202,41 @@ class SelfTests(unittest.TestCase): ) with self.assertRaises(ValueError): load_validations(path) + for bad_grace in (36, "not-a-timestamp", "2026-09-01T00:00:00"): + path.write_text( + json.dumps( + [ + { + "workflow": ".github/workflows/ci.yml", + "max_age_hours": 36, + "never_ran_grace_until": bad_grace, + } + ] + ) + ) + with self.assertRaises(ValueError): + load_validations(path) + path.write_text( + json.dumps( + [ + { + "workflow": ".github/workflows/ci.yml", + "max_age_hours": 36, + "never_ran_grace_until": "2026-09-02T06:37:00Z", + } + ] + ) + ) + self.assertEqual( + load_validations(path), + [ + ( + ".github/workflows/ci.yml", + 36, + datetime(2026, 9, 2, 6, 37, tzinfo=timezone.utc), + ) + ], + ) def test_check_reports_missing_runs(self) -> None: with tempfile.TemporaryDirectory() as tmp: