mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-01 00:32:21 +00:00
Chris An
4fcbf60f10
* ci: unify validation planning, artifacts and suite orchestration * fix(ci): preserve JSON diagnostics for release checks * fix(ci): keep manual attempts without plans visible * fix(ci): distinguish nextest setup scripts from test results * ci: reuse extracted archives and consolidate duplicate checks
245 lines
15 KiB
Python
245 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""Historical successes must not hide incomplete evidence or newer attempts."""
|
|
import base64
|
|
import copy
|
|
from datetime import datetime, timedelta, timezone
|
|
import json
|
|
import unittest
|
|
from unittest import mock
|
|
|
|
import functional_chain_health as health
|
|
from functional_chain_evidence import SUITES
|
|
|
|
|
|
class HealthTests(unittest.TestCase):
|
|
def setUp(self):
|
|
now = datetime.now(timezone.utc)
|
|
self.now = now.isoformat()
|
|
self.run = {"id": 123, "run_attempt": 2, "head_sha": "a" * 40, "html_url": "https://github.com/rustfs/rustfs/actions/runs/123",
|
|
"run_started_at": (now - timedelta(hours=1)).isoformat(), "event": "workflow_run"}
|
|
self.candidate = {"manifest": {"build_run_id": 456, "build_run_attempt": 3, "source_ref": "release", "source_sha": "b" * 40},
|
|
"workflow_sha": "c" * 40, "workflow_ref": "main", "build_started_at": (now - timedelta(hours=2)).isoformat()}
|
|
self.chain = {"run_id": 123, "attempt": 2, "workflow_sha": "a" * 40, "testing_sha": "d" * 40, "candidate": self.candidate}
|
|
self.summary = {"schema": 1, "complete": True, "chain": self.chain, "completed_at": (now - timedelta(minutes=1)).isoformat(),
|
|
"suites": [{"schema": 1, "suite": suite, "chain": self.chain, "valid": True, "report_sha256": "f" * 64,
|
|
"counts": {"PASS": 1, "FAIL": 0, "SKIP": 0, "UNSUPPORTED": 0, "RUNNING": 0}} for suite in SUITES]}
|
|
self.config = {"content": base64.b64encode(("d" * 40 + "\n").encode()).decode()}
|
|
|
|
def validate(self, summary=None, candidate=None, config=None):
|
|
with mock.patch.object(health, "resolve", return_value=candidate or self.candidate), mock.patch.object(health, "api", return_value=config or self.config):
|
|
return health.validate_summary(summary or self.summary, self.run)
|
|
|
|
def test_release_success_preserves_both_sources(self):
|
|
result = self.validate()
|
|
self.assertEqual(result["source_ref"], "release")
|
|
self.assertEqual(result["source_sha"], "b" * 40)
|
|
self.assertEqual(result["workflow_sha"], "a" * 40)
|
|
self.assertEqual(result["candidate"]["workflow_sha"], "c" * 40)
|
|
|
|
def test_substituted_producer_pin_attempt_or_empty_suite_fails(self):
|
|
with self.assertRaises(ValueError):
|
|
self.validate(candidate={**self.candidate, "workflow_sha": "e" * 40})
|
|
# The prepare step's >24h staleness fallback legitimately sets
|
|
# testing_sha to auto-testing main HEAD, so the pin FILE is no longer
|
|
# consulted at all; the sha FORMAT of the chain's testing_sha is the
|
|
# remaining invariant.
|
|
wrong = copy.deepcopy(self.summary)
|
|
wrong["chain"]["testing_sha"] = "short"
|
|
with self.assertRaises(ValueError):
|
|
self.validate(wrong)
|
|
wrong = copy.deepcopy(self.summary)
|
|
wrong["chain"]["attempt"] = 1
|
|
with self.assertRaises(ValueError):
|
|
self.validate(wrong)
|
|
wrong = copy.deepcopy(self.summary)
|
|
wrong["suites"][0]["counts"]["PASS"] = 0
|
|
with self.assertRaises(ValueError):
|
|
self.validate(wrong)
|
|
|
|
def test_new_failure_retains_last_complete_success_without_becoming_healthy(self):
|
|
complete = self.validate()
|
|
previous = {"schema": 1, "observed_at": self.now, "last_complete_success": {"release": complete}}
|
|
current = {"schema": 1, "observed_at": self.now, "last_complete_success": {}, "healthy": False,
|
|
"latest_attempt": {"conclusion": "failure"}}
|
|
result = health.merge_history(current, previous)
|
|
self.assertFalse(result["healthy"])
|
|
self.assertEqual(result["latest_attempt"]["conclusion"], "failure")
|
|
self.assertEqual(result["last_complete_success"]["release"]["source_sha"], "b" * 40)
|
|
self.assertTrue(result["last_complete_success"]["release"]["retained_history"])
|
|
|
|
def test_expired_history_is_not_fresh_and_null_or_stale_state_cannot_publish(self):
|
|
complete = self.validate()
|
|
complete["expires_at"] = "2000-01-01T00:00:00Z"
|
|
previous = {"schema": 1, "observed_at": self.now, "last_complete_success": {"release": complete}}
|
|
current = {"schema": 1, "observed_at": self.now, "last_complete_success": {}, "healthy": False}
|
|
self.assertFalse(health.merge_history(current, previous)["last_complete_success"]["release"]["fresh"])
|
|
with self.assertRaises(ValueError):
|
|
health.merge_history(current, None)
|
|
with self.assertRaises(ValueError):
|
|
health.merge_history({**current, "observed_at": "2000-01-01T00:00:00Z"}, previous)
|
|
existing = {"sha": "old-blob", "content": base64.b64encode(b"null").decode()}
|
|
with mock.patch.object(health, "api", return_value=existing), mock.patch.object(health.subprocess, "run") as write:
|
|
with self.assertRaises(ValueError):
|
|
health.publish(current)
|
|
write.assert_not_called()
|
|
|
|
def test_collection_rejects_a_concurrent_rerun(self):
|
|
run = {**self.run, "status": "completed", "conclusion": "failure"}
|
|
responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, {"workflow_runs": [run], "total_count": 1}, run,
|
|
{"workflow_runs": [{**run, "run_attempt": 3, "status": "queued"}], "total_count": 1}]
|
|
with mock.patch.object(health, "api", side_effect=responses):
|
|
with self.assertRaisesRegex(ValueError, "changed during inspection"):
|
|
health.collect()
|
|
|
|
def test_health_follows_configured_build_source_not_workflow_main(self):
|
|
run = {**self.run, "status": "completed", "conclusion": "success"}
|
|
complete = self.validate()
|
|
for expected, healthy in (("release", True), ("refs/heads/release", True), ("main", False)):
|
|
responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, {"workflow_runs": [run], "total_count": 1}, run,
|
|
{"workflow_runs": [run], "total_count": 1}, {"object": {"sha": "b" * 40}}]
|
|
with mock.patch.object(health, "api", side_effect=responses), mock.patch.object(health, "complete_success", return_value=complete):
|
|
result = health.collect(source_ref=expected)
|
|
self.assertEqual(result["healthy"], healthy)
|
|
|
|
def test_same_source_new_execution_refreshes_completion_even_for_old_build(self):
|
|
self.candidate["build_started_at"] = "2000-01-01T00:00:00Z"
|
|
complete = self.validate()
|
|
self.assertGreater(health.timestamp(complete["expires_at"]), datetime.now(timezone.utc))
|
|
run = {**self.run, "status": "completed", "conclusion": "success"}
|
|
payload = {"workflow_runs": [run], "total_count": 1}
|
|
responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, payload, run, payload,
|
|
{"object": {"sha": "b" * 40}}]
|
|
with mock.patch.object(health, "api", side_effect=responses), mock.patch.object(health, "complete_success", return_value=complete):
|
|
self.assertTrue(health.collect(source_ref="release")["healthy"])
|
|
wrong = copy.deepcopy(self.summary)
|
|
wrong["chain"]["attempt"] = 1
|
|
with self.assertRaises(ValueError):
|
|
self.validate(wrong)
|
|
|
|
def test_success_on_stale_sha_or_incomplete_listing_never_becomes_healthy(self):
|
|
run = {**self.run, "status": "completed", "conclusion": "success"}
|
|
payload = {"workflow_runs": [run], "total_count": 1}
|
|
complete = self.validate()
|
|
responses = [{"state": "active"}, {"object": {"sha": "c" * 40}}, payload, run, payload,
|
|
{"object": {"sha": "c" * 40}}]
|
|
with mock.patch.object(health, "api", side_effect=responses), mock.patch.object(health, "complete_success", return_value=complete):
|
|
result = health.collect(source_ref="release")
|
|
self.assertFalse(result["healthy"])
|
|
self.assertFalse(result["last_complete_success"]["release"]["fresh"])
|
|
with mock.patch.object(health, "api", side_effect=[{"workflow_runs": [run], "total_count": 2},
|
|
{"workflow_runs": [], "total_count": 2}]):
|
|
with self.assertRaisesRegex(ValueError, "incomplete chain run page"):
|
|
health.list_runs(1)
|
|
|
|
def test_optional_omission_is_explicit_and_partial_manual_is_not_full_health(self):
|
|
from validation_registry import load_registry, select_plan
|
|
value = copy.deepcopy(self.summary)
|
|
value["plan"] = select_plan(load_registry())
|
|
value["suites"] = value["suites"][:-1]
|
|
value["skipped_lanes"] = ["performance"]
|
|
value["skip_reasons"] = {"performance": "optional fleet unavailable during preflight"}
|
|
self.assertEqual(self.validate(value)["skipped_lanes"], ["performance"])
|
|
del value["skip_reasons"]
|
|
with self.assertRaises(ValueError):
|
|
self.validate(value)
|
|
value = copy.deepcopy(self.summary)
|
|
value["plan"] = select_plan(load_registry(), requested="s3")
|
|
value["suites"] = [value["suites"][1]]
|
|
with self.assertRaisesRegex(ValueError, "partial"):
|
|
self.validate(value)
|
|
|
|
def test_new_failed_attempt_does_not_hide_behind_recent_complete_success(self):
|
|
old = {**self.run, "status": "completed", "conclusion": "success"}
|
|
new = {**old, "id": 124, "run_started_at": self.now, "conclusion": "cancelled"}
|
|
payload = {"workflow_runs": [new, old], "total_count": 2}
|
|
responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, payload, new, old, payload,
|
|
{"object": {"sha": "b" * 40}}]
|
|
with mock.patch.object(health, "api", side_effect=responses), mock.patch.object(health, "complete_success", return_value=self.validate()):
|
|
result = health.collect(source_ref="release")
|
|
self.assertFalse(result["healthy"])
|
|
self.assertEqual(result["latest_attempt"]["conclusion"], "cancelled")
|
|
self.assertTrue(result["last_complete_success"]["release"]["fresh"])
|
|
|
|
def test_manual_attempt_without_plan_cannot_hide_behind_recent_success(self):
|
|
old = {**self.run, "status": "completed", "conclusion": "success"}
|
|
complete = self.validate()
|
|
for status, conclusion in (("completed", "failure"), ("completed", "cancelled"), ("in_progress", None), ("completed", "success")):
|
|
with self.subTest(status=status, conclusion=conclusion):
|
|
new = {**old, "id": 124, "run_started_at": self.now, "event": "workflow_dispatch",
|
|
"status": status, "conclusion": conclusion}
|
|
payload = {"workflow_runs": [new, old], "total_count": 2}
|
|
responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, payload, new,
|
|
{"total_count": 0, "artifacts": []}, old, payload, {"object": {"sha": "b" * 40}}]
|
|
|
|
def completed(run):
|
|
if run["id"] == new["id"]:
|
|
raise ValueError("missing complete-chain artifact")
|
|
return complete
|
|
|
|
with mock.patch.object(health, "api", side_effect=responses), \
|
|
mock.patch.object(health, "complete_success", side_effect=completed):
|
|
result = health.collect(source_ref="release")
|
|
self.assertFalse(result["healthy"])
|
|
self.assertEqual(result["latest_attempt"]["run_id"], 124)
|
|
self.assertEqual(result["latest_attempt"]["conclusion"], conclusion)
|
|
self.assertNotEqual(result["latest_attempt"]["verification"], "complete")
|
|
self.assertTrue(result["last_complete_success"]["release"]["fresh"])
|
|
|
|
|
|
def test_manual_full_plan_failure_is_latest_but_verified_subset_is_not(self):
|
|
import hashlib
|
|
import io
|
|
import zipfile
|
|
from validation_registry import load_registry, select_plan
|
|
run = {**self.run, "event": "workflow_dispatch"}
|
|
name = "functional-plan-123-2"
|
|
for suites, expected in (("", True), ("s3", False)):
|
|
buffer = io.BytesIO()
|
|
with zipfile.ZipFile(buffer, "w") as archive:
|
|
archive.writestr("suite-plan.json", json.dumps(select_plan(load_registry(), requested=suites)))
|
|
raw = buffer.getvalue()
|
|
artifact = {"id": 77, "name": name, "expired": False, "size_in_bytes": len(raw),
|
|
"digest": "sha256:" + hashlib.sha256(raw).hexdigest(),
|
|
"workflow_run": {"id": 123, "head_sha": "a" * 40}}
|
|
with mock.patch.object(health, "api", side_effect=[{"total_count": 1, "artifacts": [artifact]}, raw]):
|
|
self.assertEqual(health.expected_attempt(run), expected)
|
|
with mock.patch.object(health, "api", return_value={"total_count": 0, "artifacts": []}):
|
|
self.assertTrue(health.expected_attempt(run))
|
|
new = {**run, "status": "completed", "conclusion": "failure", "id": 124, "run_started_at": self.now}
|
|
old = {**self.run, "status": "completed", "conclusion": "success"}
|
|
payload = {"workflow_runs": [new, old], "total_count": 2}
|
|
complete = self.validate()
|
|
responses = [{"state": "active"}, {"object": {"sha": "b" * 40}}, payload, new, old, payload,
|
|
{"object": {"sha": "b" * 40}}]
|
|
with mock.patch.object(health, "api", side_effect=responses), \
|
|
mock.patch.object(health, "expected_attempt", return_value=True), \
|
|
mock.patch.object(health, "complete_success", return_value=complete):
|
|
result = health.collect(source_ref="release")
|
|
self.assertFalse(result["healthy"])
|
|
self.assertEqual(result["latest_attempt"]["run_id"], 124)
|
|
|
|
def test_artifact_pagination_must_be_complete_and_unique(self):
|
|
first = [{"id": index} for index in range(100)]
|
|
final = [{"id": 100}]
|
|
with mock.patch.object(health, "api", side_effect=[{"total_count": 101, "artifacts": first},
|
|
{"total_count": 101, "artifacts": final}]):
|
|
self.assertEqual(len(health.run_artifacts(1)), 101)
|
|
for final in ([], [{"id": 0}]):
|
|
with mock.patch.object(health, "api", side_effect=[{"total_count": 101, "artifacts": first},
|
|
{"total_count": 101, "artifacts": final}]):
|
|
with self.assertRaises(ValueError):
|
|
health.run_artifacts(1)
|
|
|
|
def test_publication_uses_the_read_blob_sha(self):
|
|
current = {"schema": 1, "observed_at": self.now, "last_complete_success": {}, "healthy": False}
|
|
existing = {"sha": "reviewed-blob", "content": base64.b64encode(json.dumps(current).encode()).decode()}
|
|
with mock.patch.object(health, "api", return_value=existing), mock.patch.object(health.subprocess, "run") as write:
|
|
health.publish(current)
|
|
body = json.loads(write.call_args.kwargs["input"])
|
|
self.assertEqual(body["sha"], "reviewed-blob")
|
|
self.assertFalse(json.loads(base64.b64decode(body["content"]))["healthy"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|