diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml index 453a87956..1ebaa47f5 100644 --- a/.github/workflows/rustfs-functional-chain.yml +++ b/.github/workflows/rustfs-functional-chain.yml @@ -252,11 +252,17 @@ jobs: env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | - if python3 scripts/check_functional_runners.py pf-testing; then - echo "performance_ready=online" >> "$GITHUB_OUTPUT" - else - echo "performance_ready=offline" >> "$GITHUB_OUTPUT" - fi + # TEMPORARY: the pf-testing environment is broken, so the + # performance lane is disabled until it is repaired. Forcing + # offline keeps the chain on its designed skip path: the lane job + # is skipped and aggregate gets --allow-skipped performance. + # Restore the probe below once the environment is healthy again. + # if python3 scripts/check_functional_runners.py pf-testing; then + # echo "performance_ready=online" >> "$GITHUB_OUTPUT" + # else + # echo "performance_ready=offline" >> "$GITHUB_OUTPUT" + # fi + echo "performance_ready=offline" >> "$GITHUB_OUTPUT" validation: if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }} @@ -784,6 +790,7 @@ jobs: run: >- python3 scripts/functional_chain_evidence.py summarize --directory "${RUNNER_TEMP}/chain-evidence" + --allow-skipped "${{ needs.prepare.outputs.performance_ready == 'online' && '' || 'performance' }}" --output "${RUNNER_TEMP}/chain-report.json" - name: Retain chain report regardless of test verdict if: always() diff --git a/scripts/functional_chain_evidence.py b/scripts/functional_chain_evidence.py index 77b096ca5..44d6819be 100644 --- a/scripts/functional_chain_evidence.py +++ b/scripts/functional_chain_evidence.py @@ -156,7 +156,7 @@ def record(chain, suite, report, output): raise error -def summarize(chain, directory, needs): +def summarize(chain, directory, needs, allow_skipped=()): """Retain failed/missing lanes without granting complete-success evidence.""" lanes = [] for suite in SUITES: @@ -174,10 +174,13 @@ def summarize(chain, directory, needs): "complete": False, "error": None, "completed_at": datetime.now(timezone.utc).isoformat()} try: require(chain is not None, "candidate preparation did not complete") - aggregate(chain, directory, {suite: value for suite, value in needs.items() if suite != "prepare"}) + aggregate(chain, directory, {suite: value for suite, value in needs.items() if suite != "prepare"}, allow_skipped) result["complete"] = True except (OSError, ValueError, KeyError, TypeError, AttributeError) as error: result["error"] = str(error) + skipped = sorted(name for name, value in needs.items() if value.get("result") == "skipped") + if skipped: + result["skipped_lanes"] = skipped return result @@ -185,6 +188,8 @@ def render_summary(result): lines = ["# RustFS functional chain report", "", "- All required suites passed with verified evidence: " + str(result["complete"]).lower(), "- Preparation: " + result["needs"].get("prepare", {}).get("result", "missing")] + if result.get("skipped_lanes"): + lines.append("- Preflight-skipped lanes (allowed by the chain driver): " + ", ".join(result["skipped_lanes"])) if result["chain"]: chain = result["chain"] manifest = chain["candidate"]["manifest"] @@ -244,7 +249,7 @@ def main(): args = parser.parse_args() if args.mode == "summarize": chain = current_chain() if os.environ.get("CHAIN_MANIFEST") else None - result = summarize(chain, args.directory, json.loads(os.environ["CHAIN_NEEDS"])) + result = summarize(chain, args.directory, json.loads(os.environ["CHAIN_NEEDS"]), args.allow_skipped.split(",")) args.output.write_text(json.dumps(result, sort_keys=True) + "\n") args.output.with_suffix(".md").write_text(render_summary(result)) if os.environ.get("GITHUB_STEP_SUMMARY"): diff --git a/scripts/functional_chain_health.py b/scripts/functional_chain_health.py index efcf54a05..5dc4305c3 100644 --- a/scripts/functional_chain_health.py +++ b/scripts/functional_chain_health.py @@ -9,7 +9,7 @@ import json from pathlib import Path import subprocess -from functional_chain_evidence import validate_records +from functional_chain_evidence import SUITES, validate_records from resolve_functional_candidate import REPOSITORY, api, read_json_artifact, require, resolve, sha WORKFLOW = "rustfs-functional-chain.yml" @@ -28,7 +28,15 @@ def validate_summary(summary, run): chain = summary["chain"] require(chain["run_id"] == run["id"] and chain["attempt"] == run["run_attempt"] and chain["workflow_sha"] == run["head_sha"], "complete evidence belongs to another run attempt") require(sha(chain["testing_sha"]), "missing test-script pin") - validate_records(chain, summary["suites"]) + # The chain skips the performance lane only when its preflight probe + # reports the pf-testing fleet offline (aggregate --allow-skipped), so a + # complete chain during such an outage carries 11 suites. Accept exactly + # that designed skip; any other missing lane stays invalid. + skipped = summary.get("skipped_lanes", []) + require(isinstance(skipped, list) and set(skipped) <= {"performance"}, + "complete evidence skipped a lane the chain driver never allows") + expected_suites = [suite for suite in SUITES if suite not in skipped] + validate_records(chain, summary["suites"], expected_suites) candidate = chain["candidate"] manifest = candidate["manifest"] require(resolve(manifest["build_run_id"], manifest["build_run_attempt"]) == candidate, "producer candidate identity changed") diff --git a/scripts/test_functional_chain.py b/scripts/test_functional_chain.py index bc145528c..45b62d4df 100644 --- a/scripts/test_functional_chain.py +++ b/scripts/test_functional_chain.py @@ -635,6 +635,22 @@ class EvidenceTests(unittest.TestCase): with self.assertRaises(ValueError): evidence.aggregate(self.chain, self.directory, self.needs) + def test_summarize_marks_a_preflight_allowed_skipped_lane(self): + needs = {**self.needs, "performance": {"result": "skipped"}} + (self.directory / "performance.json").unlink() + result = evidence.summarize(self.chain, self.directory, needs, ["performance"]) + self.assertTrue(result["complete"]) + self.assertEqual(result["skipped_lanes"], ["performance"]) + reportable = {**result, "chain": {**self.chain, "testing_sha": "d" * 40, + "candidate": {"workflow_sha": "b" * 40, + "manifest": {"build_run_id": 1, "build_run_attempt": 1, + "source_ref": "main", "source_sha": "a" * 40, + "package_sha256": "e" * 64}}}} + self.assertIn("Preflight-skipped lanes", evidence.render_summary(reportable)) + denied = evidence.summarize(self.chain, self.directory, needs) + self.assertFalse(denied["complete"]) + self.assertIn("without preflight permission", denied["error"]) + def test_partial_rerun_missing_artifact_and_zero_test_fail(self): path = self.directory / "s3.json" original = json.loads(path.read_text()) @@ -837,6 +853,10 @@ class EnvelopeTests(unittest.TestCase): self.assertIn("Probe performance fleet runner", prepare) self.assertIn("check_functional_runners.py pf-testing", prepare) self.assertIn("performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}", prepare) + # TEMPORARY: the pf-testing environment is broken and the probe is + # disabled in the workflow, so prepare must force the lane offline. + # Remove this assertion together with the probe restore. + self.assertIn('echo "performance_ready=offline" >> "$GITHUB_OUTPUT"', prepare) def test_every_lane_retains_failed_evidence_and_identifies_its_own_attempt(self): paths = list((candidate.ROOT / ".github/workflows").glob("rustfs-*-test.yml")) diff --git a/scripts/test_functional_chain_health.py b/scripts/test_functional_chain_health.py index b03aedf7a..8a752c060 100644 --- a/scripts/test_functional_chain_health.py +++ b/scripts/test_functional_chain_health.py @@ -36,6 +36,24 @@ class HealthTests(unittest.TestCase): self.assertEqual(result["workflow_sha"], "a" * 40) self.assertEqual(result["candidate"]["workflow_sha"], "c" * 40) + def test_complete_evidence_with_skipped_performance_is_accepted(self): + skipped = copy.deepcopy(self.summary) + skipped["skipped_lanes"] = ["performance"] + skipped["suites"] = [record for record in skipped["suites"] if record["suite"] != "performance"] + result = self.validate(skipped) + self.assertEqual(result["run_id"], 123) + + def test_complete_evidence_with_a_functional_lane_skipped_is_rejected(self): + skipped = copy.deepcopy(self.summary) + skipped["skipped_lanes"] = ["performance", "heal"] + skipped["suites"] = [record for record in skipped["suites"] if record["suite"] not in ("performance", "heal")] + with self.assertRaises(ValueError): + self.validate(skipped) + missing = copy.deepcopy(self.summary) + missing["suites"] = [record for record in missing["suites"] if record["suite"] != "heal"] + with self.assertRaises(ValueError): + self.validate(missing) + def test_substituted_producer_pin_attempt_or_empty_suite_fails(self): with self.assertRaises(ValueError): self.validate(candidate={**self.candidate, "workflow_sha": "e" * 40})