ci(chain): disable the performance lane while pf-testing is broken

The pf-testing environment is temporarily broken, so the nightly chain
would end red in the performance lane. Take the chain through its
designed skip path instead of deleting the lane:

- prepare: comment out the pf-testing probe and force
  performance_ready=offline, so the lane job is skipped and the
  aggregate step passes --allow-skipped performance (marked TEMPORARY;
  restore the probe once the environment is healthy again).
- summarize now forwards --allow-skipped; it previously recomputed the
  gate without it and would have stamped every nightly report
  complete=false during the outage. It also records skipped_lanes and
  renders them in the report.
- the health collector validated complete evidence against the full
  12-suite manifest; it now accepts exactly the designed skip
  (performance only) so nightly completes stay verifiable.
- tests pin all three directions: skipped performance accepted, any
  other lane skipped or missing rejected, and prepare forcing offline.

Verified locally: 63/63 unit tests, an 11-suite aggregate/summarize
simulation, and the summarize CLI against a fixture evidence directory.

.github/workflows/rustfs-functional-chain.yml

@@ -252,11 +252,17 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
         run: |
-          if python3 scripts/check_functional_runners.py pf-testing; then
-            echo "performance_ready=online" >> "$GITHUB_OUTPUT"
-          else
-            echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
-          fi
+          # TEMPORARY: the pf-testing environment is broken, so the
+          # performance lane is disabled until it is repaired. Forcing
+          # offline keeps the chain on its designed skip path: the lane job
+          # is skipped and aggregate gets --allow-skipped performance.
+          # Restore the probe below once the environment is healthy again.
+          # if python3 scripts/check_functional_runners.py pf-testing; then
+          #   echo "performance_ready=online" >> "$GITHUB_OUTPUT"
+          # else
+          #   echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
+          # fi
+          echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
 
   validation:
     if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}
@@ -784,6 +790,7 @@ jobs:
         run: >-
           python3 scripts/functional_chain_evidence.py summarize
           --directory "${RUNNER_TEMP}/chain-evidence"
+          --allow-skipped "${{ needs.prepare.outputs.performance_ready == 'online' && '' || 'performance' }}"
           --output "${RUNNER_TEMP}/chain-report.json"
       - name: Retain chain report regardless of test verdict
         if: always()

scripts/functional_chain_evidence.py

@@ -156,7 +156,7 @@ def record(chain, suite, report, output):
         raise error
 
 
-def summarize(chain, directory, needs):
+def summarize(chain, directory, needs, allow_skipped=()):
     """Retain failed/missing lanes without granting complete-success evidence."""
     lanes = []
     for suite in SUITES:
@@ -174,10 +174,13 @@ def summarize(chain, directory, needs):
               "complete": False, "error": None, "completed_at": datetime.now(timezone.utc).isoformat()}
     try:
         require(chain is not None, "candidate preparation did not complete")
-        aggregate(chain, directory, {suite: value for suite, value in needs.items() if suite != "prepare"})
+        aggregate(chain, directory, {suite: value for suite, value in needs.items() if suite != "prepare"}, allow_skipped)
         result["complete"] = True
     except (OSError, ValueError, KeyError, TypeError, AttributeError) as error:
         result["error"] = str(error)
+    skipped = sorted(name for name, value in needs.items() if value.get("result") == "skipped")
+    if skipped:
+        result["skipped_lanes"] = skipped
     return result
 
 
@@ -185,6 +188,8 @@ def render_summary(result):
     lines = ["# RustFS functional chain report", "",
              "- All required suites passed with verified evidence: " + str(result["complete"]).lower(),
              "- Preparation: " + result["needs"].get("prepare", {}).get("result", "missing")]
+    if result.get("skipped_lanes"):
+        lines.append("- Preflight-skipped lanes (allowed by the chain driver): " + ", ".join(result["skipped_lanes"]))
     if result["chain"]:
         chain = result["chain"]
         manifest = chain["candidate"]["manifest"]
@@ -244,7 +249,7 @@ def main():
     args = parser.parse_args()
     if args.mode == "summarize":
         chain = current_chain() if os.environ.get("CHAIN_MANIFEST") else None
-        result = summarize(chain, args.directory, json.loads(os.environ["CHAIN_NEEDS"]))
+        result = summarize(chain, args.directory, json.loads(os.environ["CHAIN_NEEDS"]), args.allow_skipped.split(","))
         args.output.write_text(json.dumps(result, sort_keys=True) + "\n")
         args.output.with_suffix(".md").write_text(render_summary(result))
         if os.environ.get("GITHUB_STEP_SUMMARY"):

scripts/functional_chain_health.py

@@ -9,7 +9,7 @@ import json
 from pathlib import Path
 import subprocess
 
-from functional_chain_evidence import validate_records
+from functional_chain_evidence import SUITES, validate_records
 from resolve_functional_candidate import REPOSITORY, api, read_json_artifact, require, resolve, sha
 
 WORKFLOW = "rustfs-functional-chain.yml"
@@ -28,7 +28,15 @@ def validate_summary(summary, run):
     chain = summary["chain"]
     require(chain["run_id"] == run["id"] and chain["attempt"] == run["run_attempt"] and chain["workflow_sha"] == run["head_sha"], "complete evidence belongs to another run attempt")
     require(sha(chain["testing_sha"]), "missing test-script pin")
-    validate_records(chain, summary["suites"])
+    # The chain skips the performance lane only when its preflight probe
+    # reports the pf-testing fleet offline (aggregate --allow-skipped), so a
+    # complete chain during such an outage carries 11 suites. Accept exactly
+    # that designed skip; any other missing lane stays invalid.
+    skipped = summary.get("skipped_lanes", [])
+    require(isinstance(skipped, list) and set(skipped) <= {"performance"},
+            "complete evidence skipped a lane the chain driver never allows")
+    expected_suites = [suite for suite in SUITES if suite not in skipped]
+    validate_records(chain, summary["suites"], expected_suites)
     candidate = chain["candidate"]
     manifest = candidate["manifest"]
     require(resolve(manifest["build_run_id"], manifest["build_run_attempt"]) == candidate, "producer candidate identity changed")

scripts/test_functional_chain.py

@@ -635,6 +635,22 @@ class EvidenceTests(unittest.TestCase):
         with self.assertRaises(ValueError):
             evidence.aggregate(self.chain, self.directory, self.needs)
 
+    def test_summarize_marks_a_preflight_allowed_skipped_lane(self):
+        needs = {**self.needs, "performance": {"result": "skipped"}}
+        (self.directory / "performance.json").unlink()
+        result = evidence.summarize(self.chain, self.directory, needs, ["performance"])
+        self.assertTrue(result["complete"])
+        self.assertEqual(result["skipped_lanes"], ["performance"])
+        reportable = {**result, "chain": {**self.chain, "testing_sha": "d" * 40,
+                                          "candidate": {"workflow_sha": "b" * 40,
+                                                        "manifest": {"build_run_id": 1, "build_run_attempt": 1,
+                                                                     "source_ref": "main", "source_sha": "a" * 40,
+                                                                     "package_sha256": "e" * 64}}}}
+        self.assertIn("Preflight-skipped lanes", evidence.render_summary(reportable))
+        denied = evidence.summarize(self.chain, self.directory, needs)
+        self.assertFalse(denied["complete"])
+        self.assertIn("without preflight permission", denied["error"])
+
     def test_partial_rerun_missing_artifact_and_zero_test_fail(self):
         path = self.directory / "s3.json"
         original = json.loads(path.read_text())
@@ -837,6 +853,10 @@ class EnvelopeTests(unittest.TestCase):
         self.assertIn("Probe performance fleet runner", prepare)
         self.assertIn("check_functional_runners.py pf-testing", prepare)
         self.assertIn("performance_ready: ${{ steps.perf_probe.outputs.performance_ready }}", prepare)
+        # TEMPORARY: the pf-testing environment is broken and the probe is
+        # disabled in the workflow, so prepare must force the lane offline.
+        # Remove this assertion together with the probe restore.
+        self.assertIn('echo "performance_ready=offline" >> "$GITHUB_OUTPUT"', prepare)
 
     def test_every_lane_retains_failed_evidence_and_identifies_its_own_attempt(self):
         paths = list((candidate.ROOT / ".github/workflows").glob("rustfs-*-test.yml"))

scripts/test_functional_chain_health.py

@@ -36,6 +36,24 @@ class HealthTests(unittest.TestCase):
         self.assertEqual(result["workflow_sha"], "a" * 40)
         self.assertEqual(result["candidate"]["workflow_sha"], "c" * 40)
 
+    def test_complete_evidence_with_skipped_performance_is_accepted(self):
+        skipped = copy.deepcopy(self.summary)
+        skipped["skipped_lanes"] = ["performance"]
+        skipped["suites"] = [record for record in skipped["suites"] if record["suite"] != "performance"]
+        result = self.validate(skipped)
+        self.assertEqual(result["run_id"], 123)
+
+    def test_complete_evidence_with_a_functional_lane_skipped_is_rejected(self):
+        skipped = copy.deepcopy(self.summary)
+        skipped["skipped_lanes"] = ["performance", "heal"]
+        skipped["suites"] = [record for record in skipped["suites"] if record["suite"] not in ("performance", "heal")]
+        with self.assertRaises(ValueError):
+            self.validate(skipped)
+        missing = copy.deepcopy(self.summary)
+        missing["suites"] = [record for record in missing["suites"] if record["suite"] != "heal"]
+        with self.assertRaises(ValueError):
+            self.validate(missing)
+
     def test_substituted_producer_pin_attempt_or_empty_suite_fails(self):
         with self.assertRaises(ValueError):
             self.validate(candidate={**self.candidate, "workflow_sha": "e" * 40})