ci(ft,heal,chain): FT gate 54, heal budget, and a temporary performance lane disable (#9057)

* ci(ft): update the lane gate to the 54-case manifest with scenario F auto-testing #118/#119 added FT scenario F and case B5, so the lane now runs 54 cases over scenarios A,B,C,C2,D,E,F and passes 54/54 — but the workflow gate still pinned the 44-case A-E manifest and failed the report step. Align the gate and the contract-test fixture with the 54-case manifest. * ci(heal): give heal a 90-minute run budget inside a 120-minute job The heal task was still progressing (scanned 494 -> 571+, failed=0) when the 45-minute step cap killed it, so the verification steps never ran and the lane produced invalid evidence (chain 40796474356). The step budget is now 90 minutes, and the job-level budget rises from 60 to 120 minutes so the enclosing job cannot cancel the raised step budget. The timeout contract now records heal as the exception to the shared 60/45 baseline. * ci(chain): force the performance probe offline while pf-testing is broken The pf-testing environment is temporarily broken, so every nightly chain would end red in the performance lane. performance is required=false in scheduled-validations.json, so forcing the probe's performance_ready output to offline takes the chain through its designed skip path: the lane job is skipped, aggregate receives --allow-skipped performance, and the report records the optional-lane skip with its reason. Marked TEMPORARY; restore the probe once the environment is healthy again. * test(table): drop a stale workflow-comment pin left by #9059 #9059 rewrote the table suite step comments but kept the assertion for the old '# Red only for harness/environment breakdowns' line, so test_security_workflow.py fails against current main. The semantics stay pinned by the continue-on-error and HARNESS_OK assertions.
2026-10-30 15:55:55 +00:00 · 2026-10-30 14:03:22 +08:00
parent 69f8e364b0
commit 589b499e3b
5 changed files with 36 additions and 23 deletions
@@ -197,21 +197,21 @@ jobs:
          jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
            .schema_version == 1 and
            .auto_testing_sha == $testing_sha and
-            .selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and
+            .selected_scenarios == ["A", "B", "C", "C2", "D", "E", "F"] and
            .complete == true and
-            .counts.expected == 44 and
-            .counts.seen == 44 and
+            .counts.expected == 54 and
+            .counts.seen == 54 and
            .counts.harness_errors == 0 and
-            (.expected_cases | length) == 44 and
-            (.expected_cases | unique | length) == 44 and
-            (.seen_cases | length) == 44 and
-            (.seen_cases | unique | length) == 44 and
+            (.expected_cases | length) == 54 and
+            (.expected_cases | unique | length) == 54 and
+            (.seen_cases | length) == 54 and
+            (.seen_cases | unique | length) == 54 and
            (.seen_cases | sort) == (.expected_cases | sort) and
            (.missing_cases | length) == 0 and
            (.harness_errors | length) == 0 and
-            (.cases | length) == 44 and
+            (.cases | length) == 54 and
            ([.cases[].case_id] | sort) == (.expected_cases | sort) and
-            ([.cases[].case_id] | unique | length) == 44 and
+            ([.cases[].case_id] | unique | length) == 54 and
            all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
            .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
            .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
@@ -281,12 +281,20 @@ jobs:
          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
          PERFORMANCE_OPTIONAL: ${{ contains(fromJSON(steps.selection.outputs.optional_suites), 'performance') }}
        run: |
-          if python3 scripts/check_functional_runners.py pf-testing; then
-            echo "performance_ready=online" >> "$GITHUB_OUTPUT"
-          else
-            echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
-            test "$PERFORMANCE_OPTIONAL" = true
-          fi
+          # TEMPORARY: the pf-testing environment is broken, so the
+          # performance lane is disabled until it is repaired. Forcing
+          # offline keeps the chain on its designed skip path: the lane job
+          # is skipped, aggregate receives --allow-skipped performance
+          # (required=false in scheduled-validations.json), and the report
+          # records the optional-lane skip with its reason.
+          # Restore the probe below once the environment is healthy again.
+          # if python3 scripts/check_functional_runners.py pf-testing; then
+          #   echo "performance_ready=online" >> "$GITHUB_OUTPUT"
+          # else
+          #   echo "performance_ready=offline" >> "$GITHUB_OUTPUT"
+          #   test "$PERFORMANCE_OPTIONAL" = true
+          # fi
+          echo "performance_ready=offline" >> "$GITHUB_OUTPUT"

  validation:
    if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }}
@@ -55,7 +55,8 @@ env:
 jobs:
  heal-test:
    runs-on: smoke-testing
-    timeout-minutes: 60
+    # 90-minute heal run (see the step below) plus setup, report and cleanup.
+    timeout-minutes: 120
    # Standalone manual run, or one link of the nightly functional chain
    # (storage -> heal -> pool). Pool expansion no longer re-runs heal.
    if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }}
@@ -154,7 +155,7 @@ jobs:
          ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}"

      - name: Run heal test (write -> outage -> heal -> verify)
-        timeout-minutes: 45
+        timeout-minutes: 90
        id: test
        # Collect reports and backlog evidence before the final suite verdict.
        continue-on-error: true
@@ -919,7 +919,10 @@ class WorkflowTimeoutTests(unittest.TestCase):
            with self.subTest(suite=suite):
                source = (candidate.ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text()
                job = yaml_block(source.splitlines(), job_id, 2)
-                job_timeout = 360 if suite == "pool-expand" else 60
+                # heal: a progressing heal outran the shared 45-minute step
+                # cap (chain 40796474356), so it gets a 90-minute run budget
+                # and a 120-minute job budget for setup, report and cleanup.
+                job_timeout = 360 if suite == "pool-expand" else 120 if suite == "heal" else 60
                self.assertIn(f"    timeout-minutes: {job_timeout}", job)
                steps = named_steps(job)
                primary = [step for step in steps.values() if any(
@@ -933,6 +936,8 @@ class WorkflowTimeoutTests(unittest.TestCase):
                        self.assertIsNotNone(timeout_input, event)
                        self.assertIn("        default: '240'", timeout_input)
                        self.assertIn("        required: false", timeout_input)
+                elif suite == "heal":
+                    self.assertIn("        timeout-minutes: 90", primary[0])
                else:
                    self.assertIn("        timeout-minutes: 45", primary[0])
                cleanup = "Cleanup environment"
@@ -540,10 +540,10 @@ class FaultToleranceWorkflowContractTests(unittest.TestCase):
            "GITHUB_RUN_ID": "314159",
        }

-    def write_result(self, *, seen: int = 44, complete: bool = True,
+    def write_result(self, *, seen: int = 54, complete: bool = True,
                     duplicate: bool = False, testing_sha: str | None = None,
                     unrelated_seen: bool = False, missing_verdict: bool = False) -> None:
-        expected = [f"CASE-{index:02d}" for index in range(44)]
+        expected = [f"CASE-{index:02d}" for index in range(54)]
        seen_cases = expected[:seen]
        if unrelated_seen:
            seen_cases = [f"OTHER-{index:02d}" for index in range(seen)]
@@ -555,14 +555,14 @@ class FaultToleranceWorkflowContractTests(unittest.TestCase):
        payload = {
            "schema_version": 1,
            "auto_testing_sha": testing_sha or self.TESTING_SHA,
-            "selected_scenarios": ["A", "B", "C", "C2", "D", "E"],
+            "selected_scenarios": ["A", "B", "C", "C2", "D", "E", "F"],
            "complete": complete,
            "expected_cases": expected,
            "seen_cases": seen_cases,
            "missing_cases": expected[seen:],
            "harness_errors": [],
            "counts": {
-                "expected": 44,
+                "expected": 54,
                "seen": seen,
                "unexpected": 1,
                "known_divergence": 0,
@@ -1186,7 +1186,6 @@ class TableSuiteTests(unittest.TestCase):
        self.assertIn("runs-on: smoke-testing", source)
        # New gate semantics: case failures stay green, harness breakdowns red.
        self.assertRegex(source, r"(?m)^        continue-on-error: true")
-        self.assertIn("# Red only for harness/environment breakdowns; case failures stay green.", source)
        self.assertIn("HARNESS_OK", source)
        self.assertIn("Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed", source)
        self.assertIn('python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}"', source)