diff --git a/.github/workflows/rustfs-fault-tolerance-test.yml b/.github/workflows/rustfs-fault-tolerance-test.yml index 495155d25..8f89920eb 100644 --- a/.github/workflows/rustfs-fault-tolerance-test.yml +++ b/.github/workflows/rustfs-fault-tolerance-test.yml @@ -197,21 +197,21 @@ jobs: jq -e --arg testing_sha "${AUTO_TESTING_SHA}" ' .schema_version == 1 and .auto_testing_sha == $testing_sha and - .selected_scenarios == ["A", "B", "C", "C2", "D", "E"] and + .selected_scenarios == ["A", "B", "C", "C2", "D", "E", "F"] and .complete == true and - .counts.expected == 44 and - .counts.seen == 44 and + .counts.expected == 54 and + .counts.seen == 54 and .counts.harness_errors == 0 and - (.expected_cases | length) == 44 and - (.expected_cases | unique | length) == 44 and - (.seen_cases | length) == 44 and - (.seen_cases | unique | length) == 44 and + (.expected_cases | length) == 54 and + (.expected_cases | unique | length) == 54 and + (.seen_cases | length) == 54 and + (.seen_cases | unique | length) == 54 and (.seen_cases | sort) == (.expected_cases | sort) and (.missing_cases | length) == 0 and (.harness_errors | length) == 0 and - (.cases | length) == 44 and + (.cases | length) == 54 and ([.cases[].case_id] | sort) == (.expected_cases | sort) and - ([.cases[].case_id] | unique | length) == 44 and + ([.cases[].case_id] | unique | length) == 54 and all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length) diff --git a/.github/workflows/rustfs-functional-chain.yml b/.github/workflows/rustfs-functional-chain.yml index 53ab15acd..59fcd711b 100644 --- a/.github/workflows/rustfs-functional-chain.yml +++ b/.github/workflows/rustfs-functional-chain.yml @@ -281,12 +281,20 @@ jobs: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} PERFORMANCE_OPTIONAL: ${{ contains(fromJSON(steps.selection.outputs.optional_suites), 'performance') }} run: | - if python3 scripts/check_functional_runners.py pf-testing; then - echo "performance_ready=online" >> "$GITHUB_OUTPUT" - else - echo "performance_ready=offline" >> "$GITHUB_OUTPUT" - test "$PERFORMANCE_OPTIONAL" = true - fi + # TEMPORARY: the pf-testing environment is broken, so the + # performance lane is disabled until it is repaired. Forcing + # offline keeps the chain on its designed skip path: the lane job + # is skipped, aggregate receives --allow-skipped performance + # (required=false in scheduled-validations.json), and the report + # records the optional-lane skip with its reason. + # Restore the probe below once the environment is healthy again. + # if python3 scripts/check_functional_runners.py pf-testing; then + # echo "performance_ready=online" >> "$GITHUB_OUTPUT" + # else + # echo "performance_ready=offline" >> "$GITHUB_OUTPUT" + # test "$PERFORMANCE_OPTIONAL" = true + # fi + echo "performance_ready=offline" >> "$GITHUB_OUTPUT" validation: if: ${{ github.event_name == 'workflow_dispatch' && !inputs.validation_runner_inspect_only && inputs.validation_retain_evidence_run_id == '' && inputs.validation_package_sha == '' && (inputs.validation_case != '' || inputs.validation_tools_only || inputs.validation_native_tools || inputs.validation_shared_tools || inputs.validation_manager_tools || inputs.validation_cleanup_run_id != '' || inputs.validation_cleanup_retained_evidence) }} diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index 14d7bda06..9ab8ec1b6 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -55,7 +55,8 @@ env: jobs: heal-test: runs-on: smoke-testing - timeout-minutes: 60 + # 90-minute heal run (see the step below) plus setup, report and cleanup. + timeout-minutes: 120 # Standalone manual run, or one link of the nightly functional chain # (storage -> heal -> pool). Pool expansion no longer re-runs heal. if: ${{ inputs.chain_manifest != '' || github.event_name == 'workflow_dispatch' }} @@ -154,7 +155,7 @@ jobs: ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" --log-file "${LOG_FILE}" - name: Run heal test (write -> outage -> heal -> verify) - timeout-minutes: 45 + timeout-minutes: 90 id: test # Collect reports and backlog evidence before the final suite verdict. continue-on-error: true diff --git a/scripts/test_functional_chain.py b/scripts/test_functional_chain.py index 04f12e43d..b8eb8dafb 100644 --- a/scripts/test_functional_chain.py +++ b/scripts/test_functional_chain.py @@ -919,7 +919,10 @@ class WorkflowTimeoutTests(unittest.TestCase): with self.subTest(suite=suite): source = (candidate.ROOT / f".github/workflows/rustfs-{suite}-test.yml").read_text() job = yaml_block(source.splitlines(), job_id, 2) - job_timeout = 360 if suite == "pool-expand" else 60 + # heal: a progressing heal outran the shared 45-minute step + # cap (chain 40796474356), so it gets a 90-minute run budget + # and a 120-minute job budget for setup, report and cleanup. + job_timeout = 360 if suite == "pool-expand" else 120 if suite == "heal" else 60 self.assertIn(f" timeout-minutes: {job_timeout}", job) steps = named_steps(job) primary = [step for step in steps.values() if any( @@ -933,6 +936,8 @@ class WorkflowTimeoutTests(unittest.TestCase): self.assertIsNotNone(timeout_input, event) self.assertIn(" default: '240'", timeout_input) self.assertIn(" required: false", timeout_input) + elif suite == "heal": + self.assertIn(" timeout-minutes: 90", primary[0]) else: self.assertIn(" timeout-minutes: 45", primary[0]) cleanup = "Cleanup environment" diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py index 4ee3450d6..5ca321f9c 100644 --- a/scripts/test_security_workflow.py +++ b/scripts/test_security_workflow.py @@ -540,10 +540,10 @@ class FaultToleranceWorkflowContractTests(unittest.TestCase): "GITHUB_RUN_ID": "314159", } - def write_result(self, *, seen: int = 44, complete: bool = True, + def write_result(self, *, seen: int = 54, complete: bool = True, duplicate: bool = False, testing_sha: str | None = None, unrelated_seen: bool = False, missing_verdict: bool = False) -> None: - expected = [f"CASE-{index:02d}" for index in range(44)] + expected = [f"CASE-{index:02d}" for index in range(54)] seen_cases = expected[:seen] if unrelated_seen: seen_cases = [f"OTHER-{index:02d}" for index in range(seen)] @@ -555,14 +555,14 @@ class FaultToleranceWorkflowContractTests(unittest.TestCase): payload = { "schema_version": 1, "auto_testing_sha": testing_sha or self.TESTING_SHA, - "selected_scenarios": ["A", "B", "C", "C2", "D", "E"], + "selected_scenarios": ["A", "B", "C", "C2", "D", "E", "F"], "complete": complete, "expected_cases": expected, "seen_cases": seen_cases, "missing_cases": expected[seen:], "harness_errors": [], "counts": { - "expected": 44, + "expected": 54, "seen": seen, "unexpected": 1, "known_divergence": 0, @@ -1186,7 +1186,6 @@ class TableSuiteTests(unittest.TestCase): self.assertIn("runs-on: smoke-testing", source) # New gate semantics: case failures stay green, harness breakdowns red. self.assertRegex(source, r"(?m)^ continue-on-error: true") - self.assertIn("# Red only for harness/environment breakdowns; case failures stay green.", source) self.assertIn("HARNESS_OK", source) self.assertIn("Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed", source) self.assertIn('python3 scripts/functional_case_report.py "${LOG_FILE}" "${CASE_TABLE}"', source)