diff --git a/.github/workflows/rustfs-fault-tolerance-test.yml b/.github/workflows/rustfs-fault-tolerance-test.yml index 2c4ee0e6e..9bdf3677f 100644 --- a/.github/workflows/rustfs-fault-tolerance-test.yml +++ b/.github/workflows/rustfs-fault-tolerance-test.yml @@ -118,6 +118,9 @@ jobs: - name: Run fault-tolerance scenarios (A, B, C, C2) id: test + # Case failures keep the run green: the report and the backlog + # issue manager carry the product signal. + continue-on-error: true run: | ARGS=(--all -y --package-url "${{ inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}") if [ "${{ inputs.strict }}" = "true" ]; then @@ -149,6 +152,20 @@ jobs: printf '%s -> %s\n' "$(basename "${f}" .code)" "$(cat "${f}")" done } > "${REPORT_FILE}" + cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + # Product gate: failing FT cases keep the run green — the report above and + # the backlog issue manager carry the signal. Only harness/environment + # breakdowns (no case verdicts at all) turn the workflow red. + FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)" + FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)" + FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} )) + HARNESS_OK=0 + if [ "${{ steps.test.outcome }}" = "success" ]; then + HARNESS_OK=1 + elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then + HARNESS_OK=1 + fi + [ "${HARNESS_OK}" = "1" ] - name: Manage backlog issues (dedup / label / auto-close) # Signal-based lifecycle: dedups against open issues by label diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index a86deb332..3d2f1a947 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -143,6 +143,9 @@ jobs: - name: Run heal test (write -> outage -> heal -> verify) id: test + # Case failures keep the run green: the report and the backlog + # issue manager carry the product signal. + continue-on-error: true run: | ./auto-testing/rustfs_heal_test.sh \ --steps "3,4,5,6,7" -y \ @@ -224,6 +227,20 @@ jobs: if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then RESULT=success fi + + # Product gate: failing steps keep the run green — they are reported + # below and tracked in rustfs/backlog. Only harness/environment + # breakdowns turn the workflow red. + STEPS_TOTAL="$(grep -cE '^\[HEAL-STEP\]' "${LOG_FILE}" 2>/dev/null || true)" + STEPS_FAIL="$(grep -cE '^\[HEAL-STEP\].* FAIL$' "${LOG_FILE}" 2>/dev/null || true)" + STEPS_TOTAL=$(( ${STEPS_TOTAL:-0} + 0 )); STEPS_FAIL=$(( ${STEPS_FAIL:-0} + 0 )) + STEPS_PASS=$(( STEPS_TOTAL - STEPS_FAIL )) + HARNESS_OK=0 + if [ '${{ steps.test.outcome }}' = 'success' ]; then + HARNESS_OK=1 + elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${STEPS_TOTAL}" -gt 0 ] && [ "${STEPS_PASS}" -ge 1 ]; then + HARNESS_OK=1 + fi { echo "# RustFS heal test report" echo "" @@ -235,19 +252,24 @@ jobs: echo "- Test Step Outcome: ${RESULT}" echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - if [ "${RESULT}" = "success" ]; then + if [ -s "${STEPS_TABLE}" ]; then cat "${STEPS_TABLE}" echo "" + fi + echo "- Product result: ${STEPS_PASS} passed, ${STEPS_FAIL} failed (failing steps are tracked in rustfs/backlog)" + echo "" + if [ -s "${STEPS_TABLE}" ]; then echo "## Log tail" echo '```text' - tail -n 200 "${LOG_FILE}" + tail -n 200 "${LOG_FILE}" 2>/dev/null || true echo '```' else echo "The suite or evidence validation failed. See this run's artifact for partial step results and suite.log." fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" - [ "${RESULT}" = "success" ] + # Red only for harness/environment breakdowns; step failures stay green. + [ "${HARNESS_OK}" = "1" ] - name: Upload functional report to dashboard if: ${{ always() && steps.evidence.outcome == 'success' }} diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml index e74f9ddaf..a88524562 100644 --- a/.github/workflows/rustfs-kms-test.yml +++ b/.github/workflows/rustfs-kms-test.yml @@ -126,6 +126,9 @@ jobs: - name: Run KMS suite id: test + # Case failures keep the run green: the report and the backlog + # issue manager carry the product signal. + continue-on-error: true run: | set -euo pipefail chmod +x auto-testing/rustfs-kms-test.sh @@ -174,6 +177,21 @@ jobs: if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then RESULT=success fi + + # Product gate: failing cases keep the run green — they are reported + # below and tracked in rustfs/backlog. Only harness/environment + # breakdowns (the suite never reached case level, or a wholesale + # failure with zero passes) turn the workflow red. + CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)" + CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)" + CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 )) + CASES_PASS=$(( CASES_TOTAL - CASES_FAIL )) + HARNESS_OK=0 + if [ '${{ steps.test.outcome }}' = 'success' ]; then + HARNESS_OK=1 + elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then + HARNESS_OK=1 + fi { echo "# RustFS KMS test report" echo "" @@ -185,19 +203,24 @@ jobs: echo "- Test Step Outcome: ${RESULT}" echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - if [ "${RESULT}" = "success" ]; then + if [ -s "${CASE_TABLE}" ]; then cat "${CASE_TABLE}" echo "" + fi + echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)" + echo "" + if [ -s "${CASE_TABLE}" ]; then echo "## Log tail" echo '```text' - tail -n 200 "${LOG_FILE}" + tail -n 200 "${LOG_FILE}" 2>/dev/null || true echo '```' else echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" - [ "${RESULT}" = "success" ] + # Red only for harness/environment breakdowns; case failures stay green. + [ "${HARNESS_OK}" = "1" ] - name: Upload functional report to dashboard if: ${{ always() && steps.evidence.outcome == 'success' }} diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml index f3736675f..6653c348b 100644 --- a/.github/workflows/rustfs-pool-expand-test.yml +++ b/.github/workflows/rustfs-pool-expand-test.yml @@ -227,6 +227,9 @@ jobs: - name: Run pool expansion & decommission test id: pool_test + # Case failures keep the run green: the report and the backlog + # issue manager carry the product signal. + continue-on-error: true run: | set -o pipefail STEPS="4,5,6" diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml index 5e8ca4454..dbfa9ca1c 100644 --- a/.github/workflows/rustfs-replication-test.yml +++ b/.github/workflows/rustfs-replication-test.yml @@ -130,6 +130,9 @@ jobs: - name: Run replication suite id: test + # Case failures keep the run green: the report and the backlog + # issue manager carry the product signal. + continue-on-error: true run: | set -euo pipefail chmod +x auto-testing/rustfs-replication-test.sh @@ -181,6 +184,21 @@ jobs: if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then RESULT=success fi + + # Product gate: failing cases keep the run green — they are reported + # below and tracked in rustfs/backlog. Only harness/environment + # breakdowns (the suite never reached case level, or a wholesale + # failure with zero passes) turn the workflow red. + CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)" + CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)" + CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 )) + CASES_PASS=$(( CASES_TOTAL - CASES_FAIL )) + HARNESS_OK=0 + if [ '${{ steps.test.outcome }}' = 'success' ]; then + HARNESS_OK=1 + elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then + HARNESS_OK=1 + fi { echo "# RustFS replication test report" echo "" @@ -193,19 +211,24 @@ jobs: echo "- Test Step Outcome: ${RESULT}" echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - if [ "${RESULT}" = "success" ]; then + if [ -s "${CASE_TABLE}" ]; then cat "${CASE_TABLE}" echo "" + fi + echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)" + echo "" + if [ -s "${CASE_TABLE}" ]; then echo "## Log tail" echo '```text' - tail -n 200 "${LOG_FILE}" + tail -n 200 "${LOG_FILE}" 2>/dev/null || true echo '```' else echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" - [ "${RESULT}" = "success" ] + # Red only for harness/environment breakdowns; case failures stay green. + [ "${HARNESS_OK}" = "1" ] - name: Upload functional report to dashboard if: ${{ always() && steps.evidence.outcome == 'success' }} diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml index 7bdc16de4..c0e6facc9 100644 --- a/.github/workflows/rustfs-s3-compat-test.yml +++ b/.github/workflows/rustfs-s3-compat-test.yml @@ -105,6 +105,9 @@ jobs: - name: Run S3 compatibility suite id: test + # Case failures keep the run green: the report and the backlog + # issue manager carry the product signal. + continue-on-error: true run: | set -euo pipefail chmod +x auto-testing/rustfs-s3-compat-test.sh @@ -150,6 +153,21 @@ jobs: if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then RESULT=success fi + + # Product gate: failing cases keep the run green — they are reported + # below and tracked in rustfs/backlog. Only harness/environment + # breakdowns (the suite never reached case level, or a wholesale + # failure with zero passes) turn the workflow red. + CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)" + CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)" + CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 )) + CASES_PASS=$(( CASES_TOTAL - CASES_FAIL )) + HARNESS_OK=0 + if [ '${{ steps.test.outcome }}' = 'success' ]; then + HARNESS_OK=1 + elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then + HARNESS_OK=1 + fi { echo "# RustFS S3 compatibility test report" echo "" @@ -162,19 +180,24 @@ jobs: echo "- Test Step Outcome: ${RESULT}" echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - if [ "${RESULT}" = "success" ]; then + if [ -s "${CASE_TABLE}" ]; then cat "${CASE_TABLE}" echo "" + fi + echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)" + echo "" + if [ -s "${CASE_TABLE}" ]; then echo "## Log tail" echo '```text' - tail -n 200 "${LOG_FILE}" + tail -n 200 "${LOG_FILE}" 2>/dev/null || true echo '```' else echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" - [ "${RESULT}" = "success" ] + # Red only for harness/environment breakdowns; case failures stay green. + [ "${HARNESS_OK}" = "1" ] - name: Upload functional report to dashboard if: ${{ always() && steps.evidence.outcome == 'success' }} diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml index b8926b7a1..2144c6e08 100644 --- a/.github/workflows/rustfs-security-test.yml +++ b/.github/workflows/rustfs-security-test.yml @@ -144,6 +144,8 @@ jobs: - name: Run security suite id: test + # Case failures keep the run green: the report and the backlog + # issue manager carry the product signal. continue-on-error: true env: REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md @@ -184,6 +186,20 @@ jobs: if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then RESULT=success fi + + # Product gate: failing cases keep the run green — they are reported + # below and tracked in rustfs/backlog. Only harness/environment + # breakdowns turn the workflow red. + VERDICTS_TOTAL="$(grep -cE '^\[(PASS|FAIL|UNSUPPORTED)\] [A-Z]' "${LOG_FILE}" 2>/dev/null || true)" + VERDICTS_FAIL="$(grep -cE '^\[FAIL\] [A-Z]' "${LOG_FILE}" 2>/dev/null || true)" + VERDICTS_TOTAL=$(( ${VERDICTS_TOTAL:-0} + 0 )); VERDICTS_FAIL=$(( ${VERDICTS_FAIL:-0} + 0 )) + VERDICTS_PASS=$(( VERDICTS_TOTAL - VERDICTS_FAIL )) + HARNESS_OK=0 + if [ "${TEST_OUTCOME}" = "success" ]; then + HARNESS_OK=1 + elif [ "${TEST_OUTCOME}" = "failure" ] && [ "${VERDICTS_TOTAL}" -gt 0 ] && [ "${VERDICTS_PASS}" -ge 1 ]; then + HARNESS_OK=1 + fi { echo "# RustFS security test report" echo "" @@ -196,16 +212,15 @@ jobs: echo "" # The dashboard prioritizes case rows over the step outcome. # Keep partial case results in the artifact when the suite fails. - if [ "${RESULT}" = "success" ]; then + if [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md" - elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then - echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics." - else - echo "The suite did not produce a non-empty report." + echo "" fi + echo "- Product result: ${VERDICTS_PASS} passed, ${VERDICTS_FAIL} failed (failing cases are tracked in rustfs/backlog)" } > "${SECURITY_ARTIFACTS_DIR}/report.md" cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}" - [ "${RESULT}" = "success" ] + # Red only for harness/environment breakdowns; case failures stay green. + [ "${HARNESS_OK}" = "1" ] - name: Upload functional report to dashboard if: ${{ always() && steps.evidence.outcome == 'success' }} diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml index 3651fe486..c350b2166 100644 --- a/.github/workflows/rustfs-storage-test.yml +++ b/.github/workflows/rustfs-storage-test.yml @@ -114,6 +114,9 @@ jobs: - name: Run storage engine suite id: test + # Case failures keep the run green: the report and the backlog + # issue manager carry the product signal. + continue-on-error: true run: | set -euo pipefail chmod +x auto-testing/rustfs-storage-test.sh @@ -165,6 +168,21 @@ jobs: if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then RESULT=success fi + + # Product gate: failing cases keep the run green — they are reported + # below and tracked in rustfs/backlog. Only harness/environment + # breakdowns (the suite never reached case level, or a wholesale + # failure with zero passes) turn the workflow red. + CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)" + CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)" + CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 )) + CASES_PASS=$(( CASES_TOTAL - CASES_FAIL )) + HARNESS_OK=0 + if [ '${{ steps.test.outcome }}' = 'success' ]; then + HARNESS_OK=1 + elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then + HARNESS_OK=1 + fi { echo "# RustFS storage engine test report" echo "" @@ -177,19 +195,24 @@ jobs: echo "- Test Step Outcome: ${RESULT}" echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - if [ "${RESULT}" = "success" ]; then + if [ -s "${CASE_TABLE}" ]; then cat "${CASE_TABLE}" echo "" + fi + echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)" + echo "" + if [ -s "${CASE_TABLE}" ]; then echo "## Log tail" echo '```text' - tail -n 200 "${LOG_FILE}" + tail -n 200 "${LOG_FILE}" 2>/dev/null || true echo '```' else echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" - [ "${RESULT}" = "success" ] + # Red only for harness/environment breakdowns; case failures stay green. + [ "${HARNESS_OK}" = "1" ] - name: Upload functional report to dashboard if: ${{ always() && steps.evidence.outcome == 'success' }} diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml index 4314331fa..ec7acba77 100644 --- a/.github/workflows/rustfs-tier-test.yml +++ b/.github/workflows/rustfs-tier-test.yml @@ -257,6 +257,8 @@ jobs: - name: Run tier suite id: test + # Case failures keep the run green: the report and the backlog + # issue manager carry the product signal. continue-on-error: true env: PACKAGE_URL_INPUT: ${{ inputs.package_url }} @@ -471,26 +473,18 @@ jobs: GATE_RC_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-gate.rc run: | set -euo pipefail + # Product gate: failing cases keep the run green (the structured gate + # rc and the report carry the per-case signal, and the backlog issue + # manager tracks failures). Only harness/environment breakdowns — + # evidence initialization or a missing structured gate result — turn + # the workflow red. failed=0 if [ "${EVIDENCE_OUTCOME}" != "success" ]; then echo "tier evidence directory initialization is ${EVIDENCE_OUTCOME}, expected success" >&2 failed=1 - fi - if [ "${TEST_OUTCOME}" != "success" ]; then - echo "tier suite step outcome is ${TEST_OUTCOME}, expected success" >&2 - failed=1 - fi - if [ "${EVIDENCE_OUTCOME}" != "success" ]; then - echo "structured gate result is unavailable because evidence initialization failed" >&2 elif [ ! -s "${GATE_RC_FILE}" ]; then echo "structured gate result is missing" >&2 failed=1 - else - GATE_RC="$(tr -d '[:space:]' < "${GATE_RC_FILE}")" - if ! [[ "${GATE_RC}" =~ ^[0-9]+$ ]] || [ "${GATE_RC}" -ne 0 ]; then - echo "structured 56-case gate failed with exit ${GATE_RC:-invalid}" >&2 - failed=1 - fi fi [ "${failed}" -eq 0 ] diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml index bad5c1119..ca4b3333e 100644 --- a/.github/workflows/rustfs-upgrade-test.yml +++ b/.github/workflows/rustfs-upgrade-test.yml @@ -160,6 +160,9 @@ jobs: - name: Run upgrade compatibility suite id: test + # Case failures keep the run green: the report and the backlog + # issue manager carry the product signal. + continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} run: | @@ -247,6 +250,21 @@ jobs: if [ '${{ steps.test.outcome }}' = 'success' ] && [ "${CASE_RESULT}" = 'success' ]; then RESULT=success fi + + # Product gate: failing cases keep the run green — they are reported + # below and tracked in rustfs/backlog. Only harness/environment + # breakdowns (the suite never reached case level, or a wholesale + # failure with zero passes) turn the workflow red. + CASES_TOTAL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| (PASS|FAIL|UNSUPPORTED|RUNNING) \|' "${CASE_TABLE}" 2>/dev/null || true)" + CASES_FAIL="$(grep -cE '^\| [A-Z][A-Z0-9]*-[0-9]+ .*\| FAIL \|' "${CASE_TABLE}" 2>/dev/null || true)" + CASES_TOTAL=$(( ${CASES_TOTAL:-0} + 0 )); CASES_FAIL=$(( ${CASES_FAIL:-0} + 0 )) + CASES_PASS=$(( CASES_TOTAL - CASES_FAIL )) + HARNESS_OK=0 + if [ '${{ steps.test.outcome }}' = 'success' ]; then + HARNESS_OK=1 + elif [ '${{ steps.test.outcome }}' = 'failure' ] && [ "${CASES_TOTAL}" -gt 0 ] && [ "${CASES_PASS}" -ge 1 ]; then + HARNESS_OK=1 + fi { echo "# RustFS upgrade compatibility report" echo "" @@ -259,21 +277,28 @@ jobs: echo "- Test Step Outcome: ${RESULT}" echo "- Suite Step Outcome: ${{ steps.test.outcome }}" echo "" - if [ "${RESULT}" = "success" ]; then + if [ -s "${MATRIX_TABLE}" ]; then cat "${MATRIX_TABLE}" echo "" + fi + if [ -s "${CASE_TABLE}" ]; then cat "${CASE_TABLE}" echo "" + fi + echo "- Product result: ${CASES_PASS} passed, ${CASES_FAIL} failed (failing cases are tracked in rustfs/backlog)" + echo "" + if [ -s "${CASE_TABLE}" ]; then echo "## Log tail" echo '```text' - tail -n 200 "${LOG_FILE}" + tail -n 200 "${LOG_FILE}" 2>/dev/null || true echo '```' else echo "The suite or evidence validation failed. See this run's artifact for partial case results and suite.log." fi } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" - [ "${RESULT}" = "success" ] + # Red only for harness/environment breakdowns; case failures stay green. + [ "${HARNESS_OK}" = "1" ] - name: Upload functional report to dashboard if: ${{ always() && steps.evidence.outcome == 'success' }} diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py index 340033605..10eee9d4e 100644 --- a/scripts/test_security_workflow.py +++ b/scripts/test_security_workflow.py @@ -170,8 +170,12 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase): self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n") self.context["steps.test.outcome"] = outcome report = self.run_step("Generate report") + # The report step is red only for harness/environment breakdowns; + # the fixture log carries no case verdicts, so failure outcomes + # stay red here, and a successful suite is green unconditionally. success = outcome == "success" and mode == "present" - self.assertEqual(report.returncode == 0, success, report.stderr) + green = outcome == "success" + self.assertEqual(report.returncode == 0, green, report.stderr) contents = (self.artifacts / "report.md").read_text() for expected in ( "https://github.com/rustfs/rustfs/actions/runs/314159", "Attempt: 2", @@ -179,8 +183,8 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase): f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}", ): self.assertIn(expected, contents) - self.assertEqual(CASE_ROW in contents, success) - self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, success) + self.assertEqual(CASE_ROW in contents, mode == "present") + self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, mode == "present") if mode == "present": raw = (self.artifacts / "suite-report.md").read_text() self.assertEqual(raw, f"CURRENT SUITE DIAGNOSTIC\n{CASE_ROW}\n") @@ -404,7 +408,9 @@ class FunctionalWorkflowTests(unittest.TestCase): steps = named_steps(job) if suite in self.DIRECT_TESTS: test = steps[self.DIRECT_TESTS[suite]] - self.assertNotRegex("\n".join(test), r'''(?m)^ ["']?continue-on-error["']?\s*:''') + # Case failures keep the run green; the step records its + # outcome for the report and the backlog issue manager. + self.assertRegex("\n".join(test), r'''(?m)^ continue-on-error: true$''') self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", steps["Generate report"]) cleanup = steps["Reset test environment (after)" if suite == "performance" else "Cleanup environment (after)"] condition = next(line.strip() for line in cleanup if line.startswith(" if:")) @@ -621,10 +627,14 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase): self.context["steps.test.outcome"] = outcome report = self.run_step("Generate report") success = outcome == "success" and log == good - self.assertEqual(report.returncode == 0, success, report.stderr) + # Report steps are red only for harness/environment breakdowns; + # a failure outcome with recorded case rows stays green + # (performance is unchanged and still gates on the suite result). + green = (outcome == "success" or (outcome == "failure" and log in (good, partial))) if suite != "performance" else success + self.assertEqual(report.returncode == 0, green, report.stderr) contents = Path(self.env["REPORT_FILE"]).read_text() self.assertNotIn("OLD RUN EVIDENCE", contents) - self.assertEqual("| PASS |" in contents, success) + self.assertEqual("| PASS |" in contents, success if suite == "performance" else log in (good, partial)) for value in ("actions/runs/314159", "Attempt: 2", "Workflow Commit: " + self.context["github.sha"], f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}"): self.assertIn(value, contents) @@ -802,11 +812,14 @@ emit_step_result() { for failed_log in failed_logs: with self.subTest(log=failed_log): Path(self.env["LOG_FILE"]).write_text(failed_log) + # A failing heal run exits non-zero; the report step stays green + # because the steps ran, and the per-step table is always published. + self.context["steps.test.outcome"] = "failure" report = self.run_step("Generate report") - self.assertNotEqual(report.returncode, 0, report.stderr) + self.assertEqual(report.returncode, 0, report.stderr) contents = Path(self.env["REPORT_FILE"]).read_text() self.assertIn("Test Step Outcome: failure", contents) - self.assertNotIn("| PASS |", contents) + self.assertIn("| PASS |", contents) if "original failure" in failed_log: self.assertIn("| 3 | original failure | FAIL |", (self.artifacts / "steps.md").read_text()) if "later step failure" in failed_log: