fix(ci): reject incomplete fault-tolerance runs (#7961)

* fix(ci): reject incomplete fault-tolerance runs

* fix(ci): validate fault-tolerance case verdicts

.github/workflows/rustfs-fault-tolerance-test.yml

@@ -4,13 +4,14 @@
 # behavior under drive and node loss against the erasure-coding contract and
 # snapshots health-endpoint responses at every tier.
 #
-#   A  single-node 4 drives (SNMD): hide 1/2/3 drives, restore
+#   A  single-node 4 drives (SNMD): revoke access to 1/2/3 drives, restore
 #   B  multi-node 4x1 (one drive per node): stop 1/2/3 nodes, restore
 #   C  multi-node 4x4 (16 drives, EC:4): stop 1 node (read-quorum boundary),
 #      stop 2 nodes, restore
 #   C2 multi-node 4x4 with EC:8: 2 nodes down puts 8 drives online -- reads
 #      satisfy the EC read quorum while the lock majority is broken (the
 #      reported divergence window: reads 503 with lock_quorum_unavailable)
+#   D  multi-node 4x1: stop 1/2 nodes, then restore
 #
 # Expectations come from product source (default_parity_count, erasure set
 # sizing). By default a "reads refused although the read quorum is met"
@@ -32,6 +33,11 @@ on:
         description: 'Direct .deb URL. Required unless the nightly default is wanted.'
         required: false
         type: string
+      auto_testing_ref:
+        description: 'auto-testing branch, tag, or SHA (defaults to main)'
+        required: false
+        default: main
+        type: string
       strict:
         description: 'Fail the suite when reads are refused despite a met read quorum'
         type: boolean
@@ -52,7 +58,7 @@ on:
 permissions:
   contents: read
 
-# The suite stops services and hides drive dirs on the shared fleet; only one
+# The suite stops services and revokes drive access on the shared fleet; only one
 # functional suite may touch the environment at a time.
 concurrency:
   group: rustfs-shared-functional-tests-v2
@@ -92,6 +98,7 @@ jobs:
             printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
             printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
             printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
+            printf 'RESULTS_FILE=%s/results.json\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
             printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
             printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
           } >> "${GITHUB_ENV}"
@@ -109,16 +116,24 @@ jobs:
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
         with:
           repository: rustfs/auto-testing
-          ref: ${{ steps.chain.outputs.testing_sha || 'main' }}
+          ref: ${{ steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main' }}
           token: ${{ secrets.PF_TESTING_GH_TOKEN }}
           path: auto-testing
           persist-credentials: false
 
+      - name: Record auto-testing revision
+        run: |
+          set -euo pipefail
+          AUTO_TESTING_SHA="$(git -C auto-testing rev-parse HEAD)"
+          printf 'AUTO_TESTING_SHA=%s\n' "${AUTO_TESTING_SHA}" >> "${GITHUB_ENV}"
+          echo "auto-testing revision: ${AUTO_TESTING_SHA}"
+
       - name: Show environment
         run: |
           uname -a
           jq --version
           aws --version
+          echo "auto-testing: ${AUTO_TESTING_SHA}"
           df -h /data | tail -1
 
       - name: Cleanup environment (before)
@@ -134,14 +149,14 @@ jobs:
           CHAIN_MANIFEST: ${{ inputs.chain_manifest }}
         run: python3 scripts/prepare_functional_package.py prepare
 
-      - name: Run fault-tolerance scenarios (A, B, C, C2)
+      - name: Run fault-tolerance scenarios (A, B, C, C2, D)
         timeout-minutes: 45
         id: test
         # Standalone case failures are reported by the backlog manager.
         # Chain evidence separately requires complete passing cases.
         continue-on-error: true
         run: |
-          ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
+          ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}" --results-file "${RESULTS_FILE}")
           if [ "${{ inputs.strict }}" = "true" ]; then
             ARGS+=(--strict)
           fi
@@ -158,12 +173,13 @@ jobs:
             echo ""
             echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
             echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
+            echo "- auto-testing: \`${AUTO_TESTING_SHA}\`"
             echo "- Strict mode: ${{ inputs.strict || 'false' }}"
             echo ""
             echo "## Per-probe results"
             echo ""
             echo '```'
-            grep -E '^FT-(CASE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
+            grep -E '^FT-(CASE|COMPLETE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
             echo '```'
             echo ""
             echo "## Health snapshots"
@@ -174,19 +190,30 @@ jobs:
             done
           } > "${REPORT_FILE}"
           cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
-          # Product gate: failing FT cases keep the run green — the report above and
-          # the backlog issue manager carry the signal. Only harness/environment
-          # breakdowns (no case verdicts at all) turn the workflow red.
-          FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)"
-          FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)"
-          FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} ))
-          HARNESS_OK=0
-          if [ "${{ steps.test.outcome }}" = "success" ]; then
-            HARNESS_OK=1
-          elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then
-            HARNESS_OK=1
-          fi
-          [ "${HARNESS_OK}" = "1" ]
+          printf 'complete=false\n' >> "${GITHUB_OUTPUT}"
+          jq -e --arg testing_sha "${AUTO_TESTING_SHA}" '
+            .schema_version == 1 and
+            .auto_testing_sha == $testing_sha and
+            .selected_scenarios == ["A", "B", "C", "C2", "D"] and
+            .complete == true and
+            .counts.expected == 38 and
+            .counts.seen == 38 and
+            .counts.harness_errors == 0 and
+            (.expected_cases | length) == 38 and
+            (.expected_cases | unique | length) == 38 and
+            (.seen_cases | length) == 38 and
+            (.seen_cases | unique | length) == 38 and
+            (.seen_cases | sort) == (.expected_cases | sort) and
+            (.missing_cases | length) == 0 and
+            (.harness_errors | length) == 0 and
+            (.cases | length) == 38 and
+            ([.cases[].case_id] | sort) == (.expected_cases | sort) and
+            ([.cases[].case_id] | unique | length) == 38 and
+            all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and
+            .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and
+            .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length)
+          ' "${RESULTS_FILE}" >/dev/null
+          printf 'complete=true\n' >> "${GITHUB_OUTPUT}"
 
       - name: Manage backlog issues (dedup / label / auto-close)
         timeout-minutes: 2
@@ -196,7 +223,7 @@ jobs:
         # green run. Never acts on cancelled runs. Logic lives in
         # auto-testing/scripts/issue_manager.py, which parses the
         # FT-CASE verdict lines from the suite log.
-        if: ${{ always() && steps.evidence.outcome == 'success' }}
+        if: ${{ always() && steps.evidence.outcome == 'success' && steps.chain_report.outputs.complete == 'true' }}
         continue-on-error: true
         env:
           GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
@@ -272,6 +299,7 @@ jobs:
           name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
           path: |
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
+            ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results.json
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
             ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
           if-no-files-found: warn

scripts/test_functional_chain.py

@@ -363,7 +363,12 @@ class RunnerTests(unittest.TestCase):
                 self.assertIn("  workflow_call:", workflow)
                 self.assertIn("  workflow_dispatch:", workflow)
                 self.assertIn("group: rustfs-shared-functional-tests-v2", workflow)
-                self.assertIn("ref: ${{ steps.chain.outputs.testing_sha || 'main' }}", workflow)
+                expected_ref = (
+                    "ref: ${{ steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main' }}"
+                    if suite == "fault-tolerance"
+                    else "ref: ${{ steps.chain.outputs.testing_sha || 'main' }}"
+                )
+                self.assertIn(expected_ref, workflow)
                 self.assertIn("steps.chain_package.outputs.package_url || inputs.package_url", workflow)
                 self.assertLess(workflow.index("prepare_functional_package.py prepare"), workflow.index("id: test"))
                 self.assertIn("prepare_functional_package.py cleanup", workflow)

scripts/test_security_workflow.py

@@ -405,7 +405,7 @@ class FunctionalWorkflowTests(unittest.TestCase):
         "kms": "Run KMS suite", "storage": "Run storage engine suite",
         "s3-compat": "Run S3 compatibility suite", "upgrade": "Run upgrade compatibility suite",
         "replication": "Run replication suite",
-        "table": "Run table suite", "fault-tolerance": "Run fault-tolerance scenarios (A, B, C, C2)",
+        "table": "Run table suite", "fault-tolerance": "Run fault-tolerance scenarios (A, B, C, C2, D)",
     }
 
     def test_failure_and_always_step_wiring(self) -> None:
@@ -504,6 +504,145 @@ class FunctionalWorkflowTests(unittest.TestCase):
                 self.assertEqual(markers, ["cleanup", "dispatch"])
 
 
+class FaultToleranceWorkflowContractTests(unittest.TestCase):
+    TESTING_SHA = "a" * 40
+
+    def setUp(self) -> None:
+        source = (ROOT / ".github/workflows/rustfs-fault-tolerance-test.yml").read_text()
+        job = yaml_block(source.splitlines(), "fault-tolerance-test", 2)
+        self.assertIsNotNone(job)
+        self.steps = named_steps(job)
+        self.report_body = shell_body(self.steps["Generate report"])
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.directory = Path(self.temp.name)
+        self.artifacts = self.directory / "artifacts"
+        (self.artifacts / "evidence").mkdir(parents=True)
+        self.log = self.artifacts / "suite.log"
+        self.log.write_text("FT-REPORT: fixture\n")
+        self.results = self.artifacts / "results.json"
+        self.output = self.directory / "github-output"
+        self.env = {
+            **os.environ,
+            "AUTO_TESTING_SHA": self.TESTING_SHA,
+            "FUNCTIONAL_ARTIFACTS_DIR": str(self.artifacts),
+            "LOG_FILE": str(self.log),
+            "REPORT_FILE": str(self.artifacts / "report.md"),
+            "RESULTS_FILE": str(self.results),
+            "GITHUB_OUTPUT": str(self.output),
+            "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"),
+            "GITHUB_SERVER_URL": "https://github.com",
+            "GITHUB_REPOSITORY": "rustfs/rustfs",
+            "GITHUB_RUN_ID": "314159",
+        }
+
+    def write_result(self, *, seen: int = 38, complete: bool = True,
+                     duplicate: bool = False, testing_sha: str | None = None,
+                     unrelated_seen: bool = False, missing_verdict: bool = False) -> None:
+        expected = [f"CASE-{index:02d}" for index in range(38)]
+        seen_cases = expected[:seen]
+        if unrelated_seen:
+            seen_cases = [f"OTHER-{index:02d}" for index in range(seen)]
+        case_ids = list(seen_cases)
+        if unrelated_seen:
+            case_ids = expected[:seen]
+        if duplicate:
+            case_ids[-1] = case_ids[0]
+        payload = {
+            "schema_version": 1,
+            "auto_testing_sha": testing_sha or self.TESTING_SHA,
+            "selected_scenarios": ["A", "B", "C", "C2", "D"],
+            "complete": complete,
+            "expected_cases": expected,
+            "seen_cases": seen_cases,
+            "missing_cases": expected[seen:],
+            "harness_errors": [],
+            "counts": {
+                "expected": 38,
+                "seen": seen,
+                "unexpected": 1,
+                "known_divergence": 0,
+                "harness_errors": 0,
+            },
+            "cases": [
+                {"case_id": case_id, "verdict": "UNEXPECTED" if index == 0 else "pass", "detail": "fixture"}
+                for index, case_id in enumerate(case_ids)
+            ],
+        }
+        if missing_verdict:
+            payload["cases"][0].pop("verdict")
+        self.results.write_text(json.dumps(payload))
+
+    def run_report(self) -> subprocess.CompletedProcess[str]:
+        replacements = {
+            "inputs.package_url || 'nightly (R2 latest)'": "fixture-package",
+            "inputs.strict || 'false'": "false",
+        }
+        rendered = re.sub(
+            r"\$\{\{\s*(.*?)\s*\}\}",
+            lambda match: replacements[match[1]],
+            self.report_body,
+        )
+        return subprocess.run(
+            ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", rendered],
+            cwd=self.directory,
+            env=self.env,
+            capture_output=True,
+            text=True,
+        )
+
+    def test_complete_product_failure_is_valid_harness_evidence(self) -> None:
+        self.write_result()
+        result = self.run_report()
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(self.output.read_text().splitlines()[-1], "complete=true")
+
+    def test_partial_or_missing_results_fail_the_harness_gate(self) -> None:
+        self.write_result(seen=17, complete=False)
+        partial = self.run_report()
+        self.assertNotEqual(partial.returncode, 0)
+        self.assertEqual(self.output.read_text().splitlines()[-1], "complete=false")
+
+        self.results.unlink()
+        self.output.unlink()
+        missing = self.run_report()
+        self.assertNotEqual(missing.returncode, 0)
+        self.assertEqual(self.output.read_text().splitlines()[-1], "complete=false")
+
+    def test_duplicate_cases_or_revision_mismatch_fail_the_harness_gate(self) -> None:
+        self.write_result(duplicate=True)
+        self.assertNotEqual(self.run_report().returncode, 0)
+
+        self.output.unlink()
+        self.write_result(testing_sha="b" * 40)
+        self.assertNotEqual(self.run_report().returncode, 0)
+
+    def test_missing_verdict_or_unrelated_seen_cases_fail_the_harness_gate(self) -> None:
+        self.write_result(missing_verdict=True)
+        self.assertNotEqual(self.run_report().returncode, 0)
+
+        self.output.unlink()
+        self.write_result(unrelated_seen=True)
+        self.assertNotEqual(self.run_report().returncode, 0)
+
+    def test_backlog_manager_only_runs_after_complete_report(self) -> None:
+        condition = next(
+            line.strip() for line in self.steps["Manage backlog issues (dedup / label / auto-close)"]
+            if line.startswith("        if:")
+        )
+        self.assertEqual(
+            condition,
+            "if: ${{ always() && steps.evidence.outcome == 'success' && steps.chain_report.outputs.complete == 'true' }}",
+        )
+
+    def test_auto_testing_ref_is_resolved_once_and_recorded(self) -> None:
+        checkout = "\n".join(self.steps["Checkout auto-testing scripts"])
+        record = "\n".join(self.steps["Record auto-testing revision"])
+        self.assertIn("steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main'", checkout)
+        self.assertIn("git -C auto-testing rev-parse HEAD", record)
+        self.assertIn("AUTO_TESTING_SHA", record)
+
+
 class FunctionalCaseReportTests(unittest.TestCase):
     def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]:
         with tempfile.TemporaryDirectory() as directory:
@@ -727,7 +866,7 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
     def test_upload_allowlist_preserves_diagnostics_without_scratch(self):
         extra = {
             "kms": ["cases.md"], "storage": ["cases.md"], "s3-compat": ["cases.md"],
-            "upgrade": ["cases.md", "matrix.md"], "replication": ["cases.md"], "heal": ["steps.md", "warp.log"], "table": ["cases.md"], "fault-tolerance": [],
+            "upgrade": ["cases.md", "matrix.md"], "replication": ["cases.md"], "heal": ["steps.md", "warp.log"], "table": ["cases.md"], "fault-tolerance": ["results.json"],
             "performance": ["version.txt", "results/master.log", "results/summary.md", "results/summary.tsv",
                             "results/get_1KiB.txt", "results/put_1MiB.txt", "results/mixed_4MiB.txt"],
         }