diff --git a/.github/workflows/rustfs-fault-tolerance-test.yml b/.github/workflows/rustfs-fault-tolerance-test.yml index 920a8a6ff..ab79aceaf 100644 --- a/.github/workflows/rustfs-fault-tolerance-test.yml +++ b/.github/workflows/rustfs-fault-tolerance-test.yml @@ -4,13 +4,14 @@ # behavior under drive and node loss against the erasure-coding contract and # snapshots health-endpoint responses at every tier. # -# A single-node 4 drives (SNMD): hide 1/2/3 drives, restore +# A single-node 4 drives (SNMD): revoke access to 1/2/3 drives, restore # B multi-node 4x1 (one drive per node): stop 1/2/3 nodes, restore # C multi-node 4x4 (16 drives, EC:4): stop 1 node (read-quorum boundary), # stop 2 nodes, restore # C2 multi-node 4x4 with EC:8: 2 nodes down puts 8 drives online -- reads # satisfy the EC read quorum while the lock majority is broken (the # reported divergence window: reads 503 with lock_quorum_unavailable) +# D multi-node 4x1: stop 1/2 nodes, then restore # # Expectations come from product source (default_parity_count, erasure set # sizing). By default a "reads refused although the read quorum is met" @@ -32,6 +33,11 @@ on: description: 'Direct .deb URL. Required unless the nightly default is wanted.' required: false type: string + auto_testing_ref: + description: 'auto-testing branch, tag, or SHA (defaults to main)' + required: false + default: main + type: string strict: description: 'Fail the suite when reads are refused despite a met read quorum' type: boolean @@ -52,7 +58,7 @@ on: permissions: contents: read -# The suite stops services and hides drive dirs on the shared fleet; only one +# The suite stops services and revokes drive access on the shared fleet; only one # functional suite may touch the environment at a time. concurrency: group: rustfs-shared-functional-tests-v2 @@ -92,6 +98,7 @@ jobs: printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}" printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}" printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}" + printf 'RESULTS_FILE=%s/results.json\n' "${FUNCTIONAL_ARTIFACTS_DIR}" printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}" printf 'TMPDIR=%s-scratch\n' "${FUNCTIONAL_ARTIFACTS_DIR}" } >> "${GITHUB_ENV}" @@ -109,16 +116,24 @@ jobs: uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: repository: rustfs/auto-testing - ref: ${{ steps.chain.outputs.testing_sha || 'main' }} + ref: ${{ steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main' }} token: ${{ secrets.PF_TESTING_GH_TOKEN }} path: auto-testing persist-credentials: false + - name: Record auto-testing revision + run: | + set -euo pipefail + AUTO_TESTING_SHA="$(git -C auto-testing rev-parse HEAD)" + printf 'AUTO_TESTING_SHA=%s\n' "${AUTO_TESTING_SHA}" >> "${GITHUB_ENV}" + echo "auto-testing revision: ${AUTO_TESTING_SHA}" + - name: Show environment run: | uname -a jq --version aws --version + echo "auto-testing: ${AUTO_TESTING_SHA}" df -h /data | tail -1 - name: Cleanup environment (before) @@ -134,14 +149,14 @@ jobs: CHAIN_MANIFEST: ${{ inputs.chain_manifest }} run: python3 scripts/prepare_functional_package.py prepare - - name: Run fault-tolerance scenarios (A, B, C, C2) + - name: Run fault-tolerance scenarios (A, B, C, C2, D) timeout-minutes: 45 id: test # Standalone case failures are reported by the backlog manager. # Chain evidence separately requires complete passing cases. continue-on-error: true run: | - ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}") + ARGS=(--all -y --package-url "${{ steps.chain_package.outputs.package_url || inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}" --results-file "${RESULTS_FILE}") if [ "${{ inputs.strict }}" = "true" ]; then ARGS+=(--strict) fi @@ -158,12 +173,13 @@ jobs: echo "" echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}" + echo "- auto-testing: \`${AUTO_TESTING_SHA}\`" echo "- Strict mode: ${{ inputs.strict || 'false' }}" echo "" echo "## Per-probe results" echo "" echo '```' - grep -E '^FT-(CASE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)" + grep -E '^FT-(CASE|COMPLETE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)" echo '```' echo "" echo "## Health snapshots" @@ -174,19 +190,30 @@ jobs: done } > "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" - # Product gate: failing FT cases keep the run green — the report above and - # the backlog issue manager carry the signal. Only harness/environment - # breakdowns (no case verdicts at all) turn the workflow red. - FT_CASES="$(grep -cE '^FT-CASE:' "${LOG_FILE}" 2>/dev/null || true)" - FT_FAILED="$(grep -cE '^FT-CASE: .* verdict=UNEXPECTED' "${LOG_FILE}" 2>/dev/null || true)" - FT_PASSED=$(( ${FT_CASES:-0} - ${FT_FAILED:-0} )) - HARNESS_OK=0 - if [ "${{ steps.test.outcome }}" = "success" ]; then - HARNESS_OK=1 - elif [ "${{ steps.test.outcome }}" = "failure" ] && [ "${FT_CASES}" -gt 0 ] && [ "${FT_PASSED}" -ge 1 ]; then - HARNESS_OK=1 - fi - [ "${HARNESS_OK}" = "1" ] + printf 'complete=false\n' >> "${GITHUB_OUTPUT}" + jq -e --arg testing_sha "${AUTO_TESTING_SHA}" ' + .schema_version == 1 and + .auto_testing_sha == $testing_sha and + .selected_scenarios == ["A", "B", "C", "C2", "D"] and + .complete == true and + .counts.expected == 38 and + .counts.seen == 38 and + .counts.harness_errors == 0 and + (.expected_cases | length) == 38 and + (.expected_cases | unique | length) == 38 and + (.seen_cases | length) == 38 and + (.seen_cases | unique | length) == 38 and + (.seen_cases | sort) == (.expected_cases | sort) and + (.missing_cases | length) == 0 and + (.harness_errors | length) == 0 and + (.cases | length) == 38 and + ([.cases[].case_id] | sort) == (.expected_cases | sort) and + ([.cases[].case_id] | unique | length) == 38 and + all(.cases[]; .verdict == "pass" or .verdict == "known-divergence" or .verdict == "UNEXPECTED") and + .counts.unexpected == ([.cases[] | select(.verdict == "UNEXPECTED")] | length) and + .counts.known_divergence == ([.cases[] | select(.verdict == "known-divergence")] | length) + ' "${RESULTS_FILE}" >/dev/null + printf 'complete=true\n' >> "${GITHUB_OUTPUT}" - name: Manage backlog issues (dedup / label / auto-close) timeout-minutes: 2 @@ -196,7 +223,7 @@ jobs: # green run. Never acts on cancelled runs. Logic lives in # auto-testing/scripts/issue_manager.py, which parses the # FT-CASE verdict lines from the suite log. - if: ${{ always() && steps.evidence.outcome == 'success' }} + if: ${{ always() && steps.evidence.outcome == 'success' && steps.chain_report.outputs.complete == 'true' }} continue-on-error: true env: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} @@ -272,6 +299,7 @@ jobs: name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }} path: | ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md + ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/results.json ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log ${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/ if-no-files-found: warn diff --git a/scripts/test_functional_chain.py b/scripts/test_functional_chain.py index 2fe1016cf..4d737781a 100644 --- a/scripts/test_functional_chain.py +++ b/scripts/test_functional_chain.py @@ -363,7 +363,12 @@ class RunnerTests(unittest.TestCase): self.assertIn(" workflow_call:", workflow) self.assertIn(" workflow_dispatch:", workflow) self.assertIn("group: rustfs-shared-functional-tests-v2", workflow) - self.assertIn("ref: ${{ steps.chain.outputs.testing_sha || 'main' }}", workflow) + expected_ref = ( + "ref: ${{ steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main' }}" + if suite == "fault-tolerance" + else "ref: ${{ steps.chain.outputs.testing_sha || 'main' }}" + ) + self.assertIn(expected_ref, workflow) self.assertIn("steps.chain_package.outputs.package_url || inputs.package_url", workflow) self.assertLess(workflow.index("prepare_functional_package.py prepare"), workflow.index("id: test")) self.assertIn("prepare_functional_package.py cleanup", workflow) diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py index 041f8512a..544603851 100644 --- a/scripts/test_security_workflow.py +++ b/scripts/test_security_workflow.py @@ -405,7 +405,7 @@ class FunctionalWorkflowTests(unittest.TestCase): "kms": "Run KMS suite", "storage": "Run storage engine suite", "s3-compat": "Run S3 compatibility suite", "upgrade": "Run upgrade compatibility suite", "replication": "Run replication suite", - "table": "Run table suite", "fault-tolerance": "Run fault-tolerance scenarios (A, B, C, C2)", + "table": "Run table suite", "fault-tolerance": "Run fault-tolerance scenarios (A, B, C, C2, D)", } def test_failure_and_always_step_wiring(self) -> None: @@ -504,6 +504,145 @@ class FunctionalWorkflowTests(unittest.TestCase): self.assertEqual(markers, ["cleanup", "dispatch"]) +class FaultToleranceWorkflowContractTests(unittest.TestCase): + TESTING_SHA = "a" * 40 + + def setUp(self) -> None: + source = (ROOT / ".github/workflows/rustfs-fault-tolerance-test.yml").read_text() + job = yaml_block(source.splitlines(), "fault-tolerance-test", 2) + self.assertIsNotNone(job) + self.steps = named_steps(job) + self.report_body = shell_body(self.steps["Generate report"]) + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.artifacts = self.directory / "artifacts" + (self.artifacts / "evidence").mkdir(parents=True) + self.log = self.artifacts / "suite.log" + self.log.write_text("FT-REPORT: fixture\n") + self.results = self.artifacts / "results.json" + self.output = self.directory / "github-output" + self.env = { + **os.environ, + "AUTO_TESTING_SHA": self.TESTING_SHA, + "FUNCTIONAL_ARTIFACTS_DIR": str(self.artifacts), + "LOG_FILE": str(self.log), + "REPORT_FILE": str(self.artifacts / "report.md"), + "RESULTS_FILE": str(self.results), + "GITHUB_OUTPUT": str(self.output), + "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), + "GITHUB_SERVER_URL": "https://github.com", + "GITHUB_REPOSITORY": "rustfs/rustfs", + "GITHUB_RUN_ID": "314159", + } + + def write_result(self, *, seen: int = 38, complete: bool = True, + duplicate: bool = False, testing_sha: str | None = None, + unrelated_seen: bool = False, missing_verdict: bool = False) -> None: + expected = [f"CASE-{index:02d}" for index in range(38)] + seen_cases = expected[:seen] + if unrelated_seen: + seen_cases = [f"OTHER-{index:02d}" for index in range(seen)] + case_ids = list(seen_cases) + if unrelated_seen: + case_ids = expected[:seen] + if duplicate: + case_ids[-1] = case_ids[0] + payload = { + "schema_version": 1, + "auto_testing_sha": testing_sha or self.TESTING_SHA, + "selected_scenarios": ["A", "B", "C", "C2", "D"], + "complete": complete, + "expected_cases": expected, + "seen_cases": seen_cases, + "missing_cases": expected[seen:], + "harness_errors": [], + "counts": { + "expected": 38, + "seen": seen, + "unexpected": 1, + "known_divergence": 0, + "harness_errors": 0, + }, + "cases": [ + {"case_id": case_id, "verdict": "UNEXPECTED" if index == 0 else "pass", "detail": "fixture"} + for index, case_id in enumerate(case_ids) + ], + } + if missing_verdict: + payload["cases"][0].pop("verdict") + self.results.write_text(json.dumps(payload)) + + def run_report(self) -> subprocess.CompletedProcess[str]: + replacements = { + "inputs.package_url || 'nightly (R2 latest)'": "fixture-package", + "inputs.strict || 'false'": "false", + } + rendered = re.sub( + r"\$\{\{\s*(.*?)\s*\}\}", + lambda match: replacements[match[1]], + self.report_body, + ) + return subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", rendered], + cwd=self.directory, + env=self.env, + capture_output=True, + text=True, + ) + + def test_complete_product_failure_is_valid_harness_evidence(self) -> None: + self.write_result() + result = self.run_report() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(self.output.read_text().splitlines()[-1], "complete=true") + + def test_partial_or_missing_results_fail_the_harness_gate(self) -> None: + self.write_result(seen=17, complete=False) + partial = self.run_report() + self.assertNotEqual(partial.returncode, 0) + self.assertEqual(self.output.read_text().splitlines()[-1], "complete=false") + + self.results.unlink() + self.output.unlink() + missing = self.run_report() + self.assertNotEqual(missing.returncode, 0) + self.assertEqual(self.output.read_text().splitlines()[-1], "complete=false") + + def test_duplicate_cases_or_revision_mismatch_fail_the_harness_gate(self) -> None: + self.write_result(duplicate=True) + self.assertNotEqual(self.run_report().returncode, 0) + + self.output.unlink() + self.write_result(testing_sha="b" * 40) + self.assertNotEqual(self.run_report().returncode, 0) + + def test_missing_verdict_or_unrelated_seen_cases_fail_the_harness_gate(self) -> None: + self.write_result(missing_verdict=True) + self.assertNotEqual(self.run_report().returncode, 0) + + self.output.unlink() + self.write_result(unrelated_seen=True) + self.assertNotEqual(self.run_report().returncode, 0) + + def test_backlog_manager_only_runs_after_complete_report(self) -> None: + condition = next( + line.strip() for line in self.steps["Manage backlog issues (dedup / label / auto-close)"] + if line.startswith(" if:") + ) + self.assertEqual( + condition, + "if: ${{ always() && steps.evidence.outcome == 'success' && steps.chain_report.outputs.complete == 'true' }}", + ) + + def test_auto_testing_ref_is_resolved_once_and_recorded(self) -> None: + checkout = "\n".join(self.steps["Checkout auto-testing scripts"]) + record = "\n".join(self.steps["Record auto-testing revision"]) + self.assertIn("steps.chain.outputs.testing_sha || inputs.auto_testing_ref || 'main'", checkout) + self.assertIn("git -C auto-testing rev-parse HEAD", record) + self.assertIn("AUTO_TESTING_SHA", record) + + class FunctionalCaseReportTests(unittest.TestCase): def report(self, text: str | None, matrix: bool = False) -> tuple[bool, str, str]: with tempfile.TemporaryDirectory() as directory: @@ -727,7 +866,7 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase): def test_upload_allowlist_preserves_diagnostics_without_scratch(self): extra = { "kms": ["cases.md"], "storage": ["cases.md"], "s3-compat": ["cases.md"], - "upgrade": ["cases.md", "matrix.md"], "replication": ["cases.md"], "heal": ["steps.md", "warp.log"], "table": ["cases.md"], "fault-tolerance": [], + "upgrade": ["cases.md", "matrix.md"], "replication": ["cases.md"], "heal": ["steps.md", "warp.log"], "table": ["cases.md"], "fault-tolerance": ["results.json"], "performance": ["version.txt", "results/master.log", "results/summary.md", "results/summary.tsv", "results/get_1KiB.txt", "results/put_1MiB.txt", "results/mixed_4MiB.txt"], }