From 2f02d1d2d8bd1026cc43dfd155586c894e7b266d Mon Sep 17 00:00:00 2001 From: Zhengchao An Date: Sat, 5 Sep 2026 16:26:13 +0800 Subject: [PATCH] fix(ci): preserve security suite failures and isolate reports (#7188) --- .config/make/tests.mak | 1 + .github/workflows/ci-docs-only.yml | 1 + .github/workflows/ci.yml | 1 + .github/workflows/rustfs-security-test.yml | 83 ++++++--- scripts/test_security_workflow.py | 197 +++++++++++++++++++++ 5 files changed, 255 insertions(+), 28 deletions(-) create mode 100644 scripts/test_security_workflow.py diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 3ee3337fa..d1297e9ad 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -40,6 +40,7 @@ script-tests: ## Run shell script tests $(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test + $(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py $(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py bash -n ./scripts/validate_object_data_cache_cold_stampede.sh $(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test diff --git a/.github/workflows/ci-docs-only.yml b/.github/workflows/ci-docs-only.yml index de78e7f7f..7c6ad22ec 100644 --- a/.github/workflows/ci-docs-only.yml +++ b/.github/workflows/ci-docs-only.yml @@ -129,6 +129,7 @@ jobs: run: | python3 ./scripts/check_test_wiring.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/test_security_workflow.py python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98fc6cc30..171f52380 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -167,6 +167,7 @@ jobs: run: | python3 ./scripts/check_test_wiring.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test + python3 ./scripts/test_security_workflow.py python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml index 2b2a5d68b..16e10a80c 100644 --- a/.github/workflows/rustfs-security-test.yml +++ b/.github/workflows/rustfs-security-test.yml @@ -74,10 +74,23 @@ env: jobs: security-test: runs-on: smoke-testing - continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + - name: Checkout repository (for the OIDC live gate script) + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Initialize security evidence + id: evidence + run: | + set -euo pipefail + umask 077 + SECURITY_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-security-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" + mkdir -- "${SECURITY_ARTIFACTS_DIR}" + printf 'SECURITY_ARTIFACTS_DIR=%s\n' "${SECURITY_ARTIFACTS_DIR}" >> "${GITHUB_ENV}" + # auto-testing is private: clone it with the dedicated PF token (not # GITHUB_TOKEN) and retry transient GitHub/network failures. - name: Checkout auto-testing scripts (with retry) @@ -98,11 +111,6 @@ jobs: echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2 exit 1 - - name: Checkout repository (for the OIDC live gate script) - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - persist-credentials: false - - name: Show environment run: | uname -a @@ -135,7 +143,8 @@ jobs: id: test continue-on-error: true env: - REPORT_FILE: /tmp/rustfs-security-report.md + REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md + TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }} RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh run: | set -euo pipefail @@ -159,29 +168,48 @@ jobs: else ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}") fi - ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" + GITHUB_STEP_SUMMARY=/dev/null ./auto-testing/rustfs-security-test.sh "${ARGS[@]}" - name: Generate report - if: always() + id: report + if: ${{ always() && steps.evidence.outcome == 'success' }} + env: + TEST_OUTCOME: ${{ steps.test.outcome }} run: | set -euo pipefail - if [ ! -f /tmp/rustfs-security-report.md ]; then - { - echo "# RustFS security test report" - echo "" - echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - echo "- Trigger: ${{ github.event_name }}" - echo "- Test Step Outcome: failure (suite did not produce a report)" - } > /tmp/rustfs-security-report.md + RESULT=failure + if [ "${TEST_OUTCOME}" = "success" ] && [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then + RESULT=success fi - cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}" + { + echo "# RustFS security test report" + echo "" + echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + echo "- Attempt: ${GITHUB_RUN_ATTEMPT}" + echo "- Workflow Commit: ${GITHUB_SHA}" + echo "- Trigger: ${GITHUB_EVENT_NAME}" + echo "- Test Step Outcome: ${RESULT}" + echo "- Suite Step Outcome: ${TEST_OUTCOME}" + echo "" + # The dashboard prioritizes case rows over the step outcome. + # Keep partial case results in the artifact when the suite fails. + if [ "${RESULT}" = "success" ]; then + cat "${SECURITY_ARTIFACTS_DIR}/suite-report.md" + elif [ -s "${SECURITY_ARTIFACTS_DIR}/suite-report.md" ]; then + echo "The suite did not complete successfully. See suite-report.md in this run's artifact for diagnostics." + else + echo "The suite did not produce a non-empty report." + fi + } > "${SECURITY_ARTIFACTS_DIR}/report.md" + cat "${SECURITY_ARTIFACTS_DIR}/report.md" >> "${GITHUB_STEP_SUMMARY}" + [ "${RESULT}" = "success" ] - name: Upload functional report to dashboard - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} continue-on-error: true env: GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} - REPORT_FILE: /tmp/rustfs-security-report.md + REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md SUITE: security run: | set -euo pipefail @@ -210,8 +238,9 @@ jobs: GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} SUITE: 'security' SUITE_LABEL: 'Security' + EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - REPORT_FILE: '/tmp/rustfs-security-report.md' + REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md LOG_FILE: '' run: | set -euo pipefail @@ -245,7 +274,7 @@ jobs: echo "" echo "## Report (errors and symptoms)" echo "" - if [ -s "${REPORT_FILE}" ]; then + if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then redact < "${REPORT_FILE}" elif [ -s "${LOG_FILE:-}" ]; then echo "(report file missing; log tail below)" @@ -263,14 +292,12 @@ jobs: echo "filed backlog issue for suite ${SUITE}" - name: Upload report and logs - if: always() + if: ${{ always() && steps.evidence.outcome == 'success' }} uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 with: - name: rustfs-security-test-${{ github.run_id }} - path: | - /tmp/rustfs-security-report.md - /tmp/rustfs-security.*/* - if-no-files-found: ignore + name: rustfs-security-test-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ env.SECURITY_ARTIFACTS_DIR }}/ + if-no-files-found: error retention-days: 3 - name: Cleanup environment (after) diff --git a/scripts/test_security_workflow.py b/scripts/test_security_workflow.py new file mode 100644 index 000000000..ae82d3fb1 --- /dev/null +++ b/scripts/test_security_workflow.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Run the security workflow's evidence and result steps without remote VMs.""" + +from __future__ import annotations + +import os +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + +from check_test_wiring import yaml_block + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github/workflows/rustfs-security-test.yml" +CASE_ROW = "| IAM-101 | user CRUD lifecycle | PASS |" + + +class SecurityWorkflowTests(unittest.TestCase): + def setUp(self) -> None: + self.source = WORKFLOW.read_text() + self.job = yaml_block(self.source.splitlines(), "security-test", 2) + self.assertIsNotNone(self.job) + starts = [i for i, line in enumerate(self.job) if line.startswith(" - name: ")] + self.steps = { + self.job[start].split(": ", 1)[1].strip('"'): self.job[start:end] + for start, end in zip(starts, starts[1:] + [len(self.job)]) + } + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.directory = Path(self.temp.name) + self.context = { + "runner.temp": self.temp.name, + "github.server_url": "https://github.com", + "github.repository": "rustfs/rustfs", + "github.run_id": "314159", + "github.run_attempt": "2", + "github.sha": "0123456789abcdef0123456789abcdef01234567", + "github.event_name": "workflow_dispatch", + "github.workspace": self.temp.name, + "inputs.package_url": "", + "inputs.rustfs_version": "test-version", + "inputs.topology": "all", + "inputs.oidc_live": "false", + "steps.evidence.outcome": "skipped", + "steps.test.outcome": "skipped", + "steps.report.outcome": "skipped", + } + self.env = { + **os.environ, "GITHUB_STEP_SUMMARY": str(self.directory / "summary.md"), + "GITHUB_ENV": str(self.directory / "github-env"), "RUNNER_TEMP": self.temp.name, "TMPDIR": self.temp.name, + } + for key in ("server_url", "repository", "run_id", "run_attempt", "sha", "event_name"): + self.env[f"GITHUB_{key.upper()}"] = self.context[f"github.{key}"] + self.context["env.SECURITY_ARTIFACTS_DIR"] = "" + self.artifacts = self.directory / "rustfs-security-314159-2" + suite = self.directory / "auto-testing/rustfs-security-test.sh" + suite.parent.mkdir() + suite.write_text( + '#!/usr/bin/env bash\nset -euo pipefail\n' + 'log_dir=$(mktemp -d "$TMPDIR/rustfs-security.XXXXXX")\n' + 'echo "CURRENT SUITE LOG" > "$log_dir/suite.log"\n' + 'case "$FAKE_REPORT" in\n' + f' present) printf "%s\\n" "CURRENT SUITE DIAGNOSTIC" "{CASE_ROW}" > "$REPORT_FILE" ;;\n' + ' empty) : > "$REPORT_FILE" ;;\n' + 'esac\n' + 'echo "UNWRAPPED SUITE SUMMARY" >> "$GITHUB_STEP_SUMMARY"\n' + 'exit "$FAKE_EXIT"\n' + ) + + def render(self, value: str) -> str: + return re.sub(r"\$\{\{\s*(.*?)\s*\}\}", lambda match: self.context[match[1]], value) + + def step_env(self, lines: list[str], indent: int = 8) -> dict[str, str]: + result = {} + for line in yaml_block(lines, "env", indent) or []: + if line.strip() and not line.lstrip().startswith("#"): + key, value = line.strip().split(": ", 1) + result[key] = self.render(value.strip("'\"")) + return result + + def run_step(self, name: str) -> subprocess.CompletedProcess[str]: + lines = self.steps[name] + start = lines.index(" run: |") + 1 + shell_lines = [] + for line in lines[start:]: + if line.strip() and not line.startswith(" "): + break + shell_lines.append(line[10:]) + self.assertTrue(shell_lines, f"missing literal shell body: {name}") + result = subprocess.run( + ["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.render("\n".join(shell_lines))], + cwd=self.directory, env={**self.env, **self.step_env(lines)}, capture_output=True, text=True, + ) + for line in lines: + if line.startswith(" id: "): + self.context[f"steps.{line.split(': ', 1)[1]}.outcome"] = "failure" if result.returncode else "success" + if Path(self.env["GITHUB_ENV"]).exists(): + for line in Path(self.env["GITHUB_ENV"]).read_text().splitlines(): + key, value = line.split("=", 1) + self.env[key] = value + self.context[f"env.{key}"] = value + return result + + def test_workflow_wiring(self) -> None: + names = list(self.steps) + self.assertLess(names.index("Checkout repository (for the OIDC live gate script)"), names.index("Checkout auto-testing scripts (with retry)")) + self.assertNotIn(" continue-on-error: true", self.job) + self.assertIn(" continue-on-error: true", self.steps["Run security suite"]) + for name in ("Initialize security evidence", "Generate report"): + self.assertNotIn(" continue-on-error: true", self.steps[name]) + self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps["Generate report"]) + self.assertNotIn("/tmp/rustfs-security", self.source) + for name in ("Upload functional report to dashboard", "File failure issue in rustfs/backlog"): + report = next(line for line in self.steps[name] if line.strip().startswith("REPORT_FILE:")) + self.assertIn("${{ env.SECURITY_ARTIFACTS_DIR }}/report.md", report) + for name in ("Upload functional report to dashboard", "Upload report and logs"): + self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name]) + artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8) + self.assertIn(" path: ${{ env.SECURITY_ARTIFACTS_DIR }}/", artifact_settings) + self.assertIn(" if-no-files-found: error", artifact_settings) + + def test_suite_report_and_result_matrix(self) -> None: + for outcome, mode, exit_code in ( + ("success", "present", 0), ("failure", "present", 7), ("failure", "missing", 7), + ("success", "missing", 0), ("success", "empty", 0), + ("skipped", "missing", 0), ("skipped", "present", 0), + ("cancelled", "missing", 0), ("cancelled", "present", 0), + ): + with self.subTest(outcome=outcome, report=mode): + self.setUp() + initialized = self.run_step("Initialize security evidence") + self.assertEqual(initialized.returncode, 0, initialized.stderr) + self.assertEqual(self.env["SECURITY_ARTIFACTS_DIR"], str(self.artifacts)) + self.env.update(FAKE_REPORT=mode, FAKE_EXIT=str(exit_code)) + if outcome != "skipped" or mode == "present": + suite = self.run_step("Run security suite") + self.assertEqual(suite.returncode, exit_code, suite.stderr) + logs = list(self.artifacts.glob("rustfs-security.*/suite.log")) + self.assertEqual(len(logs), 1) + self.assertEqual(logs[0].read_text(), "CURRENT SUITE LOG\n") + self.context["steps.test.outcome"] = outcome + report = self.run_step("Generate report") + success = outcome == "success" and mode == "present" + self.assertEqual(report.returncode == 0, success, report.stderr) + contents = (self.artifacts / "report.md").read_text() + for expected in ( + "https://github.com/rustfs/rustfs/actions/runs/314159", "Attempt: 2", + f"Workflow Commit: {self.context['github.sha']}", "Trigger: workflow_dispatch", + f"Test Step Outcome: {'success' if success else 'failure'}", f"Suite Step Outcome: {outcome}", + ): + self.assertIn(expected, contents) + self.assertEqual(CASE_ROW in contents, success) + self.assertEqual("CURRENT SUITE DIAGNOSTIC" in contents, success) + if mode == "present": + raw = (self.artifacts / "suite-report.md").read_text() + self.assertEqual(raw, f"CURRENT SUITE DIAGNOSTIC\n{CASE_ROW}\n") + summary = Path(self.env["GITHUB_STEP_SUMMARY"]).read_text() + self.assertEqual(summary, contents) + self.assertNotIn("UNWRAPPED SUITE SUMMARY", summary) + + def test_existing_evidence_directory_is_rejected(self) -> None: + self.artifacts.mkdir() + stale = self.artifacts / "suite-report.md" + stale.write_text("OLD RUN REPORT") + self.assertNotEqual(self.run_step("Initialize security evidence").returncode, 0) + self.assertEqual(stale.read_text(), "OLD RUN REPORT") + self.assertFalse(Path(self.env["GITHUB_ENV"]).exists()) + (self.artifacts / "report.md").write_text("OLD RUN REPORT") + self.context.update({ + "env.SECURITY_ARTIFACTS_DIR": str(self.artifacts), "secrets.PF_TESTING_GH_TOKEN": "fake-local-token", + }) + fake_bin = self.directory / "bin" + fake_bin.mkdir() + gh = fake_bin / "gh" + gh.write_text( + '#!/usr/bin/env bash\nset -euo pipefail\n' + 'if [ "$1 $2" = "issue create" ]; then\n' + ' while [ "$#" -gt 0 ]; do\n' + ' if [ "$1" = "--body-file" ]; then cat "$2" > "$CAPTURE_BODY"; fi\n' + ' shift\n' + ' done\n' + 'fi\n' + ) + gh.chmod(0o755) + body = self.directory / "issue-body.md" + self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body)) + result = self.run_step("File failure issue in rustfs/backlog") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("OLD RUN REPORT", body.read_text()) + self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text()) + + +if __name__ == "__main__": + unittest.main()