From 37ac098542c98b97922f46a5c8d6ec79eb01549f Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 15:30:38 +0000 Subject: [PATCH] Fail closed on missing backend CI admission Admit backend runtime or a deliberate skip only from explicit, consistent classifier and compiler results. Keep every required shard and runtime suite, reject unavailable flags/results, and distinguish an inapplicable API run from executed test success. Exercise both actual matrix gates and the aggregate with adverse-result fixtures. Change-source: pulse-maintainer --- .github/workflows/build-and-test.yml | 54 +++++++++++----- .../subsystems/deployment-installability.md | 28 +++++--- .../installtests/build_release_assets_test.go | 12 +++- scripts/tests/test_go_test_compilation.py | 64 +++++++++++++++---- 4 files changed, 122 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index c050f54f6c..08b20d0140 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -471,17 +471,23 @@ jobs: steps: - name: Require compiled backend tests env: + CODE_CHANGED: ${{ needs.changes.outputs.code }} COMPILE_REQUIRED: ${{ needs.changes.outputs.go_compile }} COMPILE_RESULT: ${{ needs.test-compile.result }} run: | - if [ "$COMPILE_REQUIRED" = true ] && [ "$COMPILE_RESULT" != success ]; then - echo "::error::Backend test compilation did not pass (result: ${COMPILE_RESULT:-unknown}); no runtime tests started." - exit 1 - fi + # Only explicit, consistent classifier/compiler results admit tests + # or a deliberate skip. Missing outputs must not look like false. + case "$CODE_CHANGED:$COMPILE_REQUIRED:$COMPILE_RESULT" in + true:true:success|true:false:skipped|false:false:skipped) ;; + *) + echo "::error::Backend CI admission is incomplete or inconsistent. Inspect Detect changed areas and Compile backend race tests; no runtime tests started." + exit 1 + ;; + esac - - name: Report documentation-only change - if: needs.changes.outputs.code != 'true' - run: echo "No backend code changed. Tests are not required for this shard." + - name: Report inapplicable backend runtime + if: needs.changes.outputs.code == 'false' + run: echo "Application runtime checks are not applicable to this classified change; no backend tests ran in this shard." - name: Checkout repository if: needs.changes.outputs.code == 'true' @@ -578,17 +584,23 @@ jobs: steps: - name: Require compiled backend tests env: + CODE_CHANGED: ${{ needs.changes.outputs.code }} COMPILE_REQUIRED: ${{ needs.changes.outputs.go_compile }} COMPILE_RESULT: ${{ needs.test-compile.result }} run: | - if [ "$COMPILE_REQUIRED" = true ] && [ "$COMPILE_RESULT" != success ]; then - echo "::error::Backend test compilation did not pass (result: ${COMPILE_RESULT:-unknown}); no runtime tests started." - exit 1 - fi + # Only explicit, consistent classifier/compiler results admit tests + # or a deliberate skip. Missing outputs must not look like false. + case "$CODE_CHANGED:$COMPILE_REQUIRED:$COMPILE_RESULT" in + true:true:success|true:false:skipped|false:false:skipped) ;; + *) + echo "::error::Backend CI admission is incomplete or inconsistent. Inspect Detect changed areas and Compile backend race tests; no runtime tests started." + exit 1 + ;; + esac - - name: Report documentation-only change - if: needs.changes.outputs.code != 'true' - run: echo "No backend code changed. Tests are not required for this shard." + - name: Report inapplicable backend runtime + if: needs.changes.outputs.code == 'false' + run: echo "Application runtime checks are not applicable to this classified change; no backend tests ran in this shard." - name: Checkout repository if: needs.changes.outputs.code == 'true' @@ -700,20 +712,30 @@ jobs: # internal/api shard succeeded (a skipped or cancelled shard fails it too). backend-api-verdict: name: Backend tests (api) - needs: backend-api + needs: [changes, backend-api] if: always() runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - name: Require every internal/api shard env: + CHANGE_RESULT: ${{ needs.changes.result }} + CODE_CHANGED: ${{ needs.changes.outputs.code }} API_SHARDS_RESULT: ${{ needs.backend-api.result }} run: | + if [ "$CHANGE_RESULT" != success ]; then + echo "::error::Changed-area classification did not pass; backend runtime cannot be verified." + exit 1 + fi if [ "$API_SHARDS_RESULT" != success ]; then echo "::error::internal/api shards did not all pass (result: ${API_SHARDS_RESULT})" exit 1 fi - echo "Every internal/api shard passed." + case "$CODE_CHANGED" in + true) echo "Every internal/api runtime shard passed." ;; + false) echo "Application runtime checks are not applicable to this classified change; no internal/api tests ran." ;; + *) echo "::error::Missing or invalid changed-area classification; backend runtime cannot be verified."; exit 1 ;; + esac script-smoke: name: Script smoke tests (shard ${{ matrix.shard }}/2) diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 59d5f766b0..34ec6da634 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -8736,18 +8736,30 @@ full compilation. Frontend-only changes do not pay for this Go-only preflight checks still run. `scripts/go-test-compile-required.sh` consumes the complete file list without an early-close/SIGPIPE path. -A failed, cancelled, skipped or unknown compile result when compilation is required must -fail the unchanged required backend check names, not let dependency skipping -report success. Documentation-only changes still expand those named checks and -accept the deliberately skipped compiler, as do changes outside Go compilation. Frontend and independent secret-scan -work do not wait for compilation; their existing audit rules remain unchanged. +Both backend matrices admit only three explicit classifier/compiler tuples: +code=true/compile=true/success, code=true/compile=false/skipped, or +code=false/compile=false/skipped. Missing, invalid or inconsistent outputs and +failed, cancelled, skipped or unknown required compilation fail the unchanged +required backend check names before checkout/runtime work. Missing output must +not be interpreted as an intentional false flag. Documentation-, receipt- and +triage-only changes still expand those named checks and accept the deliberately +skipped compiler. Frontend and independent secret-scan work do not wait for +compilation; their existing audit rules remain unchanged. + +The API aggregate also requires successful classification and every expanded +shard. It reports runtime success only for code=true; code=false explicitly +reports that no API tests ran because runtime checks are inapplicable. An +unknown classifier or skipped/failed/cancelled shard never reports success. `scripts/tests/test_go_test_compilation.py` exercises the actual compiler on inert multi-package fixtures, rejects a test-only removed API and a normal-source compile defect, and proves init/TestMain/test/example code is never executed. -It executes both matrix prerequisite scripts against successful and adverse -results and the documentation-only skip. `TestCITestCompilationPreservesChecks` -retains the required names, full race execution and separate dependency audit. +It executes both actual matrix prerequisite scripts and the API aggregate over +successful, missing, invalid, inconsistent, failed, cancelled and deliberately +skipped results, including the distinct no-tests message. +`TestCITestCompilationPreservesChecks` executes these admission/verdict controls +as well as retaining the required names, full race execution and separate +dependency audit. These proofs establish CI source admission, not hosted queue-time relief. diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 2e382c931a..f1acf951bd 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -4273,8 +4273,10 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) { verdict := workflowJobBlock(t, workflow, "backend-api-verdict") for _, required := range []string{ "name: Backend tests (api)\n", - "needs: backend-api", + "needs: [changes, backend-api]", "if: always()", + "CHANGE_RESULT: ${{ needs.changes.result }}", + "CODE_CHANGED: ${{ needs.changes.outputs.code }}", "API_SHARDS_RESULT: ${{ needs.backend-api.result }}", `if [ "$API_SHARDS_RESULT" != success ]; then`, } { @@ -5646,6 +5648,14 @@ func TestCITestCompilationPreservesChecks(t *testing.T) { if strings.Count(workflow, "name: Require compiled backend tests") != 2 { t.Fatal("both required backend matrices must report compilation failures") } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_go_test_compilation.py"), "-v", + "GoTestCompilationTest.test_required_checks_admit_only_explicit_consistent_results", + "GoTestCompilationTest.test_actual_api_verdict_rejects_unknown_results_and_distinguishes_no_tests") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("required backend admission and truthful verdict controls: %v\n%s", err, out) + } for _, path := range []string{ repoFile("scripts", "compile-go-tests.sh"), repoFile("scripts", "tests", "test_go_test_compilation.py"), diff --git a/scripts/tests/test_go_test_compilation.py b/scripts/tests/test_go_test_compilation.py index b8af39061e..853e8222ec 100644 --- a/scripts/tests/test_go_test_compilation.py +++ b/scripts/tests/test_go_test_compilation.py @@ -130,25 +130,37 @@ func TestStaleFixture(t *testing.T) { _ = api.Resource{Alerts: []api.ResourceAle # Frontend work does not wait for the Go-only gate. self.assertIn("needs: changes", job(source, "frontend")) - def test_required_checks_fail_on_bad_compile_and_pass_docs_only_skip(self): + def test_required_checks_admit_only_explicit_consistent_results(self): source = WORKFLOW.read_text() for name in ("backend", "backend-api"): section = job(source, name) self.assertIn("needs: [changes, test-compile]", section) self.assertIn("if: ${{ !cancelled() && needs.changes.result == 'success' }}", section) + self.assertIn("CODE_CHANGED: ${{ needs.changes.outputs.code }}", section) + self.assertIn("COMPILE_REQUIRED: ${{ needs.changes.outputs.go_compile }}", section) self.assertIn("COMPILE_RESULT: ${{ needs.test-compile.result }}", section) self.assertLess(section.index("Require compiled backend tests"), section.index("Checkout repository")) script = prerequisite_script(section) - for code in ("true", "false"): - for result in ("success", "failure", "cancelled", "skipped", ""): - with self.subTest(job=name, code=code, result=result): - observed = subprocess.run( - ["bash", "-euo", "pipefail", "-c", script], - env={**os.environ, "COMPILE_REQUIRED": code, "COMPILE_RESULT": result}, - capture_output=True, text=True, - ) - expected = 0 if code == "false" or result == "success" else 1 - self.assertEqual(observed.returncode, expected, observed.stdout + observed.stderr) + admitted = {("true", "true", "success"), ("true", "false", "skipped"), + ("false", "false", "skipped")} + for code in ("true", "false", "", "unknown", "true\nfalse"): + for required in ("true", "false", "", "unknown"): + for result in ("success", "failure", "cancelled", "skipped", ""): + with self.subTest(job=name, code=code, required=required, result=result): + observed = subprocess.run( + ["bash", "-euo", "pipefail", "-c", script], + env={**os.environ, "CODE_CHANGED": code, + "COMPILE_REQUIRED": required, "COMPILE_RESULT": result}, + capture_output=True, text=True, timeout=5, + ) + expected = 0 if (code, required, result) in admitted else 1 + self.assertEqual(observed.returncode, expected, observed.stdout + observed.stderr) + if expected: + self.assertIn("no runtime tests started", observed.stdout) + self.assertIn("Detect changed areas", observed.stdout) + self.assertIn("- name: Report inapplicable backend runtime\n" + " if: needs.changes.outputs.code == 'false'", section) + self.assertIn("no backend tests ran in this shard", section) # Compile-only success cannot replace these full race/vet runs. self.assertIn("go test -race -timeout 50m", section) self.assertIn("shard: [rest-0, rest-1]", job(source, "backend")) @@ -156,6 +168,36 @@ func TestStaleFixture(t *testing.T) { _ = api.Resource{Alerts: []api.ResourceAle self.assertIn("name: Backend tests (api)", job(source, "backend-api-verdict")) self.assertIn("if: always()", job(source, "backend-api-verdict")) + def test_actual_api_verdict_rejects_unknown_results_and_distinguishes_no_tests(self): + section = job(WORKFLOW.read_text(), "backend-api-verdict") + self.assertIn("needs: [changes, backend-api]", section) + for binding in ("CHANGE_RESULT: ${{ needs.changes.result }}", + "CODE_CHANGED: ${{ needs.changes.outputs.code }}", + "API_SHARDS_RESULT: ${{ needs.backend-api.result }}"): + self.assertIn(binding, section) + script = section.split(" run: |\n", 1)[1] + script = "\n".join(line[10:] for line in script.splitlines()) + for change in ("success", "failure", "cancelled", "skipped", ""): + for code in ("true", "false", "", "unknown"): + for shards in ("success", "failure", "cancelled", "skipped", ""): + with self.subTest(change=change, code=code, shards=shards): + observed = subprocess.run( + ["bash", "-euo", "pipefail", "-c", script], + env={**os.environ, "CHANGE_RESULT": change, + "CODE_CHANGED": code, "API_SHARDS_RESULT": shards}, + capture_output=True, text=True, timeout=5, + ) + expected = 0 if change == shards == "success" and code in ("true", "false") else 1 + self.assertEqual(observed.returncode, expected, observed.stdout + observed.stderr) + if expected == 0: + if code == "true": + self.assertIn("Every internal/api runtime shard passed", observed.stdout) + self.assertNotIn("not applicable", observed.stdout) + else: + self.assertIn("not applicable", observed.stdout) + self.assertIn("no internal/api tests ran", observed.stdout) + self.assertNotIn("passed", observed.stdout) + if __name__ == "__main__": unittest.main()