gsadmin/pulse
1
0
Fork 0
mirror of https://github.com/rcourtman/Pulse.git synced 2026-11-04 09:34:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fail closed on missing backend CI admission (#3452)

Browse Source 
Admit backend runtime or a deliberate skip only from explicit,
consistent classifier and compiler results. Keep every required shard
and runtime suite, reject unavailable flags/results, and distinguish an
inapplicable API run from executed test success. Exercise both actual
matrix gates and the aggregate with adverse-result fixtures.

---
Fleet candidate `20261028T152025Z-delivery-trust` from the
`delivery-trust` lane. Reviewed tip
`37ac098542c98b97922f46a5c8d6ec79eb01549f` on `main`.

[How Pulse handles
triage](https://github.com/rcourtman/Pulse/blob/main/docs/AI_TRANSPARENCY.md)
This commit is contained in:
pulse-triage[bot]
2026-10-28 16:23:05 +00:00
committed by GitHub
parent 864ce12afa 37ac098542
commit 1ca941d4e6
4 changed files with 122 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/build-and-test.yml
+38 -16
View File
@@ -471,17 +471,23 @@ jobs:
steps:
- name: Require compiled backend tests
env:
CODE_CHANGED: ${{ needs.changes.outputs.code }}
COMPILE_REQUIRED: ${{ needs.changes.outputs.go_compile }}
COMPILE_RESULT: ${{ needs.test-compile.result }}
run: |
if [ "$COMPILE_REQUIRED" = true ] && [ "$COMPILE_RESULT" != success ]; then
echo "::error::Backend test compilation did not pass (result: ${COMPILE_RESULT:-unknown}); no runtime tests started."
exit 1
fi
# Only explicit, consistent classifier/compiler results admit tests
# or a deliberate skip. Missing outputs must not look like false.
case "$CODE_CHANGED:$COMPILE_REQUIRED:$COMPILE_RESULT" in
true:true:success|true:false:skipped|false:false:skipped) ;;
*)
echo "::error::Backend CI admission is incomplete or inconsistent. Inspect Detect changed areas and Compile backend race tests; no runtime tests started."
exit 1
;;
esac
- name: Report documentation-only change
if: needs.changes.outputs.code != 'true'
run: echo "No backend code changed. Tests are not required for this shard."
- name: Report inapplicable backend runtime
if: needs.changes.outputs.code == 'false'
run: echo "Application runtime checks are not applicable to this classified change; no backend tests ran in this shard."
- name: Checkout repository
if: needs.changes.outputs.code == 'true'
@@ -578,17 +584,23 @@ jobs:
steps:
- name: Require compiled backend tests
env:
CODE_CHANGED: ${{ needs.changes.outputs.code }}
COMPILE_REQUIRED: ${{ needs.changes.outputs.go_compile }}
COMPILE_RESULT: ${{ needs.test-compile.result }}
run: |
if [ "$COMPILE_REQUIRED" = true ] && [ "$COMPILE_RESULT" != success ]; then
echo "::error::Backend test compilation did not pass (result: ${COMPILE_RESULT:-unknown}); no runtime tests started."
exit 1
fi
# Only explicit, consistent classifier/compiler results admit tests
# or a deliberate skip. Missing outputs must not look like false.
case "$CODE_CHANGED:$COMPILE_REQUIRED:$COMPILE_RESULT" in
true:true:success|true:false:skipped|false:false:skipped) ;;
*)
echo "::error::Backend CI admission is incomplete or inconsistent. Inspect Detect changed areas and Compile backend race tests; no runtime tests started."
exit 1
;;
esac
- name: Report documentation-only change
if: needs.changes.outputs.code != 'true'
run: echo "No backend code changed. Tests are not required for this shard."
- name: Report inapplicable backend runtime
if: needs.changes.outputs.code == 'false'
run: echo "Application runtime checks are not applicable to this classified change; no backend tests ran in this shard."
- name: Checkout repository
if: needs.changes.outputs.code == 'true'
@@ -700,20 +712,30 @@ jobs:
# internal/api shard succeeded (a skipped or cancelled shard fails it too).
backend-api-verdict:
name: Backend tests (api)
needs: backend-api
needs: [changes, backend-api]
if: always()
runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
- name: Require every internal/api shard
env:
CHANGE_RESULT: ${{ needs.changes.result }}
CODE_CHANGED: ${{ needs.changes.outputs.code }}
API_SHARDS_RESULT: ${{ needs.backend-api.result }}
run: |
if [ "$CHANGE_RESULT" != success ]; then
echo "::error::Changed-area classification did not pass; backend runtime cannot be verified."
exit 1
fi
if [ "$API_SHARDS_RESULT" != success ]; then
echo "::error::internal/api shards did not all pass (result: ${API_SHARDS_RESULT})"
exit 1
fi
echo "Every internal/api shard passed."
case "$CODE_CHANGED" in
true) echo "Every internal/api runtime shard passed." ;;
false) echo "Application runtime checks are not applicable to this classified change; no internal/api tests ran." ;;
*) echo "::error::Missing or invalid changed-area classification; backend runtime cannot be verified."; exit 1 ;;
esac
script-smoke:
name: Script smoke tests (shard ${{ matrix.shard }}/2)
docs/release-control/v6/internal/subsystems/deployment-installability.md
+20 -8
View File
@@ -8736,18 +8736,30 @@ full compilation. Frontend-only changes do not pay for this Go-only preflight
checks still run. `scripts/go-test-compile-required.sh` consumes the complete
file list without an early-close/SIGPIPE path.
A failed, cancelled, skipped or unknown compile result when compilation is required must
fail the unchanged required backend check names, not let dependency skipping
report success. Documentation-only changes still expand those named checks and
accept the deliberately skipped compiler, as do changes outside Go compilation. Frontend and independent secret-scan
work do not wait for compilation; their existing audit rules remain unchanged.
Both backend matrices admit only three explicit classifier/compiler tuples:
code=true/compile=true/success, code=true/compile=false/skipped, or
code=false/compile=false/skipped. Missing, invalid or inconsistent outputs and
failed, cancelled, skipped or unknown required compilation fail the unchanged
required backend check names before checkout/runtime work. Missing output must
not be interpreted as an intentional false flag. Documentation-, receipt- and
triage-only changes still expand those named checks and accept the deliberately
skipped compiler. Frontend and independent secret-scan work do not wait for
compilation; their existing audit rules remain unchanged.
The API aggregate also requires successful classification and every expanded
shard. It reports runtime success only for code=true; code=false explicitly
reports that no API tests ran because runtime checks are inapplicable. An
unknown classifier or skipped/failed/cancelled shard never reports success.
`scripts/tests/test_go_test_compilation.py` exercises the actual compiler on
inert multi-package fixtures, rejects a test-only removed API and a normal-source
compile defect, and proves init/TestMain/test/example code is never executed.
It executes both matrix prerequisite scripts against successful and adverse
results and the documentation-only skip. `TestCITestCompilationPreservesChecks`
retains the required names, full race execution and separate dependency audit.
It executes both actual matrix prerequisite scripts and the API aggregate over
successful, missing, invalid, inconsistent, failed, cancelled and deliberately
skipped results, including the distinct no-tests message.
`TestCITestCompilationPreservesChecks` executes these admission/verdict controls
as well as retaining the required names, full race execution and separate
dependency audit.
These proofs establish CI source admission, not hosted queue-time relief.
scripts/installtests/build_release_assets_test.go
+11 -1
View File
@@ -4273,8 +4273,10 @@ func TestBackendAPIShardsKeepRequiredCheckExhaustive(t *testing.T) {
verdict := workflowJobBlock(t, workflow, "backend-api-verdict")
for _, required := range []string{
"name: Backend tests (api)\n",
"needs: backend-api",
"needs: [changes, backend-api]",
"if: always()",
"CHANGE_RESULT: ${{ needs.changes.result }}",
"CODE_CHANGED: ${{ needs.changes.outputs.code }}",
"API_SHARDS_RESULT: ${{ needs.backend-api.result }}",
`if [ "$API_SHARDS_RESULT" != success ]; then`,
} {
@@ -5646,6 +5648,14 @@ func TestCITestCompilationPreservesChecks(t *testing.T) {
if strings.Count(workflow, "name: Require compiled backend tests") != 2 {
t.Fatal("both required backend matrices must report compilation failures")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_go_test_compilation.py"), "-v",
"GoTestCompilationTest.test_required_checks_admit_only_explicit_consistent_results",
"GoTestCompilationTest.test_actual_api_verdict_rejects_unknown_results_and_distinguishes_no_tests")
if out, err := cmd.CombinedOutput(); err != nil {
t.Fatalf("required backend admission and truthful verdict controls: %v\n%s", err, out)
}
for _, path := range []string{
repoFile("scripts", "compile-go-tests.sh"),
repoFile("scripts", "tests", "test_go_test_compilation.py"),
scripts/tests/test_go_test_compilation.py
+53 -11
View File
@@ -130,25 +130,37 @@ func TestStaleFixture(t *testing.T) { _ = api.Resource{Alerts: []api.ResourceAle
# Frontend work does not wait for the Go-only gate.
self.assertIn("needs: changes", job(source, "frontend"))
def test_required_checks_fail_on_bad_compile_and_pass_docs_only_skip(self):
def test_required_checks_admit_only_explicit_consistent_results(self):
source = WORKFLOW.read_text()
for name in ("backend", "backend-api"):
section = job(source, name)
self.assertIn("needs: [changes, test-compile]", section)
self.assertIn("if: ${{ !cancelled() && needs.changes.result == 'success' }}", section)
self.assertIn("CODE_CHANGED: ${{ needs.changes.outputs.code }}", section)
self.assertIn("COMPILE_REQUIRED: ${{ needs.changes.outputs.go_compile }}", section)
self.assertIn("COMPILE_RESULT: ${{ needs.test-compile.result }}", section)
self.assertLess(section.index("Require compiled backend tests"), section.index("Checkout repository"))
script = prerequisite_script(section)
for code in ("true", "false"):
for result in ("success", "failure", "cancelled", "skipped", ""):
with self.subTest(job=name, code=code, result=result):
observed = subprocess.run(
["bash", "-euo", "pipefail", "-c", script],
env={**os.environ, "COMPILE_REQUIRED": code, "COMPILE_RESULT": result},
capture_output=True, text=True,
)
expected = 0 if code == "false" or result == "success" else 1
self.assertEqual(observed.returncode, expected, observed.stdout + observed.stderr)
admitted = {("true", "true", "success"), ("true", "false", "skipped"),
("false", "false", "skipped")}
for code in ("true", "false", "", "unknown", "true\nfalse"):
for required in ("true", "false", "", "unknown"):
for result in ("success", "failure", "cancelled", "skipped", ""):
with self.subTest(job=name, code=code, required=required, result=result):
observed = subprocess.run(
["bash", "-euo", "pipefail", "-c", script],
env={**os.environ, "CODE_CHANGED": code,
"COMPILE_REQUIRED": required, "COMPILE_RESULT": result},
capture_output=True, text=True, timeout=5,
)
expected = 0 if (code, required, result) in admitted else 1
self.assertEqual(observed.returncode, expected, observed.stdout + observed.stderr)
if expected:
self.assertIn("no runtime tests started", observed.stdout)
self.assertIn("Detect changed areas", observed.stdout)
self.assertIn("- name: Report inapplicable backend runtime\n"
" if: needs.changes.outputs.code == 'false'", section)
self.assertIn("no backend tests ran in this shard", section)
# Compile-only success cannot replace these full race/vet runs.
self.assertIn("go test -race -timeout 50m", section)
self.assertIn("shard: [rest-0, rest-1]", job(source, "backend"))
@@ -156,6 +168,36 @@ func TestStaleFixture(t *testing.T) { _ = api.Resource{Alerts: []api.ResourceAle
self.assertIn("name: Backend tests (api)", job(source, "backend-api-verdict"))
self.assertIn("if: always()", job(source, "backend-api-verdict"))
def test_actual_api_verdict_rejects_unknown_results_and_distinguishes_no_tests(self):
section = job(WORKFLOW.read_text(), "backend-api-verdict")
self.assertIn("needs: [changes, backend-api]", section)
for binding in ("CHANGE_RESULT: ${{ needs.changes.result }}",
"CODE_CHANGED: ${{ needs.changes.outputs.code }}",
"API_SHARDS_RESULT: ${{ needs.backend-api.result }}"):
self.assertIn(binding, section)
script = section.split(" run: |\n", 1)[1]
script = "\n".join(line[10:] for line in script.splitlines())
for change in ("success", "failure", "cancelled", "skipped", ""):
for code in ("true", "false", "", "unknown"):
for shards in ("success", "failure", "cancelled", "skipped", ""):
with self.subTest(change=change, code=code, shards=shards):
observed = subprocess.run(
["bash", "-euo", "pipefail", "-c", script],
env={**os.environ, "CHANGE_RESULT": change,
"CODE_CHANGED": code, "API_SHARDS_RESULT": shards},
capture_output=True, text=True, timeout=5,
)
expected = 0 if change == shards == "success" and code in ("true", "false") else 1
self.assertEqual(observed.returncode, expected, observed.stdout + observed.stderr)
if expected == 0:
if code == "true":
self.assertIn("Every internal/api runtime shard passed", observed.stdout)
self.assertNotIn("not applicable", observed.stdout)
else:
self.assertIn("not applicable", observed.stdout)
self.assertIn("no internal/api tests ran", observed.stdout)
self.assertNotIn("passed", observed.stdout)
if __name__ == "__main__":
unittest.main()