mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-26 21:47:20 +00:00
Integrate updater exit observations and real-embed CI verification
Preserve the exact three reviewed Delivery commits. Retained pipeline and workflow recipe controls cover this bounded diff; full source and installed restoration outcomes remain unproved. Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
@@ -269,6 +269,27 @@ jobs:
|
||||
working-directory: frontend-modern
|
||||
run: npm run test
|
||||
|
||||
# Use this checkout's production assets, not another job's embed stub.
|
||||
# Keep the existing job deadline, audit and full frontend tests gating.
|
||||
- name: Set up Go for real-embed verification
|
||||
uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
|
||||
with:
|
||||
go-version-file: go.mod
|
||||
cache: true
|
||||
|
||||
- name: Verify backend with real frontend embed
|
||||
run: |
|
||||
set -euo pipefail
|
||||
checkout_sha=$(git rev-parse HEAD)
|
||||
printf 'REAL_EMBED_CHECKOUT=%s\n' "$checkout_sha"
|
||||
test -s frontend-modern/dist/index.html
|
||||
test -d frontend-modern/dist/assets
|
||||
diff -qr --no-dereference frontend-modern/dist internal/api/frontend-modern/dist
|
||||
go version
|
||||
go build -mod=readonly -p=2 -o "$RUNNER_TEMP/pulse-real-embed" ./cmd/pulse
|
||||
go vet -mod=readonly -p=2 ./cmd/pulse ./internal/api
|
||||
sha256sum "$RUNNER_TEMP/pulse-real-embed"
|
||||
|
||||
# An audit service failure or real advisory stays gating, but it must not
|
||||
# suppress formatting, tests, type-checking, or production-build proof.
|
||||
- name: Require frontend dependency audit
|
||||
|
||||
@@ -83,6 +83,34 @@ matrix and smoke controls retain their cryptographic, metadata and return-trap
|
||||
coverage. These are source/fixture controls, not native systemd/LXC recovery,
|
||||
published installer trust, #2785's stop cause or installed acceptance.
|
||||
|
||||
The helper records the installer and log collector's separate observed exit
|
||||
statuses immediately after their pipeline, before logging or recovery can
|
||||
overwrite them. A failed collector does not establish installer failure;
|
||||
zero exits establish pipeline completion only, not version, service or update
|
||||
acceptance. The existing signed transaction and confined service-state tests
|
||||
cover installer-only, collector-only, simultaneous and successful exits with
|
||||
unchanged prior-activity, mask, backup and restoration safeguards. These facts
|
||||
cannot reconstruct exits or signals from an earlier interrupted attempt, or
|
||||
identify who stopped a host.
|
||||
|
||||
### Ordinary CI compiles the backend with the same checkout's real frontend
|
||||
|
||||
The existing Frontend job retains its production bundle, size, whole-tree tests,
|
||||
audit and 30-minute ceiling. After those tests it selects Go from `go.mod`,
|
||||
requires a nonempty built index/assets tree and compares the complete generated
|
||||
frontend and embed copy before backend compilation and server/API vet. Missing,
|
||||
extra or changed embedded files stop the phase; it never creates a placeholder
|
||||
or imports assets from another job/source. Build and vet use the read-only Go
|
||||
graph and two workers; the binary stays in the runner's temporary directory.
|
||||
The log records the checkout and binary digest without publishing an artifact.
|
||||
|
||||
`TestFrontendCIRequiresSameCheckoutRealEmbedBackend` executes the exact phase
|
||||
with private assets and Git/compiler doubles, covering mismatch refusal,
|
||||
declared command/ordering checks and terminal compiler/vet failures. These
|
||||
controls are not real compilation. Actual containing CI checkout/phase logs
|
||||
remain necessary before using the result as real-embed source proof; unrelated
|
||||
stub builds, workflow success, releases and installed recovery remain distinct.
|
||||
|
||||
### Core E2E images are built once and admitted in the same run
|
||||
|
||||
The secret-free Core E2E workflow builds its existing `e2e_runtime` server and
|
||||
|
||||
@@ -4308,6 +4308,20 @@ func assertInternalAPIShardSelectionExhaustive(t *testing.T, workflowShards int)
|
||||
}
|
||||
}
|
||||
|
||||
// Execute the existing CI phase with confined compiler/Git doubles, including
|
||||
// every-byte embed admission and terminal build/vet failures. This proves the
|
||||
// recipe and workflow wiring, not a real frontend/backend build.
|
||||
func TestFrontendCIRequiresSameCheckoutRealEmbedBackend(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "python3", repoFile("scripts", "tests", "test_frontend_real_embed.py"), "-v")
|
||||
cmd.Env = append(os.Environ(), "PULSE_BUILD_WORKFLOW_UNDER_TEST="+repoFile(".github", "workflows", "build-and-test.yml"))
|
||||
out, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
t.Fatalf("same-checkout real-embed CI admission controls: %v\n%s", err, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrontendDependencySecurityAuditsAreRequired(t *testing.T) {
|
||||
workflowPath := repoFile(".github", "workflows", "build-and-test.yml")
|
||||
assertFileContainsAll(t, workflowPath,
|
||||
|
||||
@@ -1020,6 +1020,17 @@ printf 'TRANSACTION_ASSERTIONS_PASSED\n'
|
||||
if !strings.Contains(string(out), "TRANSACTION_ASSERTIONS_PASSED") {
|
||||
t.Fatalf("transaction assertions did not finish:\n%s", out)
|
||||
}
|
||||
if !preflight {
|
||||
installerExit := "0"
|
||||
switch outcome {
|
||||
case "failure", "symlink", "directory", "fifo":
|
||||
installerExit = "23"
|
||||
}
|
||||
observed := "(installer exit: " + installerExit + "; log collector exit: 0)"
|
||||
if !strings.Contains(string(out), observed) {
|
||||
t.Fatalf("missing observed pipeline exits %q after signed installer execution:\n%s", observed, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stable admission is object-bound, not a line-oriented tag/flag accumulator.
|
||||
|
||||
@@ -574,6 +574,7 @@ perform_update() {
|
||||
log info "Installer signature verified"
|
||||
|
||||
local update_accepted="false"
|
||||
local -a installer_pipeline_status=()
|
||||
if env \
|
||||
"PULSE_SERVICE_NAME=$service_name" \
|
||||
"PULSE_INSTALL_DIR=$INSTALL_DIR" \
|
||||
@@ -582,8 +583,10 @@ perform_update() {
|
||||
while IFS= read -r line; do
|
||||
log info "installer: $line"
|
||||
done; then
|
||||
|
||||
log info "Installer completed; verifying update"
|
||||
# Capture both observed exits before any command overwrites PIPESTATUS.
|
||||
# Completion of this pipeline still requires binary/service verification.
|
||||
installer_pipeline_status=("${PIPESTATUS[@]}")
|
||||
log info "Installer completed; verifying update (installer exit: ${installer_pipeline_status[0]}; log collector exit: ${installer_pipeline_status[1]})"
|
||||
|
||||
# Verify new version
|
||||
local installed_version
|
||||
@@ -632,7 +635,10 @@ perform_update() {
|
||||
log error "Version mismatch after update. Expected: $new_version, Got: $installed_version"
|
||||
fi
|
||||
else
|
||||
log error "Update installation failed"
|
||||
installer_pipeline_status=("${PIPESTATUS[@]}")
|
||||
# A collector failure is not evidence that the installer itself failed.
|
||||
# These are exit observations, not a diagnosis of a host interruption.
|
||||
log error "Installer pipeline failed (installer exit: ${installer_pipeline_status[0]}; log collector exit: ${installer_pipeline_status[1]})"
|
||||
fi
|
||||
|
||||
if [[ "$update_accepted" == "true" ]]; then
|
||||
|
||||
@@ -66,7 +66,10 @@ sys.exit(97)
|
||||
|
||||
HARNESS = r'''
|
||||
source "$UPDATER"
|
||||
log() { printf '[%s] %s\n' "$1" "${*:2}"; }
|
||||
log() {
|
||||
printf '[%s] %s\n' "$1" "${*:2}"
|
||||
if [[ "${2:-}" == 'installer: '* ]]; then return "$COLLECTOR_EXIT"; fi
|
||||
}
|
||||
detect_service_name() { printf '%s\n' "$SERVICE_NAME"; }
|
||||
sleep() { :; } # Only the liveness wait; timeout/manager deadlines are real.
|
||||
verify_release_signature() { :; }
|
||||
@@ -105,7 +108,8 @@ exit "$status"
|
||||
|
||||
|
||||
class UpdateServiceState(unittest.TestCase):
|
||||
def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update"):
|
||||
def run_fixture(self, *, mode="normal", state="active", load="loaded", success=False, flow="update",
|
||||
installer_exit=None, collector_exit=0):
|
||||
with tempfile.TemporaryDirectory(prefix="pulse-updater-state-") as directory:
|
||||
f = Path(directory)
|
||||
for name in ("tools", "tmp", "install/bin", "config"):
|
||||
@@ -127,13 +131,15 @@ class UpdateServiceState(unittest.TestCase):
|
||||
'cp "$FIXTURE/new-pulse" "$PULSE_INSTALL_DIR/bin/pulse"\n'
|
||||
'echo v6.6.0 > "$PULSE_INSTALL_DIR/VERSION"\n'
|
||||
'echo "$INSTALLER_STATE" > "$FIXTURE/state"\n'
|
||||
'printf "fixture installation output\\n"\n'
|
||||
'exit "$INSTALLER_EXIT"\n')
|
||||
env = dict(os.environ)
|
||||
env.update(FIXTURE=str(f), UPDATER=str(UPDATER), MODE=mode, FLOW=flow,
|
||||
PULSE_INSTALL_DIR=str(f / "install"), PULSE_CONFIG_DIR=str(f / "config"),
|
||||
PULSE_SERVICE_NAME="pulse-custom@department", LOAD_STATE=load,
|
||||
INSTALLER_STATE="active" if mode in ("stop-noop", "stop-error", "stop-hang", "after-error", "readback-error", "readback-empty") else "inactive",
|
||||
INSTALLER_EXIT="0" if success else "23",
|
||||
INSTALLER_EXIT=str(installer_exit) if installer_exit is not None else ("0" if success else "23"),
|
||||
COLLECTOR_EXIT=str(collector_exit),
|
||||
PATH=str(f / "tools") + os.pathsep + os.environ["PATH"])
|
||||
start = time.monotonic()
|
||||
result = subprocess.run(["bash", "-c", HARNESS], env=env, capture_output=True, timeout=15)
|
||||
@@ -197,6 +203,40 @@ class UpdateServiceState(unittest.TestCase):
|
||||
self.assertFalse(r["stopped"], r)
|
||||
self.assertEqual(r["backups"], [], r)
|
||||
|
||||
def test_pipeline_failure_records_both_exits_before_unchanged_recovery(self):
|
||||
# Distinct simultaneous failures detect swapped or overwritten statuses;
|
||||
# collector-only failure must not be misreported as installer failure.
|
||||
for installer, collector in ((23, 0), (0, 73), (42, 73)):
|
||||
for state, load in (("active", "loaded"), ("inactive", "loaded"),
|
||||
("failed", "loaded"), ("inactive", "masked")):
|
||||
with self.subTest(installer=installer, collector=collector, state=state, load=load):
|
||||
r = self.run_fixture(state=state, load=load, installer_exit=installer,
|
||||
collector_exit=collector)
|
||||
observed = f"Installer pipeline failed (installer exit: {installer}; log collector exit: {collector})"
|
||||
self.assertIn(observed, r["output"], r)
|
||||
self.assertLess(r["output"].index(observed), r["output"].index("Restoring from backup"), r)
|
||||
self.assertNotIn("Installer completed", r["output"], r)
|
||||
self.assertNotIn("Update successfully installed and verified", r["output"], r)
|
||||
self.assertEqual(r["exit"], 1, r)
|
||||
self.assertTrue(r["installed"], r)
|
||||
self.assertEqual(r["binary"], b"#!/bin/sh\necho 'Pulse v6.5.0'\n", r)
|
||||
self.assertEqual(r["version"], "v6.5.0\n", r)
|
||||
self.assertEqual(r["started"], state == "active", r)
|
||||
self.assertEqual(r["backups"], [], r)
|
||||
|
||||
def test_completed_pipeline_records_exits_without_claiming_verification(self):
|
||||
for state in ("active", "inactive"):
|
||||
with self.subTest(state=state):
|
||||
r = self.run_fixture(state=state, success=True)
|
||||
observed = "Installer completed; verifying update (installer exit: 0; log collector exit: 0)"
|
||||
self.assertIn(observed, r["output"], r)
|
||||
self.assertLess(r["output"].index(observed), r["output"].index("Version verified:"), r)
|
||||
self.assertNotIn("Installer pipeline failed", r["output"], r)
|
||||
self.assertEqual(r["exit"], 0, r)
|
||||
self.assertIn(b"v6.6.0", r["binary"], r)
|
||||
self.assertEqual(r["started"], state == "active", r)
|
||||
self.assertEqual(r["backups"], [], r)
|
||||
|
||||
def test_uncertain_rollback_stop_retains_backup_without_replacement_or_start(self):
|
||||
for mode in ("stop-noop", "stop-error", "readback-error", "readback-empty", "after-error"):
|
||||
with self.subTest(mode=mode):
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Check ordinary CI's real-embed phase without a host build or network.
|
||||
|
||||
The exact workflow shell runs with private asset fixtures and compiler/Git
|
||||
doubles. This is recipe/admission proof, not real frontend/backend acceptance.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
WORKFLOW = Path(os.environ.get("PULSE_BUILD_WORKFLOW_UNDER_TEST", ROOT / ".github/workflows/build-and-test.yml")).resolve()
|
||||
GO_SETUP = "Set up Go for real-embed verification"
|
||||
VERIFY = "Verify backend with real frontend embed"
|
||||
|
||||
|
||||
def recipe(source):
|
||||
job = re.search(r"(?ms)^ frontend:\n(.*?)(?=^ [a-z][a-z0-9-]*:|\Z)", source)
|
||||
if not job:
|
||||
raise AssertionError("frontend job missing")
|
||||
job = job[1]
|
||||
if "timeout-minutes: 30" not in job or "if: needs.changes.outputs.code == 'true'" not in job:
|
||||
raise AssertionError("ordinary frontend admission/deadline changed")
|
||||
parsed = re.findall(r"(?ms)^ - name: ([^\n]+)\n(.*?)(?=^ - name:|\Z)", job)
|
||||
steps = dict(parsed)
|
||||
if len(parsed) != len(steps):
|
||||
raise AssertionError("duplicate named frontend step")
|
||||
order = list(steps)
|
||||
required = ["Checkout repository", "Build frontend bundle (with embed copy)",
|
||||
"Check frontend bundle size budget", "Frontend unit tests", GO_SETUP, VERIFY,
|
||||
"Require frontend dependency audit"]
|
||||
if any(name not in order for name in required) or [order.index(name) for name in required] != sorted(order.index(name) for name in required):
|
||||
raise AssertionError("same-checkout real build, tests, compiler or audit ordering missing")
|
||||
if sum("uses: actions/checkout@" in step for step in steps.values()) != 1 or "run: make frontend" not in steps[required[1]]:
|
||||
raise AssertionError("must use this checkout's production frontend build")
|
||||
if re.search(r"^ (ref|repository):", steps[required[0]], re.M):
|
||||
raise AssertionError("frontend proof must use the ordinary exact checkout")
|
||||
if "go-version-file: go.mod" not in steps[GO_SETUP] or "actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e" not in steps[GO_SETUP]:
|
||||
raise AssertionError("declared compiler setup missing")
|
||||
for name in (GO_SETUP, VERIFY):
|
||||
if re.search(r"^ (if|continue-on-error|working-directory):", steps[name], re.M):
|
||||
raise AssertionError("real-embed verification must run at root and retain failure")
|
||||
run = re.search(r"(?ms)^ run: \|\n((?: [^\n]*\n)+)", steps[VERIFY])
|
||||
if not run:
|
||||
raise AssertionError("real-embed shell missing")
|
||||
shell = "\n".join(line[10:] for line in run[1].splitlines()) + "\n"
|
||||
for command in ("set -euo pipefail", "checkout_sha=$(git rev-parse HEAD)", "test -s frontend-modern/dist/index.html",
|
||||
"test -d frontend-modern/dist/assets",
|
||||
"diff -qr --no-dereference frontend-modern/dist internal/api/frontend-modern/dist",
|
||||
'go build -mod=readonly -p=2 -o "$RUNNER_TEMP/pulse-real-embed" ./cmd/pulse',
|
||||
"go vet -mod=readonly -p=2 ./cmd/pulse ./internal/api"):
|
||||
if command not in shell:
|
||||
raise AssertionError("required real-embed command missing: " + command)
|
||||
return shell
|
||||
|
||||
|
||||
COMPILER = r'''#!/usr/bin/env python3
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
args = sys.argv[1:]
|
||||
with Path(os.environ["TRACE"]).open("a") as out:
|
||||
out.write(json.dumps(args) + "\n")
|
||||
if args[0] == os.environ["FAIL_PHASE"]:
|
||||
sys.exit(43)
|
||||
if args[0] == "build":
|
||||
Path(args[args.index("-o") + 1]).write_bytes(b"fixture binary, not a real build\n")
|
||||
elif args != ["version"] and args[0] != "vet":
|
||||
sys.exit(97)
|
||||
'''
|
||||
|
||||
|
||||
class FrontendRealEmbed(unittest.TestCase):
|
||||
def run_recipe(self, *, mutation="", fail_phase="", git_exit=0):
|
||||
shell = recipe(WORKFLOW.read_text())
|
||||
with tempfile.TemporaryDirectory(prefix="pulse-real-embed-") as directory:
|
||||
f = Path(directory)
|
||||
for tree in ("frontend-modern/dist", "internal/api/frontend-modern/dist"):
|
||||
(f / tree / "assets").mkdir(parents=True)
|
||||
(f / tree / "index.html").write_text("<html>fixture production index</html>\n")
|
||||
(f / tree / "assets/app.js").write_text("fixture application bytes\n")
|
||||
embed = f / "internal/api/frontend-modern/dist"
|
||||
if mutation == "missing index": (f / "frontend-modern/dist/index.html").unlink()
|
||||
if mutation == "empty index": (f / "frontend-modern/dist/index.html").write_text("")
|
||||
if mutation == "missing assets":
|
||||
(f / "frontend-modern/dist/assets/app.js").unlink()
|
||||
(f / "frontend-modern/dist/assets").rmdir()
|
||||
if mutation == "changed served byte": (embed / "assets/app.js").write_text("different bytes\n")
|
||||
if mutation == "extra served file": (embed / "extra.js").write_text("extra bytes\n")
|
||||
if mutation == "missing embedded file": (embed / "assets/app.js").unlink()
|
||||
for name in ("tools", "runner-temp"): (f / name).mkdir()
|
||||
(f / "tools/go").write_text(COMPILER)
|
||||
(f / "tools/git").write_text("#!/bin/sh\n[ \"$*\" = 'rev-parse HEAD' ] || exit 97\n[ \"$GIT_EXIT\" = 0 ] || exit \"$GIT_EXIT\"\necho 0123456789012345678901234567890123456789\n")
|
||||
for name in ("go", "git"): (f / "tools" / name).chmod(0o700)
|
||||
env = dict(os.environ, PATH=str(f / "tools") + os.pathsep + os.environ["PATH"],
|
||||
TRACE=str(f / "trace"), FAIL_PHASE=fail_phase, GIT_EXIT=str(git_exit), RUNNER_TEMP=str(f / "runner-temp"))
|
||||
result = subprocess.run(["bash", "-c", shell], cwd=f, env=env, capture_output=True, text=True, timeout=10)
|
||||
calls = [json.loads(line) for line in (f / "trace").read_text().splitlines()] if (f / "trace").exists() else []
|
||||
output = f / "runner-temp/pulse-real-embed"
|
||||
return result, calls, str(output), output.exists()
|
||||
|
||||
def test_success_uses_readonly_graph_and_same_checkout_assets(self):
|
||||
result, calls, output, built = self.run_recipe()
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
self.assertEqual(calls, [["version"], ["build", "-mod=readonly", "-p=2", "-o", output, "./cmd/pulse"],
|
||||
["vet", "-mod=readonly", "-p=2", "./cmd/pulse", "./internal/api"]])
|
||||
self.assertTrue(built)
|
||||
self.assertIn("REAL_EMBED_CHECKOUT=0123456789012345678901234567890123456789", result.stdout)
|
||||
|
||||
def test_missing_or_changed_embed_refuses_before_compiler(self):
|
||||
for mutation in ("missing index", "empty index", "missing assets", "changed served byte", "extra served file", "missing embedded file"):
|
||||
with self.subTest(mutation=mutation):
|
||||
result, calls, _, built = self.run_recipe(mutation=mutation)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(calls, [])
|
||||
self.assertFalse(built)
|
||||
|
||||
def test_build_and_vet_failures_are_terminal(self):
|
||||
for phase, called in (("build", ["version", "build"]), ("vet", ["version", "build", "vet"])):
|
||||
with self.subTest(phase=phase):
|
||||
result, calls, _, _ = self.run_recipe(fail_phase=phase)
|
||||
self.assertEqual(result.returncode, 43)
|
||||
self.assertEqual([call[0] for call in calls], called)
|
||||
|
||||
def test_failed_checkout_read_is_not_a_build_grant(self):
|
||||
result, calls, _, built = self.run_recipe(git_exit=46)
|
||||
self.assertEqual(result.returncode, 46)
|
||||
self.assertEqual(calls, [])
|
||||
self.assertFalse(built)
|
||||
self.assertNotIn("REAL_EMBED_CHECKOUT=", result.stdout)
|
||||
|
||||
def test_missing_hidden_or_foreign_workflow_phases_are_rejected(self):
|
||||
source = WORKFLOW.read_text()
|
||||
for before, after in (("run: make frontend", "run: echo stub"),
|
||||
("go-version-file: go.mod", "go-version: stable"),
|
||||
(f"- name: {VERIFY}", "- name: unrelated phase"),
|
||||
(f"- name: {VERIFY}\n", f"- name: {VERIFY}\n continue-on-error: true\n"),
|
||||
(f"- name: {VERIFY}\n", f"- name: {VERIFY}\n if: false\n"),
|
||||
(f"- name: {VERIFY}\n", f"- name: {VERIFY}\n working-directory: frontend-modern\n"),
|
||||
(f"- name: {GO_SETUP}\n", "- name: Checkout repository\n"),
|
||||
("persist-credentials: false\n", "persist-credentials: false\n ref: other-source\n"),
|
||||
("go vet -mod=readonly -p=2 ./cmd/pulse ./internal/api", "echo skipped vet")):
|
||||
with self.subTest(after=after):
|
||||
self.assertIn(before, source)
|
||||
with self.assertRaises(AssertionError): recipe(source.replace(before, after))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user