From f56e28e6afd2c4586fbaf3db761a488c9615ca6b Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:05:16 +0100 Subject: [PATCH 1/7] Own demo update and recovery through sustained host observation Compose the existing signed stable demo paths with the Relay host lock, durable systemd ownership, full persistent estate capture and five-minute forward and recovery watches. Preserve uncertain attempts and failed rollback, refuse destructive headroom pruning, and keep native installed acceptance separate from connected source controls. Change-source: pulse-maintainer --- .github/scripts/demo-runtime-transaction.py | 504 ++++++++++++++++++ .github/scripts/dispatch-demo-runtime.py | 113 ++++ .github/scripts/recover-demo-runtime.sh | 395 +------------- .../tests/test_demo_runtime_transaction.py | 443 +++++++++++++++ .github/workflows/recover-demo-server.yml | 34 +- .github/workflows/update-demo-server.yml | 326 +---------- .../v6/internal/DEMO_RUNTIME_TRANSACTION.md | 60 +++ .../subsystems/deployment-installability.md | 32 ++ .../v6/internal/subsystems/registry.json | 28 +- .../installtests/build_release_assets_test.go | 43 +- scripts/installtests/demo_transaction_test.go | 56 ++ 11 files changed, 1278 insertions(+), 756 deletions(-) create mode 100644 .github/scripts/demo-runtime-transaction.py create mode 100644 .github/scripts/dispatch-demo-runtime.py create mode 100644 .github/scripts/tests/test_demo_runtime_transaction.py create mode 100644 docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md create mode 100644 scripts/installtests/demo_transaction_test.go diff --git a/.github/scripts/demo-runtime-transaction.py b/.github/scripts/demo-runtime-transaction.py new file mode 100644 index 0000000000..60193e378e --- /dev/null +++ b/.github/scripts/demo-runtime-transaction.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python3 +"""Demo-only host transaction. Called by the reviewed CI dispatcher, not a worker tool.""" +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import signal +import socket +import stat +import subprocess +import sys +import time +from urllib.parse import urlsplit + +WINDOW = 300 +INTERVAL = 5 +READY_TIMEOUT = 60 +SERVICES = ("pulse", "pulse-relay", "caddy") +ROOT = Path("/var/lib/pulse-deploy/demo") +LOCK = Path("/var/lib/pulse-deploy/relay/deploy.lock") +PATHS = {"binary": Path("/opt/pulse/bin/pulse"), + "unit": Path("/etc/systemd/system/pulse.service"), + "dropins": Path("/etc/systemd/system/pulse.service.d"), + "data": Path("/etc/pulse")} +RELAY_HEALTH = "https://relay.pulserelay.pro/health" +COUNT_KEYS = ("nodes", "vms_per_node", "lxcs_per_node", "docker_hosts", + "docker_containers", "generic_hosts", "k8s_clusters", "k8s_nodes", + "k8s_pods", "k8s_deployments") +DURATION_KEYS = ("seed_duration", "sample_interval", "update_interval") +# Only generated demo operational memory is reset on the already-unhealthy +# recovery route. Its complete original estate remains in the private snapshot. +DEMO_HISTORY = ("ai_incidents.json", "ai_incidents.json.tmp", + "alerts/alert-history.json", "alerts/alert-history.backup.json", + "alerts/alert-history.json.imported", "alerts/alert-history.backup.json.imported", + "alerts/events.db", "alerts/events.db-shm", "alerts/events.db-wal") +CRASH = re.compile(r"\bpanic\b|\bfatal error\b|segmentation fault|core dumped|watchdog timeout", re.I) +TERMINAL = {"committed", "healthy_noop", "rolled_back", "refused"} + + +class Failure(Exception): + """Only fixed stage names, never command output or private response bodies.""" + + +def atomic_json(path, value): + tmp = path.with_suffix(".tmp") + with open(tmp, "w", encoding="utf-8") as stream: + os.chmod(tmp, 0o600) + json.dump(value, stream, sort_keys=True) + stream.flush() + os.fsync(stream.fileno()) + os.replace(tmp, path) + fd = os.open(path.parent, os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def validate(request): + if set(request) != {"mode", "hostname", "local_url", "public_url", "version", "profile", "control_sha", "run_id", "run_attempt", "installer_sha256"}: + raise Failure("request-shape") + if request["mode"] not in {"update", "recover"}: + raise Failure("request-mode") + for field, pattern in (("version", r"v[0-9]+\.[0-9]+\.[0-9]+"), + ("hostname", r"[a-zA-Z0-9][a-zA-Z0-9.-]{0,252}"), + ("control_sha", r"[0-9a-f]{40}"), + ("run_id", r"[1-9][0-9]*"), ("run_attempt", r"[1-9][0-9]*")): + if not isinstance(request[field], str) or not re.fullmatch(pattern, request[field]): + raise Failure("request-identity") + for field, public in (("local_url", False), ("public_url", True)): + try: + url = urlsplit(request[field]) + valid = (url.scheme == ("https" if public else "http") + and url.hostname and not url.username and not url.password + and not url.query and not url.fragment + and (url.path == "/api/health" if public else url.path in {"", "/"}) + and (public or url.hostname in {"localhost", "127.0.0.1", "::1"}) + and 0 < (url.port or (443 if public else 80)) < 65536) + except (TypeError, ValueError): + valid = False + if not valid: + raise Failure("request-url") + profile = request["profile"] + if not isinstance(profile, dict) or set(profile) != set(COUNT_KEYS + DURATION_KEYS): + raise Failure("request-profile") + for name, value in profile.items(): + pattern = r"[1-9][0-9]{0,3}" if name in COUNT_KEYS else r"[1-9][0-9]{0,3}[smhd]" + if not isinstance(value, str) or not re.fullmatch(pattern, value): + raise Failure("request-profile") + digest = request["installer_sha256"] + if (request["mode"] == "update" and not re.fullmatch(r"[0-9a-f]{64}", digest)) or (request["mode"] == "recover" and digest != ""): + raise Failure("request-installer") + + +def copy_path(src, dst): + """Copy without following symlinks, retaining ownership as well as modes.""" + info = src.lstat() + if stat.S_ISLNK(info.st_mode): + dst.symlink_to(os.readlink(src)) + elif stat.S_ISDIR(info.st_mode): + dst.mkdir(mode=0o700) + for child in src.iterdir(): + copy_path(child, dst / child.name) + shutil.copystat(src, dst, follow_symlinks=False) + elif stat.S_ISREG(info.st_mode): + shutil.copy2(src, dst, follow_symlinks=False) + else: + raise Failure("snapshot-special-file") + os.chown(dst, info.st_uid, info.st_gid, follow_symlinks=False) + if not stat.S_ISLNK(info.st_mode): + fd = os.open(dst, os.O_RDONLY | (os.O_DIRECTORY if stat.S_ISDIR(info.st_mode) else 0)) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def remove_path(path): + if path.is_symlink() or path.is_file(): + path.unlink() + elif path.is_dir(): + shutil.rmtree(path) + + +def estate_hash(paths): + rows = [] + for label, root in sorted(paths.items()): + exists = root.exists() or root.is_symlink() + if not exists: + rows.append((label, "absent")) + continue + for path in [root] + (sorted(root.rglob("*")) if root.is_dir() and not root.is_symlink() else []): + info = path.lstat() + kind = stat.S_IFMT(info.st_mode) + content = os.readlink(path) if stat.S_ISLNK(info.st_mode) else "" + if stat.S_ISREG(info.st_mode): + digest = hashlib.sha256() + with open(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + content = digest.hexdigest() + rows.append((label, str(path.relative_to(root)), kind, stat.S_IMODE(info.st_mode), info.st_uid, info.st_gid, content)) + return hashlib.sha256(json.dumps(rows, sort_keys=True).encode()).hexdigest() + + +class Host: + now = staticmethod(time.monotonic) + sleep = staticmethod(time.sleep) + + def command(self, argv, *, timeout=20, env=None): + try: + result = subprocess.run(argv, capture_output=True, timeout=timeout, env=env, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise Failure("command-unavailable") from exc + if result.returncode or len(result.stdout) > 1024 * 1024: + raise Failure("command-failed") + return result.stdout.decode("utf-8", errors="strict") + + def identity(self, hostname): + if socket.gethostname() != hostname or os.geteuid() != 0: + raise Failure("host-identity") + unit = self.command(["systemctl", "show", "pulse", "--property=FragmentPath,User,Group,ExecStart,DropInPaths", "--no-pager"]) + fields = dict(line.split("=", 1) for line in unit.splitlines() if "=" in line) + if (fields.get("FragmentPath") != str(PATHS["unit"]) + or fields.get("User") != "pulse" or fields.get("Group") != "pulse" + or str(PATHS["binary"]) not in fields.get("ExecStart", "")): + raise Failure("service-identity") + for name in ("binary", "unit", "data"): + if PATHS[name].is_symlink() or not PATHS[name].exists(): + raise Failure("estate-identity") + if PATHS["dropins"].is_symlink(): + raise Failure("dropin-identity") + for path in fields.get("DropInPaths", "").split(): + if Path(path).parent != PATHS["dropins"] or Path(path).is_symlink(): + raise Failure("unsupported-dropin") + if self.command(["systemctl", "is-enabled", "pulse"]).strip() != "enabled": + raise Failure("service-disabled") + + def state(self): + states = {} + for service in SERVICES: + text = self.command(["systemctl", "show", service, "--property=ActiveState,SubState,MainPID,NRestarts", "--no-pager"]) + fields = dict(line.split("=", 1) for line in text.splitlines() if "=" in line) + if (set(fields) != {"ActiveState", "SubState", "MainPID", "NRestarts"} + or not fields["MainPID"].isdigit() or not fields["NRestarts"].isdigit()): + raise Failure("service-observation") + states[service] = fields + return states + + def status(self, url): + # No redirects, response bodies, credential arguments or ambient curl config. + try: + result = self.command(["curl", "--disable", "--silent", "--connect-timeout", "3", "--max-time", "8", "--output", "/dev/null", "--write-out", "%{http_code}", "--", url], timeout=10) + except Failure: + return "000" + return result if re.fullmatch(r"[1-5][0-9]{2}", result) else "000" + + def version(self, url): + try: + body = self.command(["curl", "--disable", "--silent", "--fail", "--connect-timeout", "3", "--max-time", "8", "--", url.rstrip("/") + "/api/version"], timeout=10) + if len(body) > 8192: + raise Failure("version-observation") + value = json.loads(body)["version"] + if not isinstance(value, str) or not re.fullmatch(r"v?[0-9]+\.[0-9]+\.[0-9]+", value): + raise Failure("version-observation") + return value.lstrip("v") + except (ValueError, KeyError, TypeError) as exc: + raise Failure("version-observation") from exc + + def cursors(self): + cursors = {} + for service in SERVICES: + text = self.command(["journalctl", "-u", service, "-n", "1", "--output=json", "--show-cursor", "--no-pager"]) + matches = re.findall(r"^-- cursor: ([^\s]+)$", text, re.M) + if len(matches) != 1: + raise Failure("journal-cursor") + cursors[service] = matches[0] + return cursors + + def check_journal(self, cursors): + for service, cursor in cursors.items(): + text = self.command(["journalctl", "-u", service, "--after-cursor", cursor, "--output=json", "--no-pager"]) + for line in text.splitlines(): + try: + entry = json.loads(line) + if not isinstance(entry.get("MESSAGE"), str) or not isinstance(entry.get("__CURSOR"), str): + raise Failure("journal-observation") + except (ValueError, TypeError, AttributeError) as exc: + raise Failure("journal-observation") from exc + if CRASH.search(entry["MESSAGE"]): + raise Failure("new-service-crash") + cursors[service] = entry["__CURSOR"] + + def stop(self): + self.command(["systemctl", "stop", "pulse"], timeout=90) + + def start(self): + self.command(["systemctl", "daemon-reload"]) + self.command(["systemctl", "start", "pulse"], timeout=90) + + def install(self, attempt, request): + installer = attempt / "installer.sh" + if hashlib.sha256(installer.read_bytes()).hexdigest() != request["installer_sha256"]: + raise Failure("installer-identity") + self.command(["bash", str(installer), "--version", request["version"]], timeout=900, + env={"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "HOME": "/root", "PULSE_SERVICE_NAME": "pulse"}) + + def profile_values(self, request): + values = {"DEMO_MODE": "true", "PULSE_MOCK_MODE": "true", "PULSE_MOCK_RANDOM_METRICS": "true", + "PULSE_MOCK_STOPPED_PERCENT": "6", "PULSE_MOCK_SEED_METRICS_STORE": "false"} + for key, value in request["profile"].items(): + key = {"seed_duration": "trends_seed_duration", "sample_interval": "trends_sample_interval"}.get(key, key) + values["PULSE_MOCK_" + key.upper()] = value + return values + + def profile_matches(self, request): + text = (PATHS["data"] / ".env").read_text().splitlines() + values = self.profile_values(request) + return (all(text.count(key + "=" + value) == 1 for key, value in values.items()) + and "demo_fixtures" in json.loads((PATHS["data"] / "billing.json").read_text()).get("capabilities", [])) + + def profile(self, request, unhealthy): + env_file = PATHS["data"] / ".env" + billing_file = PATHS["data"] / "billing.json" + if env_file.is_symlink() or billing_file.is_symlink(): + raise Failure("configuration-symlink") + values = self.profile_values(request) + text = env_file.read_text() + for key, value in values.items(): + pattern = re.compile(r"^[ \t]*" + re.escape(key) + r"=.*$", re.M) + text = pattern.sub(key + "=" + value, text) if pattern.search(text) else text.rstrip("\n") + "\n" + key + "=" + value + "\n" + env_file.write_text(text) + os.chmod(env_file, 0o600) + billing = json.loads(billing_file.read_text()) + if not isinstance(billing, dict) or not isinstance(billing.get("capabilities", []), list): + raise Failure("billing-shape") + billing["capabilities"] = sorted(set(billing.get("capabilities", []) + ["demo_fixtures"])) + billing.pop("integrity", None) + billing_file.write_text(json.dumps(billing)) + os.chmod(billing_file, 0o600) + if unhealthy: + for relative in DEMO_HISTORY: + path = PATHS["data"] / relative + # Never follow a writable parent symlink to a path outside the estate. + if any(p.is_symlink() for p in path.parents if p != PATHS["data"].parent): + raise Failure("history-symlink") + if path.is_symlink(): + raise Failure("history-symlink") + if path.is_file(): + path.unlink() + + +def active(state): + return state["ActiveState"] == "active" and state["SubState"] == "running" and int(state["MainPID"]) > 0 + + +class Transaction: + def __init__(self, host, request, attempt): + self.host, self.request, self.attempt = host, request, attempt + self.receipt = {"schema_version": 2, "status": "intent", "mutated": False, + "control_sha": request["control_sha"], "run_id": request["run_id"], + "run_attempt": request["run_attempt"], "mode": request["mode"], + "expected_version": request["version"], "installer_sha256": request["installer_sha256"], + "forward": {}, "recovery": {}, "recovery_required": False} + + def save(self, status): + self.receipt["status"] = status + atomic_json(self.attempt / "receipt.json", self.receipt) + + def urls(self): + return (self.request["local_url"].rstrip("/") + "/api/health", self.request["public_url"], RELAY_HEALTH) + + def healthy(self, states=None): + return all(active(s) for s in (states or self.host.state()).values()) and all(self.host.status(url) == "200" for url in self.urls()) + + def ready(self, cursors): + deadline = self.host.now() + READY_TIMEOUT + while True: + self.host.check_journal(cursors) + if self.healthy(): + return + if self.host.now() >= deadline: + raise Failure("listener-readiness") + self.host.sleep(INTERVAL) + + def watch(self, phase, expected_version, cursors): + baseline = self.host.state() + started = self.host.now() + evidence = self.receipt[phase] + evidence.update({"required_seconds": WINDOW, "samples": 0, "elapsed_seconds": 0}) + while True: + self.host.check_journal(cursors) + state = self.host.state() + if state != baseline or not self.healthy(state): + raise Failure("sustained-health") + if self.host.version(self.request["local_url"]) != expected_version: + raise Failure("runtime-version") + evidence["samples"] += 1 + elapsed = self.host.now() - started + evidence["elapsed_seconds"] = elapsed + if elapsed >= WINDOW: + return + self.host.sleep(min(INTERVAL, WINDOW - elapsed)) + + def capture(self): + snapshot = self.attempt / "snapshot" + size = sum(path.lstat().st_size for root in PATHS.values() if root.exists() + for path in ([root] + (list(root.rglob("*")) if root.is_dir() else [])) + if path.is_file() and not path.is_symlink()) + # Both our independent estate and the signed installer's own backup need + # space. Refuse shortage; never delete backups or databases for headroom. + if any(shutil.disk_usage(parent).free < 2 * size + 64 * 1024 * 1024 + for parent in (self.attempt, PATHS["data"].parent)): + raise Failure("snapshot-headroom") + snapshot.mkdir(mode=0o700) + manifest = {} + for label, path in PATHS.items(): + manifest[label] = path.exists() or path.is_symlink() + if manifest[label]: + copy_path(path, snapshot / label) + atomic_json(snapshot / "paths.json", manifest) + digest = estate_hash(PATHS) + self.receipt["snapshot_sha256"] = digest + self.receipt["snapshot_retained"] = True + return digest + + def restore(self): + snapshot = self.attempt / "snapshot" + for label, exists in json.loads((snapshot / "paths.json").read_text()).items(): + remove_path(PATHS[label]) + if exists: + copy_path(snapshot / label, PATHS[label]) + if estate_hash(PATHS) != self.receipt["snapshot_sha256"]: + raise Failure("restored-estate-identity") + + def run(self): + old_handlers = {} + healthy_baseline = False + stopped = False + original_version = "" + states = {} + + def cancel(_signum, _frame): + raise Failure("cancelled-forward") + + try: + validate(self.request) + self.host.identity(self.request["hostname"]) + for entry in ROOT.glob("attempts/*"): + if entry == self.attempt: + continue + receipt = entry / "receipt.json" + if not receipt.exists() or json.loads(receipt.read_text()).get("status") not in TERMINAL: + raise Failure("prior-recovery-required") + states = self.host.state() + if not all(active(states[name]) for name in ("pulse-relay", "caddy")) or self.host.status(RELAY_HEALTH) != "200": + raise Failure("cohost-unhealthy") + healthy_baseline = self.healthy(states) + if healthy_baseline: + original_version = self.host.version(self.request["local_url"]) + if self.request["mode"] == "recover" and original_version != self.request["version"].lstrip("v"): + raise Failure("recovery-source-mismatch") + elif self.request["mode"] == "update": + raise Failure("update-baseline-unhealthy") + self.receipt["healthy_baseline"] = healthy_baseline + # The no-op is observed too; an immediate healthy sample is not success. + if self.request["mode"] == "recover" and healthy_baseline and self.host.profile_matches(self.request): + self.watch("forward", original_version, self.host.cursors()) + self.save("healthy_noop") + return 0 + old_handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)} + for sig in old_handlers: + signal.signal(sig, cancel) + self.save("capturing") + stopped = True + self.host.stop() + self.capture() + self.save("applying") + self.receipt["mutated"] = True + cursors = self.host.cursors() + if self.request["mode"] == "update": + self.host.install(self.attempt, self.request) + # The installer may start the unit; finish profile work quiescent. + self.host.stop() + self.host.profile(self.request, not healthy_baseline) + self.host.start() + self.ready(cursors) + self.watch("forward", self.request["version"].lstrip("v"), cursors) + # No unrelated Relay/Caddy restart is owned by this transaction. + final = self.host.state() + for service in ("pulse-relay", "caddy"): + if final[service] != states[service]: + raise Failure("cohost-identity-changed") + self.save("committed") + return 0 + except Exception as exc: + self.receipt["failure"] = str(exc) if isinstance(exc, Failure) else "transaction-observation" + # Owned restoration, observation and the terminal receipt survive cancellation. + for sig in old_handlers: + signal.signal(sig, signal.SIG_IGN) + if stopped: + try: + self.save("recovering") + self.host.stop() + if "snapshot_sha256" in self.receipt: + self.restore() + if not healthy_baseline: + self.receipt["recovery_required"] = True + self.receipt["rollback"] = "unavailable-unhealthy-baseline" + self.save("recovery_required") + else: + cursors = self.host.cursors() + self.host.start() + self.ready(cursors) + self.watch("recovery", original_version, cursors) + final = self.host.state() + if any(final[name] != states[name] for name in ("pulse-relay", "caddy")): + raise Failure("cohost-identity-changed") + self.receipt["rollback"] = "verified" + self.save("rolled_back") + except Exception as recovery: + self.receipt["rollback"] = "failed" + self.receipt["recovery_failure"] = str(recovery) if isinstance(recovery, Failure) else "recovery-observation" + self.receipt["recovery_required"] = True + self.save("rollback_failed") + else: + self.save("refused") + return 1 + finally: + for sig, handler in old_handlers.items(): + signal.signal(sig, handler) + + +def main(): + # Attempt paths are generated by the fixed CI bootstrap, not CLI overrides. + if len(sys.argv) != 2: + return 2 + attempt = Path(sys.argv[1]) + if attempt.parent != ROOT / "attempts" or not re.fullmatch(r"[0-9a-f]{64}", attempt.name) or attempt.is_symlink(): + return 2 + request = json.loads((attempt / "request.json").read_text()) + transaction = Transaction(Host(), request, attempt) + fd = None + try: + LOCK.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + fd = os.open(LOCK, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return transaction.run() + except (OSError, ValueError): + transaction.receipt["failure"] = "host-lock-unavailable" + transaction.save("refused") + return 1 + finally: + if fd is not None: + os.close(fd) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/dispatch-demo-runtime.py b/.github/scripts/dispatch-demo-runtime.py new file mode 100644 index 0000000000..499602fd17 --- /dev/null +++ b/.github/scripts/dispatch-demo-runtime.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""The two existing demo CI paths share one retained, independently owned child.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys + +BOOTSTRAP = r''' +import base64, fcntl, hashlib, json, os, pathlib, subprocess, sys, time +root = pathlib.Path('/var/lib/pulse-deploy/demo') +payload = json.loads(sys.stdin.read(512 * 1024)) +assert os.geteuid() == 0 and set(payload) == {'request', 'source', 'installer'} +source = base64.b64decode(payload['source'], validate=True) +installer = base64.b64decode(payload['installer'], validate=True) +assert len(source) < 128 * 1024 and len(installer) < 256 * 1024 +assert (hashlib.sha256(installer).hexdigest() == payload['request']['installer_sha256']) if installer else (payload['request']['installer_sha256'] == '') +identity = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(',', ':')).encode()).hexdigest() +root.mkdir(mode=0o700, parents=True, exist_ok=True) +assert not root.is_symlink() and root.stat().st_uid == 0 +os.chmod(root, 0o700) +(root / 'attempts').mkdir(mode=0o700, exist_ok=True) +assert not (root / 'attempts').is_symlink() +fd = os.open(root / 'submit.lock', os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) +fcntl.flock(fd, fcntl.LOCK_EX) +attempt = root / 'attempts' / identity +if not attempt.exists(): + attempt.mkdir(mode=0o700) + for name, data in [('transaction.py', source), ('installer.sh', installer), + ('request.json', json.dumps(payload['request']).encode()), + ('intent.json', json.dumps({'schema_version': 1, 'identity': identity, 'state': 'intent'}).encode())]: + with open(attempt / name, 'xb') as stream: + os.chmod(attempt / name, 0o600) + stream.write(data); stream.flush(); os.fsync(stream.fileno()) + # Intent is durable before starting the unit. An uncertain launch is never replayed. + dfd = os.open(attempt, os.O_DIRECTORY); os.fsync(dfd); os.close(dfd) + try: + child = subprocess.run(['systemd-run', '--quiet', '--collect', + '--unit=pulse-demo-' + identity[:32], + '--property=RuntimeMaxSec=45min', + '--property=TimeoutStopSec=20min', '--property=KillMode=mixed', + '/usr/bin/python3', str(attempt / 'transaction.py'), str(attempt)], + capture_output=True, timeout=30) + launch = {'schema_version': 1, 'state': 'submitted' if child.returncode == 0 else 'uncertain', + 'unit': 'pulse-demo-' + identity[:32]} + except (OSError, subprocess.TimeoutExpired): + launch = {'schema_version': 1, 'state': 'uncertain'} + with open(attempt / 'launch.json', 'x') as stream: + os.chmod(attempt / 'launch.json', 0o600) + json.dump(launch, stream); stream.flush(); os.fsync(stream.fileno()) +os.close(fd) +# Closing or losing this observer never stops the independently owned unit. +deadline = time.monotonic() + 4000 +terminal = {'committed', 'healthy_noop', 'rolled_back', 'rollback_failed', 'recovery_required', 'refused'} +while time.monotonic() < deadline: + try: + receipt = json.loads((attempt / 'receipt.json').read_text()) + except (FileNotFoundError, ValueError): + receipt = None + if receipt and receipt.get('status') in terminal: + receipt['request_identity'] = identity + print(json.dumps(receipt)) + sys.exit(0 if receipt['status'] in {'committed', 'healthy_noop'} else 1) + launch = attempt / 'launch.json' + if launch.exists() and json.loads(launch.read_text()).get('state') == 'uncertain': + break + time.sleep(5) +print(json.dumps({'schema_version': 2, 'status': 'uncertain', 'request_identity': identity, + 'failure': 'terminal-receipt-unavailable', 'recovery_required': True})) +sys.exit(1) +''' + + +def payload(mode, environ=os.environ): + profile = {key: environ['MOCK_' + key.upper()] for key in + ('nodes', 'vms_per_node', 'lxcs_per_node', 'docker_hosts', 'docker_containers', + 'generic_hosts', 'k8s_clusters', 'k8s_nodes', 'k8s_pods', 'k8s_deployments', + 'seed_duration', 'sample_interval', 'update_interval')} + installer = Path('/tmp/pulse-install.sh').read_bytes() if mode == 'update' else b'' + request = {'mode': mode, 'hostname': environ['DEMO_EXPECTED_HOSTNAME'], + 'local_url': environ['DEMO_LOCAL_BASE_URL'], 'public_url': environ['DEMO_PUBLIC_HEALTH_URL'], + 'version': environ['EXPECTED_VERSION'], 'profile': profile, + 'control_sha': environ['GITHUB_SHA'], 'run_id': environ['GITHUB_RUN_ID'], + 'run_attempt': environ['GITHUB_RUN_ATTEMPT'], + 'installer_sha256': hashlib.sha256(installer).hexdigest() if installer else ''} + return {'request': request, 'source': base64.b64encode(Path(__file__).with_name('demo-runtime-transaction.py').read_bytes()).decode(), + 'installer': base64.b64encode(installer).decode()} + + +def main(): + if len(sys.argv) != 2 or sys.argv[1] not in {'update', 'recover'}: + return 2 + # Same project CI SSH identity and pinned host verification as before. This + # is not an installed worker dispatcher or a new project credential. + command = ['ssh', '-i', str(Path.home() / '.ssh/id_ed25519'), '-o', 'IdentitiesOnly=yes', + '-o', 'StrictHostKeyChecking=yes', '-o', 'UserKnownHostsFile=' + str(Path.home() / '.ssh/known_hosts'), + os.environ['DEMO_SERVER_USER'] + '@' + os.environ['DEMO_SERVER_HOST'], + 'sudo python3 -c ' + shlex.quote(BOOTSTRAP)] + try: + result = subprocess.run(command, input=json.dumps(payload(sys.argv[1])).encode(), timeout=4100, check=False) + except (OSError, subprocess.TimeoutExpired): + print(json.dumps({'schema_version': 2, 'status': 'uncertain', 'failure': 'observer-unavailable', 'recovery_required': True})) + return 1 + return result.returncode + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/scripts/recover-demo-runtime.sh b/.github/scripts/recover-demo-runtime.sh index 132c5cf48f..81612502a7 100755 --- a/.github/scripts/recover-demo-runtime.sh +++ b/.github/scripts/recover-demo-runtime.sh @@ -1,395 +1,4 @@ #!/usr/bin/env bash set -euo pipefail - -if [ "$#" -ne 16 ]; then - echo "usage: recover-demo-runtime.sh EXPECTED_HOSTNAME LOCAL_BASE_URL EXPECTED_VERSION MOCK_NODES MOCK_VMS_PER_NODE MOCK_LXCS_PER_NODE MOCK_DOCKER_HOSTS MOCK_DOCKER_CONTAINERS MOCK_GENERIC_HOSTS MOCK_K8S_CLUSTERS MOCK_K8S_NODES MOCK_K8S_PODS MOCK_K8S_DEPLOYMENTS MOCK_SEED_DURATION MOCK_SAMPLE_INTERVAL MOCK_UPDATE_INTERVAL" >&2 - exit 2 -fi - -EXPECTED_HOSTNAME="$1" -LOCAL_BASE_URL="${2%/}" -EXPECTED_VERSION="${3#v}" -MOCK_NODES="$4" -MOCK_VMS_PER_NODE="$5" -MOCK_LXCS_PER_NODE="$6" -MOCK_DOCKER_HOSTS="$7" -MOCK_DOCKER_CONTAINERS="$8" -MOCK_GENERIC_HOSTS="$9" -MOCK_K8S_CLUSTERS="${10}" -MOCK_K8S_NODES="${11}" -MOCK_K8S_PODS="${12}" -MOCK_K8S_DEPLOYMENTS="${13}" -MOCK_SEED_DURATION="${14}" -MOCK_SAMPLE_INTERVAL="${15}" -MOCK_UPDATE_INTERVAL="${16}" -SERVICE_NAME="pulse" -RELAY_SERVICE_NAME="pulse-relay" -EXPECTED_BINARY="/opt/pulse/bin/pulse" -EXPECTED_UNIT="/etc/systemd/system/pulse.service" -MUTATED=false -BACKUP_DIR="" -HISTORY_BACKUP_ARCHIVE="" -DEMO_HISTORY_PATHS=( - ai_incidents.json - ai_incidents.json.tmp - alerts/alert-history.json - alerts/alert-history.backup.json - alerts/alert-history.json.imported - alerts/alert-history.backup.json.imported - alerts/events.db - alerts/events.db-shm - alerts/events.db-wal -) - -for fixture_count in \ - "$MOCK_NODES" \ - "$MOCK_VMS_PER_NODE" \ - "$MOCK_LXCS_PER_NODE" \ - "$MOCK_DOCKER_HOSTS" \ - "$MOCK_DOCKER_CONTAINERS" \ - "$MOCK_GENERIC_HOSTS" \ - "$MOCK_K8S_CLUSTERS" \ - "$MOCK_K8S_NODES" \ - "$MOCK_K8S_PODS" \ - "$MOCK_K8S_DEPLOYMENTS"; do - [[ "$fixture_count" =~ ^[1-9][0-9]*$ ]] -done -[[ "$MOCK_SEED_DURATION" =~ ^[1-9][0-9]*[smhd]$ ]] -[[ "$MOCK_SAMPLE_INTERVAL" =~ ^[1-9][0-9]*[smh]$ ]] -[[ "$MOCK_UPDATE_INTERVAL" =~ ^[1-9][0-9]*[smh]$ ]] - -log() { - printf '[demo-recovery] %s\n' "$*" >&2 -} - -unit_hash() { - sudo sha256sum "$EXPECTED_UNIT" | awk '{print $1}' -} - -binary_hash() { - sudo sha256sum "$EXPECTED_BINARY" | awk '{print $1}' -} - -dropin_manifest_hash() { - local paths path - paths="$(systemctl show "$SERVICE_NAME" --property=DropInPaths --value)" - if [ -z "$paths" ]; then - printf '%s' 'none' | sha256sum | awk '{print $1}' - return - fi - for path in $paths; do - sudo test -f "$path" - printf '%s\t' "$path" - sudo sha256sum "$path" | awk '{print $1}' - done | sort | sha256sum | awk '{print $1}' -} - -runtime_config_hash() { - local path - for path in /etc/pulse/.env /etc/pulse/billing.json; do - sudo test -f "$path" - printf '%s\t' "$path" - sudo sha256sum "$path" | awk '{print $1}' - done | sha256sum | awk '{print $1}' -} - -set_env_value() { - local key="$1" - local value="$2" - local env_file="/etc/pulse/.env" - if sudo grep -Eq "^[[:space:]]*${key}=" "$env_file"; then - sudo sed -i "s|^[[:space:]]*${key}=.*|${key}=${value}|" "$env_file" - else - printf '\n%s=%s\n' "$key" "$value" | sudo tee -a "$env_file" >/dev/null - fi -} - -runtime_profile_matches() { - sudo grep -Fxq "PULSE_MOCK_NODES=${MOCK_NODES}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_VMS_PER_NODE=${MOCK_VMS_PER_NODE}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_LXCS_PER_NODE=${MOCK_LXCS_PER_NODE}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_DOCKER_HOSTS=${MOCK_DOCKER_HOSTS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_DOCKER_CONTAINERS=${MOCK_DOCKER_CONTAINERS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_GENERIC_HOSTS=${MOCK_GENERIC_HOSTS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_K8S_CLUSTERS=${MOCK_K8S_CLUSTERS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_K8S_NODES=${MOCK_K8S_NODES}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_K8S_PODS=${MOCK_K8S_PODS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_K8S_DEPLOYMENTS=${MOCK_K8S_DEPLOYMENTS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_SEED_METRICS_STORE=false" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_TRENDS_SEED_DURATION=${MOCK_SEED_DURATION}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_TRENDS_SAMPLE_INTERVAL=${MOCK_SAMPLE_INTERVAL}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_UPDATE_INTERVAL=${MOCK_UPDATE_INTERVAL}" /etc/pulse/.env -} - -cleanup_config_backup() { - [ -n "$BACKUP_DIR" ] || return 0 - sudo unlink "$BACKUP_DIR/runtime.env" 2>/dev/null || true - sudo unlink "$BACKUP_DIR/billing.json" 2>/dev/null || true - if [ -n "$HISTORY_BACKUP_ARCHIVE" ]; then - sudo unlink "$HISTORY_BACKUP_ARCHIVE" 2>/dev/null || true - HISTORY_BACKUP_ARCHIVE="" - fi - rmdir "$BACKUP_DIR" 2>/dev/null || true - BACKUP_DIR="" -} - -clear_demo_operational_history() { - local rel_path - local -a existing=() - for rel_path in "${DEMO_HISTORY_PATHS[@]}"; do - if sudo test -f "/etc/pulse/${rel_path}"; then - existing+=("$rel_path") - fi - done - ((${#existing[@]} > 0)) || return 0 - - HISTORY_BACKUP_ARCHIVE="$BACKUP_DIR/demo-operational-history.tar" - sudo tar --create --preserve-permissions \ - --file "$HISTORY_BACKUP_ARCHIVE" --directory /etc/pulse -- "${existing[@]}" - for rel_path in "${existing[@]}"; do - sudo unlink "/etc/pulse/${rel_path}" - done -} - -restore_demo_operational_history() { - local rel_path - [ -n "$HISTORY_BACKUP_ARCHIVE" ] || return 0 - sudo test -f "$HISTORY_BACKUP_ARCHIVE" || return 0 - for rel_path in "${DEMO_HISTORY_PATHS[@]}"; do - sudo unlink "/etc/pulse/${rel_path}" 2>/dev/null || true - done - sudo tar --extract --preserve-permissions \ - --file "$HISTORY_BACKUP_ARCHIVE" --directory /etc/pulse -} - -restore_runtime_config() { - [ -n "$BACKUP_DIR" ] || return 0 - sudo cp --archive "$BACKUP_DIR/runtime.env" /etc/pulse/.env - sudo cp --archive "$BACKUP_DIR/billing.json" /etc/pulse/billing.json -} - -local_version() { - curl --connect-timeout 3 --max-time 8 -fsS \ - "${LOCAL_BASE_URL}/api/version" | jq -er '.version' -} - -local_healthy() { - curl --connect-timeout 3 --max-time 8 -fsS \ - "${LOCAL_BASE_URL}/api/health" >/dev/null && local_version >/dev/null -} - -emit_evidence() { - local status="$1" - local before_pid="$2" - local after_pid="$3" - local before_relay_pid="$4" - local after_relay_pid="$5" - local before_binary_sha="$6" - local after_binary_sha="$7" - local before_unit_sha="$8" - local after_unit_sha="$9" - local before_dropins_sha="${10}" - local after_dropins_sha="${11}" - local version="${12}" - local before_config_sha="${13}" - local after_config_sha="${14}" - jq -n \ - --arg status "$status" \ - --argjson mutated "$MUTATED" \ - --arg hostname "$(hostname)" \ - --arg service "$SERVICE_NAME" \ - --arg before_pid "$before_pid" \ - --arg after_pid "$after_pid" \ - --arg relay_service "$RELAY_SERVICE_NAME" \ - --arg before_relay_pid "$before_relay_pid" \ - --arg after_relay_pid "$after_relay_pid" \ - --arg binary "$EXPECTED_BINARY" \ - --arg before_binary_sha256 "$before_binary_sha" \ - --arg after_binary_sha256 "$after_binary_sha" \ - --arg unit "$EXPECTED_UNIT" \ - --arg before_unit_sha256 "$before_unit_sha" \ - --arg after_unit_sha256 "$after_unit_sha" \ - --arg before_dropins_sha256 "$before_dropins_sha" \ - --arg after_dropins_sha256 "$after_dropins_sha" \ - --arg before_runtime_config_sha256 "$before_config_sha" \ - --arg after_runtime_config_sha256 "$after_config_sha" \ - --arg version "$version" \ - --arg expected_version "$EXPECTED_VERSION" \ - --arg mock_nodes "$MOCK_NODES" \ - --arg mock_vms_per_node "$MOCK_VMS_PER_NODE" \ - --arg mock_lxcs_per_node "$MOCK_LXCS_PER_NODE" \ - --arg mock_docker_hosts "$MOCK_DOCKER_HOSTS" \ - --arg mock_docker_containers "$MOCK_DOCKER_CONTAINERS" \ - --arg mock_generic_hosts "$MOCK_GENERIC_HOSTS" \ - --arg mock_k8s_clusters "$MOCK_K8S_CLUSTERS" \ - --arg mock_k8s_nodes "$MOCK_K8S_NODES" \ - --arg mock_k8s_pods "$MOCK_K8S_PODS" \ - --arg mock_k8s_deployments "$MOCK_K8S_DEPLOYMENTS" \ - --arg mock_seed_duration "$MOCK_SEED_DURATION" \ - --arg mock_sample_interval "$MOCK_SAMPLE_INTERVAL" \ - --arg mock_update_interval "$MOCK_UPDATE_INTERVAL" \ - '{schema_version: 1, status: $status, mutated: $mutated, - hostname: $hostname, service: $service, - before_pid: $before_pid, after_pid: $after_pid, - relay_service: $relay_service, - before_relay_pid: $before_relay_pid, - after_relay_pid: $after_relay_pid, - binary: $binary, - before_binary_sha256: $before_binary_sha256, - after_binary_sha256: $after_binary_sha256, - unit: $unit, - before_unit_sha256: $before_unit_sha256, - after_unit_sha256: $after_unit_sha256, - before_dropins_sha256: $before_dropins_sha256, - after_dropins_sha256: $after_dropins_sha256, - before_runtime_config_sha256: $before_runtime_config_sha256, - after_runtime_config_sha256: $after_runtime_config_sha256, - version: $version, expected_version: $expected_version, - runtime_profile: {mock_nodes: $mock_nodes, - mock_vms_per_node: $mock_vms_per_node, - mock_lxcs_per_node: $mock_lxcs_per_node, - mock_docker_hosts: $mock_docker_hosts, - mock_docker_containers: $mock_docker_containers, - mock_generic_hosts: $mock_generic_hosts, - mock_k8s_clusters: $mock_k8s_clusters, - mock_k8s_nodes: $mock_k8s_nodes, - mock_k8s_pods: $mock_k8s_pods, - mock_k8s_deployments: $mock_k8s_deployments, - mock_seed_duration: $mock_seed_duration, - mock_sample_interval: $mock_sample_interval, - mock_update_interval: $mock_update_interval}}' -} - -[ "$(hostname)" = "$EXPECTED_HOSTNAME" ] || { - log "target hostname does not match the governed demo environment" - exit 1 -} -[ "$(systemctl show "$SERVICE_NAME" --property=FragmentPath --value)" = "$EXPECTED_UNIT" ] -[ "$(systemctl show "$SERVICE_NAME" --property=User --value)" = "pulse" ] -[ "$(systemctl show "$SERVICE_NAME" --property=Group --value)" = "pulse" ] -[ "$(systemctl is-enabled "$SERVICE_NAME")" = "enabled" ] -[ "$(systemctl is-active "$RELAY_SERVICE_NAME")" = "active" ] -systemctl show "$SERVICE_NAME" --property=ExecStart --value | grep -Fq "$EXPECTED_BINARY" -sudo test -x "$EXPECTED_BINARY" -sudo test -f "$EXPECTED_UNIT" - -BEFORE_PID="$(systemctl show "$SERVICE_NAME" --property=MainPID --value)" -BEFORE_RELAY_PID="$(systemctl show "$RELAY_SERVICE_NAME" --property=MainPID --value)" -BEFORE_BINARY_SHA="$(binary_hash)" -BEFORE_UNIT_SHA="$(unit_hash)" -BEFORE_DROPINS_SHA="$(dropin_manifest_hash)" -BEFORE_CONFIG_SHA="$(runtime_config_hash)" - -if local_healthy && runtime_profile_matches; then - VERSION="$(local_version)" - [ "${VERSION#v}" = "$EXPECTED_VERSION" ] - emit_evidence healthy_noop "$BEFORE_PID" "$BEFORE_PID" \ - "$BEFORE_RELAY_PID" "$BEFORE_RELAY_PID" \ - "$BEFORE_BINARY_SHA" "$BEFORE_BINARY_SHA" \ - "$BEFORE_UNIT_SHA" "$BEFORE_UNIT_SHA" \ - "$BEFORE_DROPINS_SHA" "$BEFORE_DROPINS_SHA" "$VERSION" \ - "$BEFORE_CONFIG_SHA" "$BEFORE_CONFIG_SHA" - exit 0 -fi - -rollback_on_error() { - local rc=$? - local failed_pid - trap - ERR - if [ "$MUTATED" = true ]; then - failed_pid="$(systemctl show "$SERVICE_NAME" --property=MainPID --value || true)" - if [[ "$failed_pid" =~ ^[1-9][0-9]*$ ]] && sudo kill -0 "$failed_pid" 2>/dev/null; then - # The HTTP listener never opened, so pprof is unavailable. SIGQUIT gives - # one bounded Go goroutine dump in the service journal before the already - # failed process is stopped; it does not widen recovery to any other unit. - log "capturing failed Pulse startup goroutines before compensation" - sudo kill -QUIT "$failed_pid" || true - sleep 2 - fi - log "recovery validation failed; compensating by stopping only pulse.service" - sudo systemctl stop "$SERVICE_NAME" || true - restore_runtime_config || true - restore_demo_operational_history || true - fi - emit_evidence compensated_unavailable "$BEFORE_PID" \ - "$(systemctl show "$SERVICE_NAME" --property=MainPID --value || true)" \ - "$BEFORE_RELAY_PID" \ - "$(systemctl show "$RELAY_SERVICE_NAME" --property=MainPID --value || true)" \ - "$BEFORE_BINARY_SHA" "$(binary_hash || true)" \ - "$BEFORE_UNIT_SHA" "$(unit_hash || true)" \ - "$BEFORE_DROPINS_SHA" "$(dropin_manifest_hash || true)" "" \ - "$BEFORE_CONFIG_SHA" "$(runtime_config_hash || true)" - cleanup_config_backup - exit "$rc" -} -BACKUP_DIR="$(mktemp -d)" -sudo cp --archive /etc/pulse/.env "$BACKUP_DIR/runtime.env" -sudo cp --archive /etc/pulse/billing.json "$BACKUP_DIR/billing.json" -trap rollback_on_error ERR - -log "applying the target-compatible demo runtime profile and restarting only pulse.service" -MUTATED=true -set_env_value PULSE_MOCK_NODES "$MOCK_NODES" -set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE" -set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE" -set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS" -set_env_value PULSE_MOCK_DOCKER_CONTAINERS "$MOCK_DOCKER_CONTAINERS" -set_env_value PULSE_MOCK_GENERIC_HOSTS "$MOCK_GENERIC_HOSTS" -set_env_value PULSE_MOCK_K8S_CLUSTERS "$MOCK_K8S_CLUSTERS" -set_env_value PULSE_MOCK_K8S_NODES "$MOCK_K8S_NODES" -set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS" -set_env_value PULSE_MOCK_K8S_DEPLOYMENTS "$MOCK_K8S_DEPLOYMENTS" -# Persistent mock-store backfill is reserved for the isolated local-dev data -# directory. The governed demo already has persistent runtime data, so a stale -# opt-in here can turn startup into an unbounded SQLite backfill. -set_env_value PULSE_MOCK_SEED_METRICS_STORE false -set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION" -set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL" -set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL" -sudo chown pulse:pulse /etc/pulse/.env -sudo chmod 600 /etc/pulse/.env -# Alert lifecycle and AI incident history on this host are generated demo -# operational memory. A failed 50-node bootstrap can leave thousands of -# lifecycle events whose v6.4 replay queues one serialized incident save per -# event before HTTP opens. Back up and clear only those fixed demo-history -# files, including the retired JSON sources that repopulate a cleared event -# log; compensation restores them if the bounded profile still fails. -clear_demo_operational_history -sudo systemctl restart "$SERVICE_NAME" - -for _ in $(seq 1 12); do - if [ "$(systemctl is-active "$SERVICE_NAME" || true)" = "active" ] && local_healthy; then - break - fi - sleep 5 -done - -[ "$(systemctl is-active "$SERVICE_NAME")" = "active" ] -[ "$(systemctl show "$SERVICE_NAME" --property=SubState --value)" = "running" ] -local_healthy -runtime_profile_matches - -AFTER_PID="$(systemctl show "$SERVICE_NAME" --property=MainPID --value)" -AFTER_RELAY_PID="$(systemctl show "$RELAY_SERVICE_NAME" --property=MainPID --value)" -AFTER_BINARY_SHA="$(binary_hash)" -AFTER_UNIT_SHA="$(unit_hash)" -AFTER_DROPINS_SHA="$(dropin_manifest_hash)" -AFTER_CONFIG_SHA="$(runtime_config_hash)" -VERSION="$(local_version)" - -[ "$AFTER_PID" -gt 0 ] -[ "$AFTER_PID" != "$BEFORE_PID" ] -[ "$AFTER_RELAY_PID" = "$BEFORE_RELAY_PID" ] -[ "$AFTER_BINARY_SHA" = "$BEFORE_BINARY_SHA" ] -[ "$AFTER_UNIT_SHA" = "$BEFORE_UNIT_SHA" ] -[ "$AFTER_DROPINS_SHA" = "$BEFORE_DROPINS_SHA" ] -[ "${VERSION#v}" = "$EXPECTED_VERSION" ] - -trap - ERR -cleanup_config_backup -emit_evidence recovered "$BEFORE_PID" "$AFTER_PID" \ - "$BEFORE_RELAY_PID" "$AFTER_RELAY_PID" \ - "$BEFORE_BINARY_SHA" "$AFTER_BINARY_SHA" \ - "$BEFORE_UNIT_SHA" "$AFTER_UNIT_SHA" \ - "$BEFORE_DROPINS_SHA" "$AFTER_DROPINS_SHA" "$VERSION" \ - "$BEFORE_CONFIG_SHA" "$AFTER_CONFIG_SHA" +# CI observer only; remote systemd owns mutation, restoration and both watches. +exec python3 .github/scripts/dispatch-demo-runtime.py recover diff --git a/.github/scripts/tests/test_demo_runtime_transaction.py b/.github/scripts/tests/test_demo_runtime_transaction.py new file mode 100644 index 0000000000..36bb810f44 --- /dev/null +++ b/.github/scripts/tests/test_demo_runtime_transaction.py @@ -0,0 +1,443 @@ +import copy +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import signal +import subprocess +import tempfile +import unittest +from unittest.mock import patch + + +SCRIPTS = Path(__file__).resolve().parents[1] + + +def load(name, filename): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / filename) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +engine = load("demo_transaction", "demo-runtime-transaction.py") +dispatcher = load("demo_dispatcher", "dispatch-demo-runtime.py") + + +def request(mode="recover"): + return {"mode": mode, "hostname": "fixture", "local_url": "http://127.0.0.1:7655", + "public_url": "https://demo.example/api/health", "version": "v6.4.5", + "control_sha": "a" * 40, "run_id": "123", "run_attempt": "1", + "installer_sha256": hashlib.sha256(b"signed-installer-fixture").hexdigest() if mode == "update" else "", + "profile": {**{name: "2" for name in engine.COUNT_KEYS}, + "seed_duration": "2h", "sample_interval": "5m", "update_interval": "15s"}} + + +class FixtureHost(engine.Host): + """Real filesystem estate with a virtual *unchanged* 300-second clock. + + Commands/services/journal/HTTP are adapters, not native or installed proof. + """ + def __init__(self): + self.clock = 0 + self.pid = 100 + self.running = True + self.runtime_version = "6.4.5" + self.starts = 0 + self.stops = 0 + self.install_count = 0 + self.fail_at = None + self.failed = False + self.rollback_failure = False + self.unhealthy = False + self.journal_failure = False + self.ready_until = 0 + self.cancel_at = None + self.cancel_in_recovery = False + self.co_host_change = False + self.co_pid = 200 + self.transaction = None + + def now(self): + return self.clock + + def sleep(self, seconds): + self.clock += seconds + + def identity(self, hostname): + if hostname != "fixture": + raise engine.Failure("host-identity") + + def state(self): + state = {"ActiveState": "active", "SubState": "running", "MainPID": str(self.pid), "NRestarts": "0"} + result = {"pulse": state, "pulse-relay": {**state, "MainPID": "200"}, "caddy": {**state, "MainPID": str(self.co_pid)}} + if not self.running: + result["pulse"] = {**state, "ActiveState": "inactive", "SubState": "dead", "MainPID": "0"} + return result + + def status(self, url): + if url == engine.RELAY_HEALTH: + return "200" + if self.clock < self.ready_until or self.unhealthy: + return "503" + if self.failed and self.rollback_failure: + return "503" + return "200" if self.running else "000" + + def version(self, url): + return self.runtime_version + + def cursors(self): + return {name: str(self.clock) for name in engine.SERVICES} + + def check_journal(self, cursors): + phase = self.transaction.receipt["status"] if self.transaction else "" + if self.journal_failure: + raise engine.Failure("journal-observation") + if self.co_host_change and self.clock >= 55: + self.co_pid = 201 + if self.cancel_at is not None and self.clock >= self.cancel_at and not self.failed: + self.failed = True + handler = signal.getsignal(signal.SIGTERM) + handler(signal.SIGTERM, None) + if self.cancel_in_recovery and phase == "recovering" and self.clock >= 350: + # Exercise the actual installed handler decision; do not signal the + # source-proof executor or kill another process to prove it. + assert signal.getsignal(signal.SIGTERM) == signal.SIG_IGN + self.cancel_in_recovery = False + if self.fail_at is not None and self.clock >= self.fail_at and not self.failed: + self.failed = True + raise engine.Failure("new-service-crash") + + def stop(self): + self.stops += 1 + self.running = False + + def start(self): + self.starts += 1 + self.pid += 1 + self.running = True + if self.transaction.receipt["status"] == "recovering": + self.runtime_version = "6.4.5" + self.unhealthy = False + + def install(self, attempt, req): + if hashlib.sha256((attempt / "installer.sh").read_bytes()).hexdigest() != req["installer_sha256"]: + raise engine.Failure("installer-identity") + self.install_count += 1 + self.runtime_version = req["version"].lstrip("v") + engine.PATHS["binary"].write_bytes(b"new executable") + engine.PATHS["unit"].write_bytes(b"new unit") + (engine.PATHS["data"] / "persisted-history").write_bytes(b"forward history") + + def profile(self, req, unhealthy): + super().profile(req, unhealthy) + if self.unhealthy: + self.unhealthy = False + + +class TransactionTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + root = Path(self.tmp.name) + self.paths = {name: root / "estate" / name for name in engine.PATHS} + for name, path in self.paths.items(): + path.parent.mkdir(parents=True, exist_ok=True) + if name in {"dropins", "data"}: + path.mkdir() + else: + path.write_bytes(b"old " + name.encode()) + path.chmod(0o755 if name == "binary" else 0o644) + (self.paths["dropins"] / "limits.conf").write_bytes(b"old limits") + (self.paths["data"] / ".env").write_text("PRIVATE_TOKEN=not-logged\n") + (self.paths["data"] / "billing.json").write_text('{"capabilities":[],"integrity":"private"}') + (self.paths["data"] / "persisted-history").write_bytes(b"original history") + (self.paths["data"] / "alerts").mkdir() + (self.paths["data"] / "alerts/events.db").write_bytes(b"synthetic opaque persistent file") + self.root = root / "control" + self.attempt = self.root / "attempts" / ("a" * 64) + self.attempt.mkdir(parents=True) + (self.attempt / "installer.sh").write_bytes(b"signed-installer-fixture") + self.addCleanup(patch.stopall) + patch.object(engine, "ROOT", self.root).start() + patch.object(engine, "PATHS", self.paths).start() + self.before = engine.estate_hash(self.paths) + self.host = FixtureHost() + + def run_transaction(self, mode="recover", version=None): + req = request(mode) + if version: + req["version"] = version + tx = engine.Transaction(self.host, req, self.attempt) + self.host.transaction = tx + result = tx.run() + self.assertEqual(tx.receipt, json.loads((self.attempt / "receipt.json").read_text())) + return result, tx.receipt + + def test_healthy_profile_change_keeps_data_and_watches_complete_window(self): + code, receipt = self.run_transaction() + self.assertEqual(code, 0) + self.assertEqual(receipt["status"], "committed") + self.assertEqual(receipt["forward"]["elapsed_seconds"], 300) + self.assertEqual(receipt["forward"]["samples"], 61) + self.assertEqual((self.paths["data"] / "alerts/events.db").read_bytes(), b"synthetic opaque persistent file") + self.assertTrue((self.attempt / "snapshot/data/persisted-history").exists()) + self.assertNotIn("not-logged", json.dumps(receipt)) + + def test_healthy_noop_requires_profile_and_full_watch_without_restart(self): + self.host.profile(request(), False) + code, receipt = self.run_transaction() + self.assertEqual(code, 0) + self.assertEqual(receipt["status"], "healthy_noop") + self.assertEqual(receipt["forward"]["elapsed_seconds"], 300) + self.assertFalse(receipt["mutated"]) + self.assertEqual(self.host.stops, 0) + + def test_update_restores_complete_changed_executable_unit_and_data_after_late_failure(self): + self.host.fail_at = 55 + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "rolled_back") + self.assertEqual(receipt["failure"], "new-service-crash") + self.assertEqual(receipt["recovery"]["elapsed_seconds"], 300) + self.assertEqual(engine.estate_hash(self.paths), self.before) + self.assertEqual(self.host.runtime_version, "6.4.5") + + def test_failure_at_299_seconds_still_collects_entire_recovery_and_cancellation(self): + self.host.fail_at = 299 + self.host.cancel_in_recovery = True + previous = signal.getsignal(signal.SIGTERM) + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["recovery"]["elapsed_seconds"], 300) + self.assertGreaterEqual(self.host.clock, 600) + self.assertFalse(self.host.cancel_in_recovery) + self.assertEqual(signal.getsignal(signal.SIGTERM), previous) + self.assertEqual(engine.estate_hash(self.paths), self.before) + + def test_forward_cancel_owns_full_recovery(self): + self.host.cancel_at = 55 + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["failure"], "cancelled-forward") + self.assertEqual(receipt["status"], "rolled_back") + self.assertEqual(receipt["recovery"]["elapsed_seconds"], 300) + + def test_failed_rollback_retains_estate_and_blocks_next_transaction(self): + self.host.fail_at = 55 + self.host.rollback_failure = True + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "rollback_failed") + self.assertTrue(receipt["snapshot_retained"]) + next_attempt = self.root / "attempts" / ("b" * 64) + next_attempt.mkdir() + self.attempt = next_attempt + self.host.rollback_failure = False + code, receipt = self.run_transaction() + self.assertEqual(receipt["failure"], "prior-recovery-required") + self.assertFalse(receipt["mutated"]) + + def test_unhealthy_recovery_retains_original_generated_history(self): + self.host.unhealthy = True + code, receipt = self.run_transaction() + self.assertEqual(code, 0) + self.assertFalse(receipt["healthy_baseline"]) + self.assertFalse((self.paths["data"] / "alerts/events.db").exists()) + self.assertEqual((self.attempt / "snapshot/data/alerts/events.db").read_bytes(), b"synthetic opaque persistent file") + self.assertEqual(receipt["forward"]["elapsed_seconds"], 300) + + def test_failed_unhealthy_recovery_is_not_a_healthy_rollback(self): + self.host.unhealthy = True + self.host.fail_at = 55 + code, receipt = self.run_transaction() + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "recovery_required") + self.assertEqual(receipt["rollback"], "unavailable-unhealthy-baseline") + self.assertEqual(engine.estate_hash(self.paths), self.before) + self.assertFalse(self.host.running) + + def test_update_refuses_unhealthy_baseline_without_stopping_or_pruning(self): + self.host.unhealthy = True + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["failure"], "update-baseline-unhealthy") + self.assertEqual(self.host.stops, 0) + self.assertEqual(engine.estate_hash(self.paths), self.before) + + def test_recovery_refuses_different_healthy_source(self): + code, receipt = self.run_transaction(version="v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["failure"], "recovery-source-mismatch") + self.assertEqual(self.host.stops, 0) + + def test_listener_readiness_does_not_spend_the_watch_window(self): + self.host.ready_until = 30 + code, receipt = self.run_transaction() + self.assertEqual(code, 0) + self.assertEqual(receipt["forward"]["elapsed_seconds"], 300) + self.assertGreaterEqual(self.host.clock, 330) + self.assertEqual(self.host.starts, 1) + + def test_missing_journal_fails_closed_and_retains_failure(self): + self.host.journal_failure = True + code, receipt = self.run_transaction() + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "rollback_failed") + self.assertEqual(receipt["recovery_failure"], "journal-observation") + + def test_cohost_restart_is_not_silently_accepted_as_recovery(self): + self.host.co_host_change = True + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "rollback_failed") + self.assertEqual(receipt["recovery_failure"], "cohost-identity-changed") + + def test_snapshot_headroom_failure_never_deletes_persistent_files(self): + with patch.object(engine.shutil, "disk_usage", return_value=type("Usage", (), {"free": 0})()): + code, receipt = self.run_transaction() + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "rolled_back") + self.assertFalse(receipt["mutated"]) + self.assertEqual(receipt["failure"], "snapshot-headroom") + self.assertEqual(engine.estate_hash(self.paths), self.before) + + def test_linked_history_cannot_remove_files_outside_estate(self): + (self.paths["data"] / "alerts/events.db").unlink() + outside = self.root / "private-file" + outside.write_text("keep") + (self.paths["data"] / "alerts/events.db").symlink_to(outside) + self.host.unhealthy = True + code, receipt = self.run_transaction() + self.assertEqual(code, 1) + self.assertEqual(receipt["failure"], "history-symlink") + self.assertEqual(outside.read_text(), "keep") + + def test_shared_host_lock_refusal_is_terminal_and_has_no_mutation(self): + lock = self.root / "host.lock" + fd = os.open(lock, os.O_CREAT | os.O_RDWR, 0o600) + self.addCleanup(os.close, fd) + engine.fcntl.flock(fd, engine.fcntl.LOCK_EX | engine.fcntl.LOCK_NB) + (self.attempt / "request.json").write_text(json.dumps(request())) + with patch.object(engine, "LOCK", lock), patch.object(engine.sys, "argv", ["transaction", str(self.attempt)]): + self.assertEqual(engine.main(), 1) + receipt = json.loads((self.attempt / "receipt.json").read_text()) + self.assertEqual(receipt["status"], "refused") + self.assertFalse(receipt["mutated"]) + self.assertEqual(engine.estate_hash(self.paths), self.before) + + +class InputAndCommandTest(unittest.TestCase): + def test_valid_current_envelope(self): + engine.validate(request()) + engine.validate(request("update")) + + def test_no_candidate_prerelease_arbitrary_service_url_or_profile(self): + cases = [("mode", "shell"), ("version", "v6.4.6-rc.1"), ("control_sha", "main"), + ("local_url", "http://other-host:7655"), ("public_url", "http://demo.example/api/health"), + ("public_url", "https://user:secret@demo.example/api/health"), + ("public_url", "https://demo.example/api/health?token=secret"), + ("installer_sha256", "arbitrary")] + for name, value in cases: + with self.subTest(name=name, value=value): + req = request(); req[name] = value + with self.assertRaises(engine.Failure): + engine.validate(req) + req = request(); req["service"] = "license" + with self.assertRaises(engine.Failure): + engine.validate(req) + req = request(); req["profile"]["nodes"] = "2; restart caddy" + with self.assertRaises(engine.Failure): + engine.validate(req) + + def test_http_observation_rejects_redirect_and_transport_failure(self): + host = engine.Host() + with patch.object(host, "command", return_value="302") as command: + self.assertEqual(host.status("https://demo.example/api/health"), "302") + self.assertNotIn("--location", command.call_args.args[0]) + self.assertIn("--disable", command.call_args.args[0]) + self.assertIn("/dev/null", command.call_args.args[0]) + with patch.object(host, "command", side_effect=engine.Failure("command-failed")): + self.assertEqual(host.status("https://demo.example/api/health"), "000") + + def test_malformed_or_unavailable_journal_is_not_no_crash(self): + host = engine.Host() + for text in ["not-json", '{"MESSAGE":"fine"}', '{"MESSAGE":[],"__CURSOR":"c"}']: + with self.subTest(text=text), patch.object(host, "command", return_value=text): + with self.assertRaises(engine.Failure): + host.check_journal({"pulse": "c"}) + with patch.object(host, "command", return_value='{"MESSAGE":"panic: fixture","__CURSOR":"d"}'): + with self.assertRaisesRegex(engine.Failure, "new-service-crash"): + host.check_journal({"pulse": "c"}) + + def test_journal_cursor_advances_without_returning_private_message(self): + host = engine.Host(); cursors = {"pulse": "c"} + with patch.object(host, "command", return_value='{"MESSAGE":"private normal message","__CURSOR":"d"}'): + host.check_journal(cursors) + self.assertEqual(cursors, {"pulse": "d"}) + + def test_no_journal_cursor_and_malformed_service_state_fail_closed(self): + host = engine.Host() + with patch.object(host, "command", return_value=""): + with self.assertRaises(engine.Failure): + host.cursors() + with self.assertRaises(engine.Failure): + host.state() + + +class DispatchLifecycleTest(unittest.TestCase): + def test_observer_budget_covers_child_and_recovery_and_no_wait_owned_by_ssh(self): + source = dispatcher.BOOTSTRAP + self.assertIn("RuntimeMaxSec=45min", source) + self.assertIn("TimeoutStopSec=20min", source) + self.assertIn("KillMode=mixed", source) + self.assertIn("+ 4000", source) + self.assertGreater(4000, 45 * 60 + 20 * 60) + self.assertNotIn("'--wait'", source) + self.assertNotIn("systemctl', 'stop'", source) + + def test_bootstrap_retains_intent_and_repeated_or_uncertain_launch_does_not_replay(self): + # Execute the real bootstrap with only its filesystem/root/systemd/clock + # adapters rebound, including a lost submission response. No host tools. + for uncertain in (False, True): + with self.subTest(uncertain=uncertain), tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "state" + source = dispatcher.BOOTSTRAP.replace("pathlib.Path('/var/lib/pulse-deploy/demo')", repr(root)) + # repr(Path) is not a Python literal: bind just the fixed path. + source = dispatcher.BOOTSTRAP.replace("'/var/lib/pulse-deploy/demo'", repr(str(root))) + source = source.replace("root.stat().st_uid == 0", "root.stat().st_uid == os.getuid()") + clock = [0] + calls = [] + payload = {"request": request(), "source": "c291cmNl", "installer": ""} + + def launch(argv, **kwargs): + calls.append(argv) + attempt = next((root / "attempts").iterdir()) + if uncertain: + raise subprocess.TimeoutExpired("systemd-run", 30) + (attempt / "receipt.json").write_text(json.dumps({"status": "committed"})) + return subprocess.CompletedProcess(argv, 0) + + class Input: + def read(self, _limit): + return json.dumps(payload) + + with patch.object(os, "geteuid", return_value=0), patch.object(subprocess, "run", side_effect=launch), \ + patch.object(dispatcher.sys, "stdin", Input()), patch("time.monotonic", side_effect=lambda: clock[0]), \ + patch("time.sleep", side_effect=lambda seconds: clock.__setitem__(0, clock[0] + seconds)), patch("builtins.print"): + for _ in range(2): + with self.assertRaises(SystemExit): + exec(compile(source, "fixed-bootstrap", "exec"), {}) + self.assertEqual(len(calls), 1) + attempt = next((root / "attempts").iterdir()) + self.assertTrue((attempt / "intent.json").exists()) + self.assertEqual((attempt / "request.json").stat().st_mode & 0o777, 0o600) + self.assertEqual(attempt.stat().st_mode & 0o777, 0o700) + if uncertain: + self.assertEqual(json.loads((attempt / "launch.json").read_text())["state"], "uncertain") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/recover-demo-server.yml b/.github/workflows/recover-demo-server.yml index c482081c59..b0dc92f2c0 100644 --- a/.github/workflows/recover-demo-server.yml +++ b/.github/workflows/recover-demo-server.yml @@ -15,7 +15,7 @@ concurrency: jobs: recover: runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 90 environment: demo-stable env: DEMO_EXPECTED_HOSTNAME: ${{ vars.DEMO_EXPECTED_HOSTNAME }} @@ -117,16 +117,8 @@ jobs: MOCK_UPDATE_INTERVAL: ${{ steps.runtime_profile.outputs.mock_update_interval }} run: | set -euo pipefail - ssh -i ~/.ssh/id_ed25519 \ - -o IdentitiesOnly=yes \ - -o StrictHostKeyChecking=yes \ - -o UserKnownHostsFile=~/.ssh/known_hosts \ - "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "bash -s -- $(printf '%q ' "$DEMO_EXPECTED_HOSTNAME" "$DEMO_LOCAL_BASE_URL" "$EXPECTED_VERSION" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_SAMPLE_INTERVAL" "$MOCK_UPDATE_INTERVAL")" \ - < .github/scripts/recover-demo-runtime.sh \ - > demo-runtime-recovery.json - jq -e '.schema_version == 1 and (.status == "recovered" or .status == "healthy_noop")' \ - demo-runtime-recovery.json + bash .github/scripts/recover-demo-runtime.sh > demo-runtime-recovery.json + jq -e '.schema_version == 2 and (.status == "committed" or .status == "healthy_noop") and .forward.elapsed_seconds >= 300' demo-runtime-recovery.json echo "mutated=$(jq -r '.mutated' demo-runtime-recovery.json)" >> "$GITHUB_OUTPUT" - name: Capture bounded Pulse failure diagnostics @@ -199,22 +191,9 @@ jobs: export PULSE_DEMO_AUTH_PASS="${DEMO_AUTH_PASS:-demo}" ./scripts/run_demo_public_browser_smoke.sh - - name: Compensate failed mutated recovery - if: failure() && steps.recovery.outputs.mutated == 'true' - env: - DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} - DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} - run: | - set -euo pipefail - BEFORE="$(ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "systemctl show pulse --property=ActiveState,SubState,MainPID --value | paste -sd, -")" - ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "sudo systemctl stop pulse" - AFTER="$(ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "systemctl show pulse --property=ActiveState,SubState,MainPID --value | paste -sd, -")" - jq -n --arg before "$BEFORE" --arg after "$AFTER" \ - '{schema_version: 1, status: "compensated_unavailable", before: $before, after: $after}' \ - > demo-runtime-rollback.json + # The retained systemd transaction already owns compensation. A later + # browser/network failure is evidence to reconcile, not authority to stop + # an unrelated or already-committed Pulse process in a second SSH session. - name: Retain bounded recovery evidence if: always() @@ -223,7 +202,6 @@ jobs: name: stable-demo-runtime-recovery-${{ github.run_id }} path: | demo-runtime-recovery.json - demo-runtime-rollback.json demo-runtime-diagnostics.txt if-no-files-found: warn retention-days: 14 diff --git a/.github/workflows/update-demo-server.yml b/.github/workflows/update-demo-server.yml index 996f7bda56..214115ebbe 100644 --- a/.github/workflows/update-demo-server.yml +++ b/.github/workflows/update-demo-server.yml @@ -189,7 +189,7 @@ jobs: needs: resolve if: needs.resolve.outputs.skip != 'true' runs-on: ubuntu-24.04 - timeout-minutes: 60 + timeout-minutes: 90 environment: ${{ needs.resolve.outputs.environment_name }} env: DEMO_EXPECTED_HOSTNAME: ${{ vars.DEMO_EXPECTED_HOSTNAME }} @@ -450,309 +450,41 @@ jobs: echo "::error::Verification-only check found a demo version mismatch; refusing to update the host." exit 1 - - name: Prepare demo host storage - if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true' - env: - DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} - DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} - WORKFLOW_OUTPUT_1: ${{ steps.config.outputs.service_name }} - run: | - set -euo pipefail - SERVICE_NAME="${WORKFLOW_OUTPUT_1}" - REMOTE_SCRIPT=$(cat <<'EOF' - set -euo pipefail - SERVICE_NAME="$1" - CONFIG_DIR="/etc/${SERVICE_NAME:-pulse}" - BACKUP_PARENT="$(dirname "$CONFIG_DIR")" - BACKUP_BASENAME="$(basename "$CONFIG_DIR")" - KEEP_BACKUPS=2 - BACKUP_MARGIN_KB=$((64 * 1024)) - - echo "Demo host filesystem before cleanup:" - df -h "$BACKUP_PARENT" /tmp || true - - mapfile -t old_backups < <( - sudo find "$BACKUP_PARENT" -maxdepth 1 -type d -name "${BACKUP_BASENAME}.backup.*" -printf '%T@ %p\n' 2>/dev/null \ - | sort -nr \ - | awk -v keep="$KEEP_BACKUPS" 'NR > keep {sub(/^[^ ]+ /, ""); print}' - ) - if ((${#old_backups[@]} > 0)); then - printf 'Removing %d old demo backup directorie(s).\n' "${#old_backups[@]}" - printf '%s\0' "${old_backups[@]}" | sudo xargs -0 rm -rf -- - fi - - if [[ -d "$CONFIG_DIR" ]]; then - config_kb="$(sudo du -sk "$CONFIG_DIR" | awk 'NR == 1 {print $1}')" - free_kb="$(df -Pk "$BACKUP_PARENT" | awk 'NR == 2 {print $4}')" - required_kb=$((config_kb + BACKUP_MARGIN_KB)) - if ((free_kb < required_kb)); then - mapfile -t remaining_backups < <( - sudo find "$BACKUP_PARENT" -maxdepth 1 -type d -name "${BACKUP_BASENAME}.backup.*" -printf '%T@ %p\n' 2>/dev/null \ - | sort -n \ - | awk '{sub(/^[^ ]+ /, ""); print}' - ) - for backup_dir in "${remaining_backups[@]}"; do - printf 'Removing demo backup to restore install headroom: %s\n' "$backup_dir" - sudo rm -rf -- "$backup_dir" - free_kb="$(df -Pk "$BACKUP_PARENT" | awk 'NR == 2 {print $4}')" - if ((free_kb >= required_kb)); then - break - fi - done - fi - pruned_volatile=false - if ((free_kb < required_kb)); then - echo "Pruning demo volatile runtime stores to restore install headroom." - if command -v systemctl >/dev/null 2>&1; then - sudo systemctl stop "$SERVICE_NAME" || true - fi - mapfile -d '' volatile_paths < <( - sudo find "$CONFIG_DIR" -xdev -type f \( \ - -name "metrics.db" -o \ - -name "metrics.db-shm" -o \ - -name "metrics.db-wal" -o \ - -name "notification_queue.db" -o \ - -name "notification_queue.db-shm" -o \ - -name "notification_queue.db-wal" -o \ - -name "deploy.db" -o \ - -name "deploy.db-shm" -o \ - -name "deploy.db-wal" \ - \) -print0 2>/dev/null || true - ) - for volatile_path in "${volatile_paths[@]}"; do - printf 'Removing demo volatile store: %s\n' "$(basename "$volatile_path")" - sudo rm -f -- "$volatile_path" - pruned_volatile=true - done - config_kb="$(sudo du -sk "$CONFIG_DIR" | awk 'NR == 1 {print $1}')" - free_kb="$(df -Pk "$BACKUP_PARENT" | awk 'NR == 2 {print $4}')" - required_kb=$((config_kb + BACKUP_MARGIN_KB)) - fi - if ((free_kb < required_kb)); then - echo "::error::Demo host does not have enough free space to back up $CONFIG_DIR before install." - echo "Config: ${config_kb}KB, available: ${free_kb}KB, required: ${required_kb}KB" - sudo find "$BACKUP_PARENT" -maxdepth 1 -type d -name "${BACKUP_BASENAME}.backup.*" -printf '%TY-%Tm-%Td %TH:%TM %p\n' 2>/dev/null | sort || true - if [[ "$pruned_volatile" == "true" ]] && command -v systemctl >/dev/null 2>&1; then - sudo systemctl start "$SERVICE_NAME" || true - fi - exit 1 - fi - fi - - echo "Demo host filesystem after cleanup:" - df -h "$BACKUP_PARENT" /tmp || true - EOF - ) - ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" "bash -s -- $(printf '%q ' "$SERVICE_NAME")" <<<"$REMOTE_SCRIPT" - - - name: Upload tagged installer - if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true' - env: - DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} - DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} - run: | - set -euo pipefail - scp -i ~/.ssh/id_ed25519 /tmp/pulse-install.sh "$DEMO_SERVER_USER@$DEMO_SERVER_HOST:/tmp/pulse-install.sh" - - - name: Update demo server - if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true' - env: - DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} - DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} - WORKFLOW_OUTPUT_1: ${{ needs.resolve.outputs.tag }} - WORKFLOW_OUTPUT_2: ${{ steps.config.outputs.service_name }} - run: | - set -euo pipefail - TAG="${WORKFLOW_OUTPUT_1}" - SERVICE_NAME="${WORKFLOW_OUTPUT_2}" - REMOTE_SCRIPT=$(cat <<'EOF' - set -euo pipefail - TAG="$1" - SERVICE_NAME="$2" - INSTALLER_ENV=() - if [ -n "$SERVICE_NAME" ]; then - INSTALLER_ENV+=("PULSE_SERVICE_NAME=$SERVICE_NAME") - fi - sudo env "${INSTALLER_ENV[@]}" bash /tmp/pulse-install.sh --version "$TAG" - rm -f /tmp/pulse-install.sh - EOF - ) - ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "bash -s -- $(printf '%q ' "$TAG" "$SERVICE_NAME")" <<<"$REMOTE_SCRIPT" - - - name: Restore demo runtime configuration + - name: Apply guarded demo transaction if: inputs.verify_only != true env: DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} - WORKFLOW_OUTPUT_1: ${{ steps.config.outputs.service_name }} - WORKFLOW_OUTPUT_2: ${{ needs.resolve.outputs.mock_nodes }} - WORKFLOW_OUTPUT_3: ${{ needs.resolve.outputs.mock_vms_per_node }} - WORKFLOW_OUTPUT_4: ${{ needs.resolve.outputs.mock_lxcs_per_node }} - WORKFLOW_OUTPUT_5: ${{ needs.resolve.outputs.mock_docker_hosts }} - WORKFLOW_OUTPUT_6: ${{ needs.resolve.outputs.mock_docker_containers }} - WORKFLOW_OUTPUT_7: ${{ needs.resolve.outputs.mock_generic_hosts }} - WORKFLOW_OUTPUT_8: ${{ needs.resolve.outputs.mock_k8s_clusters }} - WORKFLOW_OUTPUT_9: ${{ needs.resolve.outputs.mock_k8s_nodes }} - WORKFLOW_OUTPUT_10: ${{ needs.resolve.outputs.mock_k8s_pods }} - WORKFLOW_OUTPUT_11: ${{ needs.resolve.outputs.mock_k8s_deployments }} - WORKFLOW_OUTPUT_12: ${{ needs.resolve.outputs.mock_seed_duration }} - WORKFLOW_OUTPUT_13: ${{ needs.resolve.outputs.mock_sample_interval }} - WORKFLOW_OUTPUT_14: ${{ needs.resolve.outputs.mock_update_interval }} + EXPECTED_VERSION: ${{ needs.resolve.outputs.tag }} + CURRENT_MATCHES: ${{ steps.current.outputs.skip_current }} + MOCK_NODES: ${{ needs.resolve.outputs.mock_nodes }} + MOCK_VMS_PER_NODE: ${{ needs.resolve.outputs.mock_vms_per_node }} + MOCK_LXCS_PER_NODE: ${{ needs.resolve.outputs.mock_lxcs_per_node }} + MOCK_DOCKER_HOSTS: ${{ needs.resolve.outputs.mock_docker_hosts }} + MOCK_DOCKER_CONTAINERS: ${{ needs.resolve.outputs.mock_docker_containers }} + MOCK_GENERIC_HOSTS: ${{ needs.resolve.outputs.mock_generic_hosts }} + MOCK_K8S_CLUSTERS: ${{ needs.resolve.outputs.mock_k8s_clusters }} + MOCK_K8S_NODES: ${{ needs.resolve.outputs.mock_k8s_nodes }} + MOCK_K8S_PODS: ${{ needs.resolve.outputs.mock_k8s_pods }} + MOCK_K8S_DEPLOYMENTS: ${{ needs.resolve.outputs.mock_k8s_deployments }} + MOCK_SEED_DURATION: ${{ needs.resolve.outputs.mock_seed_duration }} + MOCK_SAMPLE_INTERVAL: ${{ needs.resolve.outputs.mock_sample_interval }} + MOCK_UPDATE_INTERVAL: ${{ needs.resolve.outputs.mock_update_interval }} run: | set -euo pipefail - SERVICE_NAME="${WORKFLOW_OUTPUT_1}" - MOCK_NODES="${WORKFLOW_OUTPUT_2}" - MOCK_VMS_PER_NODE="${WORKFLOW_OUTPUT_3}" - MOCK_LXCS_PER_NODE="${WORKFLOW_OUTPUT_4}" - MOCK_DOCKER_HOSTS="${WORKFLOW_OUTPUT_5}" - MOCK_DOCKER_CONTAINERS="${WORKFLOW_OUTPUT_6}" - MOCK_GENERIC_HOSTS="${WORKFLOW_OUTPUT_7}" - MOCK_K8S_CLUSTERS="${WORKFLOW_OUTPUT_8}" - MOCK_K8S_NODES="${WORKFLOW_OUTPUT_9}" - MOCK_K8S_PODS="${WORKFLOW_OUTPUT_10}" - MOCK_K8S_DEPLOYMENTS="${WORKFLOW_OUTPUT_11}" - MOCK_SEED_DURATION="${WORKFLOW_OUTPUT_12}" - MOCK_SAMPLE_INTERVAL="${WORKFLOW_OUTPUT_13}" - MOCK_UPDATE_INTERVAL="${WORKFLOW_OUTPUT_14}" - REMOTE_SCRIPT=$(cat <<'EOF' - set -euo pipefail - SERVICE_NAME="${1:-pulse}" - DEMO_LOCAL_BASE_URL="$2" - MOCK_NODES="$3" - MOCK_VMS_PER_NODE="$4" - MOCK_LXCS_PER_NODE="$5" - MOCK_DOCKER_HOSTS="$6" - MOCK_DOCKER_CONTAINERS="$7" - MOCK_GENERIC_HOSTS="$8" - MOCK_K8S_CLUSTERS="$9" - MOCK_K8S_NODES="${10}" - MOCK_K8S_PODS="${11}" - MOCK_K8S_DEPLOYMENTS="${12}" - MOCK_SEED_DURATION="${13}" - MOCK_SAMPLE_INTERVAL="${14}" - MOCK_UPDATE_INTERVAL="${15}" - CONFIG_DIR="/etc/${SERVICE_NAME:-pulse}" - SERVICE_USER="$(systemctl show "$SERVICE_NAME" --property=User --value 2>/dev/null || true)" - SERVICE_GROUP="$(systemctl show "$SERVICE_NAME" --property=Group --value 2>/dev/null || true)" - SERVICE_USER="${SERVICE_USER:-pulse}" - SERVICE_GROUP="${SERVICE_GROUP:-$SERVICE_USER}" + MODE=update + if [ "$CURRENT_MATCHES" = "true" ]; then MODE=recover; fi + python3 .github/scripts/dispatch-demo-runtime.py "$MODE" > demo-runtime-transaction.json + jq -e '.schema_version == 2 and (.status == "committed" or .status == "healthy_noop") and .forward.elapsed_seconds >= 300' demo-runtime-transaction.json - resolve_config_dir() { - local fallback="$CONFIG_DIR" - local unit_env="" - local fragment="" - unit_env="$(systemctl show "$SERVICE_NAME" --property=Environment --value 2>/dev/null || true)" - for entry in $unit_env; do - case "$entry" in - PULSE_DATA_DIR=*) - printf '%s\n' "${entry#PULSE_DATA_DIR=}" - return 0 - ;; - esac - done - - fragment="$(systemctl show "$SERVICE_NAME" --property=FragmentPath --value 2>/dev/null || true)" - if [ -n "$fragment" ] && sudo test -r "$fragment"; then - local unit_data_dir - unit_data_dir="$(sudo sed -n -E 's/^Environment="?PULSE_DATA_DIR=([^"]*)"?$/\1/p' "$fragment" | tail -1)" - if [ -n "$unit_data_dir" ]; then - printf '%s\n' "$unit_data_dir" - return 0 - fi - fi - - printf '%s\n' "$fallback" - } - - CONFIG_DIR="$(resolve_config_dir)" - ENV_FILE="$CONFIG_DIR/.env" - - set_env_value() { - local key="$1" - local value="$2" - sudo mkdir -p "$CONFIG_DIR" - if sudo test -f "$ENV_FILE"; then - if sudo grep -Eq "^[[:space:]]*${key}=" "$ENV_FILE"; then - sudo sed -i "s|^[[:space:]]*${key}=.*|${key}=${value}|" "$ENV_FILE" - else - printf '\n%s=%s\n' "$key" "$value" | sudo tee -a "$ENV_FILE" >/dev/null - fi - else - printf '# Pulse demo runtime configuration\n%s=%s\n' "$key" "$value" | sudo tee "$ENV_FILE" >/dev/null - fi - } - - ensure_demo_fixture_entitlement() { - local billing_file="$CONFIG_DIR/billing.json" - local tmp_file - tmp_file="$(mktemp)" - sudo mkdir -p "$CONFIG_DIR" - - if sudo test -s "$billing_file" && sudo jq ' - .capabilities = (((.capabilities // []) + ["demo_fixtures"]) | unique) - | .limits = (.limits // {}) - | .meters_enabled = (.meters_enabled // []) - | .plan_version = (if (.plan_version // "") == "" then "community" else .plan_version end) - | .subscription_state = (if (.subscription_state // "") == "" then "active" else .subscription_state end) - | del(.integrity) - ' "$billing_file" > "$tmp_file"; then - : - else - cat > "$tmp_file" <<'JSON' - {"capabilities":["demo_fixtures"],"limits":{},"meters_enabled":[],"plan_version":"community","subscription_state":"active"} - JSON - fi - - sudo tee "$billing_file" < "$tmp_file" >/dev/null - rm -f "$tmp_file" - sudo chown "$SERVICE_USER:$SERVICE_GROUP" "$billing_file" || sudo chown "$SERVICE_USER" "$billing_file" || true - sudo chmod 600 "$billing_file" - echo "Demo fixture entitlement ensured in governed demo billing state." - } - - set_env_value DEMO_MODE true - set_env_value PULSE_MOCK_MODE true - set_env_value PULSE_MOCK_NODES "$MOCK_NODES" - set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE" - set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE" - set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS" - set_env_value PULSE_MOCK_DOCKER_CONTAINERS "$MOCK_DOCKER_CONTAINERS" - set_env_value PULSE_MOCK_GENERIC_HOSTS "$MOCK_GENERIC_HOSTS" - set_env_value PULSE_MOCK_K8S_CLUSTERS "$MOCK_K8S_CLUSTERS" - set_env_value PULSE_MOCK_K8S_NODES "$MOCK_K8S_NODES" - set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS" - set_env_value PULSE_MOCK_K8S_DEPLOYMENTS "$MOCK_K8S_DEPLOYMENTS" - set_env_value PULSE_MOCK_RANDOM_METRICS true - set_env_value PULSE_MOCK_STOPPED_PERCENT 6 - # Never carry the isolated local-dev metrics-store opt-in onto the - # persistent public demo data directory. - set_env_value PULSE_MOCK_SEED_METRICS_STORE false - set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION" - set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL" - set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL" - - ensure_demo_fixture_entitlement - - sudo chown "$SERVICE_USER:$SERVICE_GROUP" "$ENV_FILE" || sudo chown "$SERVICE_USER" "$ENV_FILE" || true - sudo chmod 600 "$ENV_FILE" - sudo systemctl daemon-reload - sudo systemctl restart "$SERVICE_NAME" - - for attempt in $(seq 1 30); do - if curl -fsS "${DEMO_LOCAL_BASE_URL}/api/health" >/dev/null; then - echo "Demo service restarted with governed demo runtime configuration." - exit 0 - fi - sleep 2 - done - - echo "::error::Demo service did not become healthy after restoring demo runtime configuration." - exit 1 - EOF - ) - ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "bash -s -- $(printf '%q ' "$SERVICE_NAME" "$DEMO_LOCAL_BASE_URL" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_SAMPLE_INTERVAL" "$MOCK_UPDATE_INTERVAL")" <<<"$REMOTE_SCRIPT" + - name: Retain guarded demo transaction evidence + if: always() && inputs.verify_only != true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: stable-demo-transaction-${{ github.run_id }} + path: demo-runtime-transaction.json + if-no-files-found: warn + retention-days: 14 - name: Verify demo runtime env: diff --git a/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md b/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md new file mode 100644 index 0000000000..af5e4322da --- /dev/null +++ b/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md @@ -0,0 +1,60 @@ +# Stable demo transaction and acceptance + +This continues the existing update/recovery route. It was built for a single +demo workflow and first-success health; the actual host also runs Relay and +Caddy. An immediate healthy sample does not cover a delayed failure, and a +workflow queue does not exclude another repository's host deployment. + +## Implemented boundary + +- Both existing mutating workflows submit the same reviewed Python engine. + Update still requires the immutable stable activation and customer-promotion + lease, exact tagged installer and trusted release signing key. Verification + only never submits it. Recovery never installs a different executable. +- The independently owned systemd child holds the existing Relay deployment + file lock through capture, installation/profile work, forward observation, + complete restoration and recovery observation. It does not restart Relay or + Caddy, upgrade a package, lift the Caddy hold or activate frozen commercial copy. +- The complete quiescent Pulse executable/unit/drop-in/data estate is retained + privately before mutation. No backup or database is deleted for space. The + unhealthy demo-only route retains the original generated operational history + before its existing bounded-profile reset. No snapshot is automatically erased. +- Readiness has a separate 60-second limit. Forward, healthy-noop and restored + paths require 300 seconds of actual local/public/Relay HTTP 200, expected + version, stable service PID/restarts and available new-journal observations. + Transport errors, redirects, missing state/journals and new crashes fail. + Recovery uses fresh cursors; a retained forward failure remains a failure. +- Failed/unhealthy/uncertain recovery remains owned and blocks subsequent + mutation. A prior failed baseline is not a healthy rollback. No crash dump is + induced. Receipt fields contain fixed diagnostics and identities, never raw + journal/HTTP bodies, environment values or snapshot contents. +- Source-bound intent precedes systemd submission. An identical request reads + its receipt and cannot start another child after a lost response. SSH loss + does not terminate the child. TERM/INT/HUP cannot abort owned recovery. The + 45-minute runtime, 20-minute stop, 4,000-second remote observer, 4,100-second + client and 90-minute workflow budgets cover both observation windows. + +## Available validation and remaining result + +`TestDemoTransactionConnectedRecoveryControls` executes the real engine and +bootstrap using a real private filesystem estate and virtual-clock command, +service, journal and HTTP adapters. Tests retain delayed 55/299-second failure, +changed executable/unit/data rollback, cancellation, failed and unhealthy +recovery, headroom refusal, shared-lock contention, source mismatch, journal +failure, cohost restart, source-bound repeated/lost submission and no-op proof. +The production window is not shortened. This is **not native acceptance**. + +Before treating this route as restored, the exact reviewed source must return +native disposable systemd/SSH-loss receipts and signed published-installer +forward/reverse data/identity results. Then reconcile actual demo workflow and +public browser readback through the release owner; source tests do not establish +an installed recovery. Delivery owns that continuation and the remaining ingress +migration. Unrestricted root/Tailscale administration can still bypass a source +transaction; this candidate is not host enforcement. Workers neither install +policy/credentials nor replay a successful publication to test it. + +Retained failed/unknown estates require the operational owner's reconciliation; +do not remove a lock, intent or snapshot merely to permit a fresh sample. A +later frontend/network failure is not authority for a second SSH session to +stop a committed or unrelated service. Keep the signed packet, failure and +native/browser acceptance distinct in the release judgment. diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index efa5d4e8c8..78533e9a3f 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -15,6 +15,36 @@ ## Purpose +### Demo transactions share the service host, not just a workflow queue + +Stable demo update and recovery keep their existing publication/activation, +exact-tag installer signing key, Tailscale, pinned SSH and environment boundaries. +The mutating paths now share the Relay host's existing +`/var/lib/pulse-deploy/relay/deploy.lock` and one retained systemd transaction. +Every successful path, including a healthy no-op, needs a complete 300-second +local/public/Relay health, version, service PID/restart and new-journal-crash +window. Readiness is separate. Missing observations fail closed. + +Before installation or runtime-profile mutation, stop only Pulse and durably +capture its executable, unit/drop-ins and entire persistent data estate. Refuse +insufficient space rather than prune backups/databases. A failed healthy change +restores that complete estate and observes another 300 seconds; the original +operation still fails. An unhealthy recovery baseline is never claimed as a +healthy rollback. Retain its original generated demo history, all snapshots and +failed/uncertain terminal evidence privately. An unresolved estate blocks later +mutation. No source-worker production capability, package upgrade, Relay copy +activation or credential is added. + +SSH observes, but never owns or cancels, the systemd child. Durable intent +precedes submission; the exact request identity is not replayed after a lost +response. Forward cancellation initiates restoration; recovery ignores +TERM/INT/HUP until its watch and terminal receipt are retained. Child, stop, +observer and workflow deadlines cover both full windows. A later browser or +network check must not stop a committed or unrelated service in another SSH +session. These connected source controls do not establish native systemd, +signed installed forward/reverse acceptance, ingress enforcement or customer +recovery. See [demo transaction acceptance](../DEMO_RUNTIME_TRANSACTION.md). + ### Superseded pull request validation Build and Test and Core E2E cancel an older run for the same pull request when @@ -732,6 +762,8 @@ release-latency optimization. 23. `.github/workflows/update-demo-server.yml` 23a. `.github/workflows/recover-demo-server.yml` 23b. `.github/scripts/recover-demo-runtime.sh` +23c. `.github/scripts/demo-runtime-transaction.py` +23d. `.github/scripts/dispatch-demo-runtime.py` 23c. `.github/scripts/resolve-demo-runtime-profile.sh` 24. `.github/workflows/validate-release-assets.yml` 25. `.github/workflows/install-sh-smoke.yml` diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index 5b59559345..69c3b883c3 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -4515,6 +4515,8 @@ ".dockerignore", ".github/ISSUE_TEMPLATE/v6_rc_feedback.yml", ".github/scripts/check-demo-reachability.sh", + ".github/scripts/demo-runtime-transaction.py", + ".github/scripts/dispatch-demo-runtime.py", ".github/scripts/internal-api-test-seconds.txt", ".github/scripts/record-internal-api-test-seconds.py", ".github/scripts/recover-demo-runtime.sh", @@ -4544,6 +4546,9 @@ ".github/workflows/retry-release-convergence.yml", ".github/workflows/update-demo-server.yml", ".github/workflows/validate-release-assets.yml", + "Dockerfile", + "Makefile", + "VERSION", "cmd/pulse-control-plane/main.go", "cmd/pulse-control-plane/provider_msp.go", "cmd/pulse-control-plane/provider_msp_backup.go", @@ -4553,13 +4558,12 @@ "cmd/pulse-control-plane/provider_msp_recover.go", "cmd/pulse-control-plane/provider_msp_status.go", "docker-compose.yml", - "Dockerfile", "docs/MSP.md", + "docs/RELEASE_NOTES.md", + "docs/UPGRADE_v6.md", "docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md", "docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md", "docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md", - "docs/RELEASE_NOTES.md", - "docs/UPGRADE_v6.md", "frontend-modern/package-lock.json", "frontend-modern/package.json", "frontend-modern/public/docs/MSP.md", @@ -4579,7 +4583,6 @@ "internal/cloudcp/provider_msp_backup.go", "internal/cloudcp/provider_msp_recovery.go", "internal/cloudcp/tenant_runtime_rollout.go", - "Makefile", "package-lock.json", "package.json", "pulse-enterprise:.github/workflows/build-pro-release.yml", @@ -4654,17 +4657,16 @@ "scripts/uninstall-sensor-proxy.sh", "scripts/validate-release.sh", "scripts/verify-github-release-integrity.sh", - "tests/integration/playwright.config.ts", "tests/integration/QUICK_START.md", "tests/integration/README.md", + "tests/integration/playwright.config.ts", "tests/integration/scripts/bootstrap-hosted-mobile-onboarding.mjs", "tests/integration/scripts/hosted-mobile-token-runtime.mjs", "tests/integration/scripts/hosted-tenant-runtime.mjs", "tests/integration/scripts/managed-dev-runtime.mjs", "tests/integration/scripts/relay-mobile-token-helper.go", "tests/integration/tests/helpers.ts", - "tests/integration/tests/runtime-defaults.ts", - "VERSION" + "tests/integration/tests/runtime-defaults.ts" ], "release_cycle_artifact_globs": [ "docs/RELEASE_NOTES.md", @@ -4749,6 +4751,8 @@ "match_files": [ ".github/ISSUE_TEMPLATE/v6_rc_feedback.yml", ".github/scripts/check-demo-reachability.sh", + ".github/scripts/demo-runtime-transaction.py", + ".github/scripts/dispatch-demo-runtime.py", ".github/scripts/recover-demo-runtime.sh", ".github/scripts/resolve-demo-runtime-profile.sh", ".github/scripts/setup-demo-ssh.sh", @@ -4771,11 +4775,12 @@ ".github/workflows/retry-release-convergence.yml", ".github/workflows/update-demo-server.yml", ".github/workflows/validate-release-assets.yml", + "VERSION", + "docs/RELEASE_NOTES.md", + "docs/UPGRADE_v6.md", "docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md", "docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md", "docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md", - "docs/RELEASE_NOTES.md", - "docs/UPGRADE_v6.md", "frontend-modern/public/docs/UPGRADE_v6.md", "frontend-modern/src/components/UpdateBanner.tsx", "frontend-modern/src/components/WhatsNewCard.tsx", @@ -4803,17 +4808,18 @@ "scripts/trigger-release-dry-run.sh", "scripts/trigger-release.sh", "scripts/trigger-stable-patch.sh", - "scripts/verify-github-release-integrity.sh", - "VERSION" + "scripts/verify-github-release-integrity.sh" ], "allow_same_subsystem_tests": false, "test_prefixes": [], "exact_files": [ + ".github/scripts/tests/test_demo_runtime_transaction.py", "frontend-modern/src/components/__tests__/whatsNewModel.test.ts", "frontend-modern/src/utils/__tests__/docsLinks.test.ts", "pulse-enterprise:scripts/validate-pro-release-line_test.sh", "pulse-pro:scripts/tests/test_validate_paid_runtime_distribution.py", "scripts/installtests/build_release_assets_test.go", + "scripts/installtests/demo_transaction_test.go", "scripts/release_control/internal/record_rc_to_ga_rehearsal_test.py", "scripts/release_control/internal/release_preflight_test.py", "scripts/release_control/mobile_release_gate_test.py", diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 94688451c0..fff9fd54fd 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -2306,7 +2306,11 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { if err != nil { t.Fatalf("read demo runtime profile resolver: %v", err) } - workflow := string(workflowBytes) + "\n" + string(profileBytes) + transactionBytes, err := os.ReadFile(repoFile(".github", "scripts", "demo-runtime-transaction.py")) + if err != nil { + t.Fatal(err) + } + workflow := string(workflowBytes) + "\n" + string(profileBytes) + "\n" + string(transactionBytes) required := []string{ `- name: Tailscale`, `uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3`, @@ -2319,7 +2323,6 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { `workflow_call:`, `verify_only:`, `Waiting for activated release assets to be available`, - `bash /tmp/pulse-install.sh --version "$TAG"`, `Refuse mutation during verification-only checks`, `uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0`, `go run ./scripts/release_update_key.go public-key-ssh`, @@ -2327,15 +2330,10 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { `Verify target host identity`, `bash .github/scripts/setup-demo-ssh.sh`, `Demo environment points at host $REMOTE_HOSTNAME but expected $DEMO_EXPECTED_HOSTNAME.`, - `Prepare demo host storage`, - `KEEP_BACKUPS=2`, - `Removing demo backup to restore install headroom: %s`, - `Pruning demo volatile runtime stores to restore install headroom.`, - `sudo find "$CONFIG_DIR" -xdev -type f`, - `-name "metrics.db"`, - `Removing demo volatile store: %s`, - `Demo host does not have enough free space to back up $CONFIG_DIR before install.`, - `Restore demo runtime configuration`, + `Apply guarded demo transaction`, + `python3 .github/scripts/dispatch-demo-runtime.py "$MODE"`, + `Retain guarded demo transaction evidence`, + `.forward.elapsed_seconds >= 300`, `Resolve target-compatible demo runtime profile`, `git grep -q 'mockEagerHistoryPVEGuestLimit'`, `git grep -q 'UpdateMetricCohort'`, @@ -2361,22 +2359,13 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { `MOCK_SEED_DURATION=2h`, `MOCK_SAMPLE_INTERVAL=5m`, `MOCK_UPDATE_INTERVAL=15s`, - `resolve_config_dir`, - `set_env_value DEMO_MODE true`, - `set_env_value PULSE_MOCK_MODE true`, - `set_env_value PULSE_MOCK_NODES "$MOCK_NODES"`, - `set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE"`, - `set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE"`, - `set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS"`, - `set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS"`, - `set_env_value PULSE_MOCK_SEED_METRICS_STORE false`, - `set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"`, - `set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"`, - `set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"`, - `ensure_demo_fixture_entitlement`, - `"demo_fixtures"`, - `del(.integrity)`, - `Demo fixture entitlement ensured in governed demo billing state.`, + `WINDOW = 300`, + `/var/lib/pulse-deploy/relay/deploy.lock`, + `snapshot-headroom`, + `restored-estate-identity`, + `demo_fixtures`, + `PULSE_MOCK_SEED_METRICS_STORE`, + `installer-identity`, `/api/license/runtime-capabilities`, `Mock mode enabled`, `Demo server mock mode did not enable after entitlement sync`, diff --git a/scripts/installtests/demo_transaction_test.go b/scripts/installtests/demo_transaction_test.go new file mode 100644 index 0000000000..f53ed0abc0 --- /dev/null +++ b/scripts/installtests/demo_transaction_test.go @@ -0,0 +1,56 @@ +package installtests + +import ( + "os" + "os/exec" + "strings" + "testing" +) + +func TestDemoTransactionConnectedRecoveryControls(t *testing.T) { + cmd := exec.Command("python3", repoFile(".github", "scripts", "tests", "test_demo_runtime_transaction.py")) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("demo connected transaction controls failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "\nOK\n") { + t.Fatalf("demo controls did not return a complete unittest verdict: %s", output) + } +} + +func TestDemoTransactionKeepsSignedAdmissionAndVerificationOnlyReadOnly(t *testing.T) { + workflow, err := os.ReadFile(repoFile(".github", "workflows", "update-demo-server.yml")) + if err != nil { + t.Fatal(err) + } + text := string(workflow) + start := strings.Index(text, "- name: Apply guarded demo transaction") + end := strings.Index(text[start:], "- name: Retain guarded demo transaction evidence") + if start < 0 || end < 0 { + t.Fatal("missing retained demo transaction entry") + } + entry := text[start : start+end] + if !strings.Contains(entry, "if: inputs.verify_only != true") { + t.Fatal("verification-only invocation must never submit a transaction") + } + for _, forbidden := range []string{"Pruning demo volatile runtime stores", "Removing demo backup to restore install headroom", "sudo systemctl restart"} { + if strings.Contains(text, forbidden) { + t.Fatalf("unguarded demo mutation remains: %s", forbidden) + } + } + for _, required := range []string{"Require exact committed activation marker for mutation", "release-activation.json", "Stable demo mutation refuses mutable, inactive, or prerelease tag", "PINNED_RELEASE_SSH_PUBLIC_KEY", "git show \"refs/tags/${TAG}:install.sh\"", "Verify public browser smoke"} { + if !strings.Contains(text, required) { + t.Fatalf("lost signed/admission/customer verification boundary: %s", required) + } + } + recovery, err := os.ReadFile(repoFile(".github", "workflows", "recover-demo-server.yml")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(recovery), "sudo systemctl stop pulse") || strings.Contains(string(recovery), "Compensate failed mutated recovery") { + t.Fatal("later workflow failure must not stop a committed or unrelated service") + } + if !strings.Contains(string(recovery), ".forward.elapsed_seconds >= 300") || !strings.Contains(string(recovery), "if: always()") { + t.Fatal("recovery must validate and retain the actual terminal observation") + } +} From 751fb67523012658c305fd0dae12fba4cfe67c4c Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:12:47 +0100 Subject: [PATCH 2/7] Bind demo controls without broadening unrelated release proof Keep new source paths on their own connected proof policy after the full lookup suite exposed six broadened metadata expectations. Preserve terminal no-op cancellation evidence and the runtime file owner. Contract-Neutral: Correct connected observation and verification routing within the existing demo operations and authority. Change-source: pulse-maintainer --- .github/scripts/demo-runtime-transaction.py | 9 ++++++--- .../tests/test_demo_runtime_transaction.py | 13 ++++++++++--- .../v6/internal/subsystems/registry.json | 19 +++++++++++++++---- scripts/installtests/demo_transaction_test.go | 5 ++++- .../release_control/subsystem_lookup_test.py | 12 ++++++++++++ 5 files changed, 47 insertions(+), 11 deletions(-) diff --git a/.github/scripts/demo-runtime-transaction.py b/.github/scripts/demo-runtime-transaction.py index 60193e378e..5fa81dda04 100644 --- a/.github/scripts/demo-runtime-transaction.py +++ b/.github/scripts/demo-runtime-transaction.py @@ -275,6 +275,8 @@ class Host: pattern = re.compile(r"^[ \t]*" + re.escape(key) + r"=.*$", re.M) text = pattern.sub(key + "=" + value, text) if pattern.search(text) else text.rstrip("\n") + "\n" + key + "=" + value + "\n" env_file.write_text(text) + data_owner = PATHS["data"].stat() + os.chown(env_file, data_owner.st_uid, data_owner.st_gid) os.chmod(env_file, 0o600) billing = json.loads(billing_file.read_text()) if not isinstance(billing, dict) or not isinstance(billing.get("capabilities", []), list): @@ -282,6 +284,7 @@ class Host: billing["capabilities"] = sorted(set(billing.get("capabilities", []) + ["demo_fixtures"])) billing.pop("integrity", None) billing_file.write_text(json.dumps(billing)) + os.chown(billing_file, data_owner.st_uid, data_owner.st_gid) os.chmod(billing_file, 0o600) if unhealthy: for relative in DEMO_HISTORY: @@ -408,14 +411,14 @@ class Transaction: elif self.request["mode"] == "update": raise Failure("update-baseline-unhealthy") self.receipt["healthy_baseline"] = healthy_baseline + old_handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)} + for sig in old_handlers: + signal.signal(sig, cancel) # The no-op is observed too; an immediate healthy sample is not success. if self.request["mode"] == "recover" and healthy_baseline and self.host.profile_matches(self.request): self.watch("forward", original_version, self.host.cursors()) self.save("healthy_noop") return 0 - old_handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)} - for sig in old_handlers: - signal.signal(sig, cancel) self.save("capturing") stopped = True self.host.stop() diff --git a/.github/scripts/tests/test_demo_runtime_transaction.py b/.github/scripts/tests/test_demo_runtime_transaction.py index 36bb810f44..6050a380c8 100644 --- a/.github/scripts/tests/test_demo_runtime_transaction.py +++ b/.github/scripts/tests/test_demo_runtime_transaction.py @@ -1,4 +1,3 @@ -import copy import hashlib import importlib.util import json @@ -195,6 +194,16 @@ class TransactionTest(unittest.TestCase): self.assertFalse(receipt["mutated"]) self.assertEqual(self.host.stops, 0) + def test_noop_cancellation_retains_terminal_failure_without_stopping_service(self): + self.host.profile(request(), False) + self.host.cancel_at = 55 + code, receipt = self.run_transaction() + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "refused") + self.assertEqual(receipt["failure"], "cancelled-forward") + self.assertEqual(self.host.stops, 0) + self.assertFalse(receipt["mutated"]) + def test_update_restores_complete_changed_executable_unit_and_data_after_late_failure(self): self.host.fail_at = 55 code, receipt = self.run_transaction("update", "v6.4.6") @@ -404,8 +413,6 @@ class DispatchLifecycleTest(unittest.TestCase): for uncertain in (False, True): with self.subTest(uncertain=uncertain), tempfile.TemporaryDirectory() as directory: root = Path(directory) / "state" - source = dispatcher.BOOTSTRAP.replace("pathlib.Path('/var/lib/pulse-deploy/demo')", repr(root)) - # repr(Path) is not a Python literal: bind just the fixed path. source = dispatcher.BOOTSTRAP.replace("'/var/lib/pulse-deploy/demo'", repr(str(root))) source = source.replace("root.stat().st_uid == 0", "root.stat().st_uid == os.getuid()") clock = [0] diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index 69c3b883c3..9cf02a0271 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -4751,8 +4751,6 @@ "match_files": [ ".github/ISSUE_TEMPLATE/v6_rc_feedback.yml", ".github/scripts/check-demo-reachability.sh", - ".github/scripts/demo-runtime-transaction.py", - ".github/scripts/dispatch-demo-runtime.py", ".github/scripts/recover-demo-runtime.sh", ".github/scripts/resolve-demo-runtime-profile.sh", ".github/scripts/setup-demo-ssh.sh", @@ -4813,13 +4811,11 @@ "allow_same_subsystem_tests": false, "test_prefixes": [], "exact_files": [ - ".github/scripts/tests/test_demo_runtime_transaction.py", "frontend-modern/src/components/__tests__/whatsNewModel.test.ts", "frontend-modern/src/utils/__tests__/docsLinks.test.ts", "pulse-enterprise:scripts/validate-pro-release-line_test.sh", "pulse-pro:scripts/tests/test_validate_paid_runtime_distribution.py", "scripts/installtests/build_release_assets_test.go", - "scripts/installtests/demo_transaction_test.go", "scripts/release_control/internal/record_rc_to_ga_rehearsal_test.py", "scripts/release_control/internal/release_preflight_test.py", "scripts/release_control/mobile_release_gate_test.py", @@ -5173,6 +5169,21 @@ "scripts/installtests/install_sh_test.go", "scripts/installtests/safe_profile_migration_test.go" ] + }, + { + "id": "demo-runtime-transaction", + "label": "demo shared-host transaction and retained recovery proof", + "match_prefixes": [], + "match_files": [ + ".github/scripts/demo-runtime-transaction.py", + ".github/scripts/dispatch-demo-runtime.py" + ], + "allow_same_subsystem_tests": false, + "test_prefixes": [], + "exact_files": [ + "scripts/installtests/demo_transaction_test.go", + ".github/scripts/tests/test_demo_runtime_transaction.py" + ] } ], "match_files": null diff --git a/scripts/installtests/demo_transaction_test.go b/scripts/installtests/demo_transaction_test.go index f53ed0abc0..ae800da558 100644 --- a/scripts/installtests/demo_transaction_test.go +++ b/scripts/installtests/demo_transaction_test.go @@ -25,8 +25,11 @@ func TestDemoTransactionKeepsSignedAdmissionAndVerificationOnlyReadOnly(t *testi } text := string(workflow) start := strings.Index(text, "- name: Apply guarded demo transaction") + if start < 0 { + t.Fatal("missing retained demo transaction entry") + } end := strings.Index(text[start:], "- name: Retain guarded demo transaction evidence") - if start < 0 || end < 0 { + if end < 0 { t.Fatal("missing retained demo transaction entry") } entry := text[start : start+end] diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py index a4dc01c378..d77aece65c 100644 --- a/scripts/release_control/subsystem_lookup_test.py +++ b/scripts/release_control/subsystem_lookup_test.py @@ -100,6 +100,18 @@ def _contract_reference(contract_path: str, needle: str, runtime_path: str) -> d class SubsystemLookupTest(unittest.TestCase): + + def test_demo_transaction_sources_have_connected_proof_without_widening_release_metadata(self): + for path in (".github/scripts/demo-runtime-transaction.py", ".github/scripts/dispatch-demo-runtime.py"): + with self.subTest(path=path): + result = lookup_paths([path]) + subsystem = next(item for item in result["subsystems"] if item["id"] == "deployment-installability") + self.assertEqual(subsystem["matched_verification_policies"][0]["id"], "demo-runtime-transaction") + self.assertEqual(sorted(subsystem["required_verification"]["exact_files"]), [ + ".github/scripts/tests/test_demo_runtime_transaction.py", + "scripts/installtests/demo_transaction_test.go", + ]) + def git(self, repo_root: Path, *args: str) -> None: subprocess.run( ["git", *args], From b3937a4803469bd555239eca1c4f2628862497ec Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:13:46 +0100 Subject: [PATCH 3/7] Use the actual lookup receipt shape in demo ownership proof Assert the existing file-level verification receipt so the new routing control exercises real ownership instead of assuming an unavailable summary field. Contract-Neutral: Test-only assertion of existing subsystem lookup output; no runtime or authority change. Change-source: pulse-maintainer --- scripts/release_control/subsystem_lookup_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py index d77aece65c..1fae39c773 100644 --- a/scripts/release_control/subsystem_lookup_test.py +++ b/scripts/release_control/subsystem_lookup_test.py @@ -105,9 +105,11 @@ class SubsystemLookupTest(unittest.TestCase): for path in (".github/scripts/demo-runtime-transaction.py", ".github/scripts/dispatch-demo-runtime.py"): with self.subTest(path=path): result = lookup_paths([path]) - subsystem = next(item for item in result["subsystems"] if item["id"] == "deployment-installability") - self.assertEqual(subsystem["matched_verification_policies"][0]["id"], "demo-runtime-transaction") - self.assertEqual(sorted(subsystem["required_verification"]["exact_files"]), [ + self.assertEqual(result["unowned_runtime_files"], []) + self.assertEqual({item["subsystem"] for item in result["impacted_subsystems"]}, {"deployment-installability"}) + requirement = result["files"][0]["matches"][0]["verification_requirement"] + self.assertEqual(requirement["id"], "demo-runtime-transaction") + self.assertEqual(sorted(requirement["exact_files"]), [ ".github/scripts/tests/test_demo_runtime_transaction.py", "scripts/installtests/demo_transaction_test.go", ]) From 4df3e8974ec0e0931dd5edc877d29449c24c935a Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:36:36 +0100 Subject: [PATCH 4/7] Add disposable native acceptance for owned demo recovery Exercise actual systemd and Caddy with a changed executable, distinct persistent data, delayed synthetic failure, observer loss and TERM during full rollback. Keep 300-second clocks, exact-source and cleanup receipts; no production credential or worker system privilege is added. Update the connected recovery contract and policy assertions, and retain bounded service identity observations. Change-source: pulse-maintainer --- .github/scripts/demo-runtime-transaction.py | 11 +- .github/scripts/dispatch-demo-runtime.py | 4 +- .github/scripts/tests/demo_runtime_native.py | 218 ++++++++++++++++++ .../tests/test_demo_runtime_transaction.py | 50 ++++ .github/workflows/demo-runtime-native.yml | 43 ++++ .../v6/internal/DEMO_RUNTIME_TRANSACTION.md | 15 ++ .../subsystems/deployment-installability.md | 13 ++ .../v6/internal/subsystems/registry.json | 6 +- scripts/installtests/demo_transaction_test.go | 34 +++ .../release_promotion_policy_test.py | 99 +++----- .../release_control/subsystem_lookup_test.py | 3 +- 11 files changed, 425 insertions(+), 71 deletions(-) create mode 100644 .github/scripts/tests/demo_runtime_native.py create mode 100644 .github/workflows/demo-runtime-native.yml diff --git a/.github/scripts/demo-runtime-transaction.py b/.github/scripts/demo-runtime-transaction.py index 5fa81dda04..436e74b46d 100644 --- a/.github/scripts/demo-runtime-transaction.py +++ b/.github/scripts/demo-runtime-transaction.py @@ -224,7 +224,7 @@ class Host: def check_journal(self, cursors): for service, cursor in cursors.items(): - text = self.command(["journalctl", "-u", service, "--after-cursor", cursor, "--output=json", "--no-pager"]) + text = self.command(["journalctl", "-u", service, "--after-cursor", cursor, "--output=json", "--quiet", "--no-pager"]) for line in text.splitlines(): try: entry = json.loads(line) @@ -335,13 +335,16 @@ class Transaction: baseline = self.host.state() started = self.host.now() evidence = self.receipt[phase] - evidence.update({"required_seconds": WINDOW, "samples": 0, "elapsed_seconds": 0}) + evidence.update({"required_seconds": WINDOW, "samples": 0, "elapsed_seconds": 0, + "initial_services": baseline}) while True: self.host.check_journal(cursors) state = self.host.state() + evidence["last_services"] = state if state != baseline or not self.healthy(state): raise Failure("sustained-health") - if self.host.version(self.request["local_url"]) != expected_version: + evidence["observed_version"] = self.host.version(self.request["local_url"]) + if evidence["observed_version"] != expected_version: raise Failure("runtime-version") evidence["samples"] += 1 elapsed = self.host.now() - started @@ -411,6 +414,8 @@ class Transaction: elif self.request["mode"] == "update": raise Failure("update-baseline-unhealthy") self.receipt["healthy_baseline"] = healthy_baseline + self.receipt["baseline_services"] = states + self.receipt["baseline_version"] = original_version old_handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)} for sig in old_handlers: signal.signal(sig, cancel) diff --git a/.github/scripts/dispatch-demo-runtime.py b/.github/scripts/dispatch-demo-runtime.py index 499602fd17..a56e018ff0 100644 --- a/.github/scripts/dispatch-demo-runtime.py +++ b/.github/scripts/dispatch-demo-runtime.py @@ -38,7 +38,8 @@ if not attempt.exists(): os.chmod(attempt / name, 0o600) stream.write(data); stream.flush(); os.fsync(stream.fileno()) # Intent is durable before starting the unit. An uncertain launch is never replayed. - dfd = os.open(attempt, os.O_DIRECTORY); os.fsync(dfd); os.close(dfd) + for directory in (attempt, root / 'attempts', root, root.parent): + dfd = os.open(directory, os.O_DIRECTORY); os.fsync(dfd); os.close(dfd) try: child = subprocess.run(['systemd-run', '--quiet', '--collect', '--unit=pulse-demo-' + identity[:32], @@ -53,6 +54,7 @@ if not attempt.exists(): with open(attempt / 'launch.json', 'x') as stream: os.chmod(attempt / 'launch.json', 0o600) json.dump(launch, stream); stream.flush(); os.fsync(stream.fileno()) + dfd = os.open(attempt, os.O_DIRECTORY); os.fsync(dfd); os.close(dfd) os.close(fd) # Closing or losing this observer never stops the independently owned unit. deadline = time.monotonic() + 4000 diff --git a/.github/scripts/tests/demo_runtime_native.py b/.github/scripts/tests/demo_runtime_native.py new file mode 100644 index 0000000000..b004b1cf05 --- /dev/null +++ b/.github/scripts/tests/demo_runtime_native.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Secret-free, fresh public CI only. Never run this fixture on a service host.""" +import base64 +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import pwd +import shutil +import socket +import subprocess +import time + +SCRIPTS = Path(__file__).resolve().parents[1] +RESULT = Path.cwd() / "demo-native-result.json" +ROOT = Path("/var/lib/pulse-deploy/demo") +OWNED = [Path("/etc/pulse"), Path("/opt/pulse"), Path("/var/lib/pulse-deploy"), + Path("/etc/systemd/system/pulse.service"), Path("/etc/systemd/system/pulse-relay.service"), + Path("/usr/local/share/ca-certificates/pulse-demo-native.crt")] + + +def command(argv, timeout=120): + result = subprocess.run(argv, capture_output=True, timeout=timeout) + if result.returncode: + raise RuntimeError("fixture-command-failed") + return result.stdout + + +def binary(version, fail_after=0, port=17655): + return ('''#!/usr/bin/python3 +import http.server,json,os,threading,time +VERSION = %r +def fail(): + time.sleep(%r) + print('panic: synthetic disposable service fault',flush=True) + os._exit(1) +if %r: threading.Thread(target=fail,daemon=True).start() +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + body=json.dumps({'version':VERSION} if self.path=='/api/version' else {'status':'healthy'}).encode() + self.send_response(200); self.end_headers(); self.wfile.write(body) + def log_message(self,*args): pass +http.server.HTTPServer(('127.0.0.1',%r),Handler).serve_forever() +''' % (version, fail_after, fail_after, port)).encode() + + +def unit(executable, user="pulse"): + return f"[Unit]\nDescription=Disposable demo native fixture\n[Service]\nUser={user}\nGroup={user}\nExecStart={executable}\nRestart=no\n[Install]\nWantedBy=multi-user.target\n" + + +def installer(version, fail_after=0): + source = binary(version, fail_after) + return ("#!/usr/bin/python3\nfrom pathlib import Path\n" + "import base64\n" + f"p=Path('/opt/pulse/bin/pulse');p.write_bytes(base64.b64decode({base64.b64encode(source).decode()!r}));p.chmod(0o755)\n" + "p=Path('/etc/systemd/system/pulse.service');p.write_text(p.read_text()+'# candidate fixture\\n')\n" + f"Path('/etc/pulse/persistent-marker').write_text({'candidate data ' + version!r})\n").encode() + + +def fixture_installer(version, fail_after=0): + # The real engine invokes bash like the published installer. This fixture + # deliberately changes executable, unit and persistent data without download. + script = installer(version, fail_after) + return ("#!/bin/bash\nset -eu\npython3 - <<'PY'\n" + script.decode().split("\n", 1)[1] + "\nPY\n").encode() + + +def wait(predicate, limit): + deadline = time.monotonic() + limit + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(1) + raise RuntimeError("fixture-observation-timeout") + + +def main(): + result = {"schema_version": 1, "control_sha": os.environ.get("GITHUB_SHA"), + "run_id": os.environ.get("GITHUB_RUN_ID"), "run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"), + "cases": [], "cleanup_complete": False, "signed_published_installer_acceptance": False} + processes = [] + admitted = False + original_caddy = None + caddyfile = Path("/etc/caddy/Caddyfile") + try: + result["stage"] = "disposable-runner-admission" + # Environment fields are context, not permission: combine them with a + # fresh empty estate, effective root, disposable hosted VM and exact job. + if (os.geteuid() != 0 or os.environ.get("GITHUB_ACTIONS") != "true" + or os.environ.get("RUNNER_ENVIRONMENT") != "github-hosted" + or socket.gethostname() in {"pulse-relay", "pulse-license", "pulse-dev"} + or any(path.exists() or path.is_symlink() for path in OWNED)): + raise RuntimeError("not-an-empty-disposable-public-ci-runner") + admitted = True + result["stage"] = "prepare-native-fixtures" + original_caddy = caddyfile.read_bytes() + command(["systemctl", "show", "--property=Version"]) + if not subprocess.run(["id", "pulse"], capture_output=True).returncode == 0: + command(["useradd", "--system", "--user-group", "--no-create-home", "pulse"]) + owner = pwd.getpwnam("pulse") + Path("/opt/pulse/bin").mkdir(parents=True) + Path("/etc/pulse").mkdir(mode=0o755) + os.chown("/etc/pulse", owner.pw_uid, owner.pw_gid) + Path("/etc/pulse/.env").write_text("DEMO_MODE=true\n") + Path("/etc/pulse/billing.json").write_text('{"capabilities":[]}') + Path("/etc/pulse/persistent-marker").write_text("original data") + for path in Path("/etc/pulse").iterdir(): + os.chown(path, owner.pw_uid, owner.pw_gid) + Path("/opt/pulse/bin/pulse").write_bytes(binary("1.0.0")); Path("/opt/pulse/bin/pulse").chmod(0o755) + Path("/opt/pulse/bin/relay").write_bytes(binary("1.0.0", port=17656)); Path("/opt/pulse/bin/relay").chmod(0o755) + for name, executable in (("pulse", "/opt/pulse/bin/pulse"), ("pulse-relay", "/opt/pulse/bin/relay")): + Path(f"/etc/systemd/system/{name}.service").write_text(unit(executable)) + command(["systemctl", "daemon-reload"]) + command(["systemctl", "enable", "--now", "pulse", "pulse-relay"]) + cert = Path("/etc/caddy/demo-native.crt"); key = Path("/etc/caddy/demo-native.key") + command(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", + "-subj", "/CN=localhost", "-addext", "subjectAltName=IP:127.0.0.1", + "-addext", "basicConstraints=critical,CA:TRUE", + "-keyout", str(key), "-out", str(cert)]) + key.chmod(0o640); shutil.chown(key, group="caddy") + shutil.copyfile(cert, OWNED[-1]); command(["update-ca-certificates"]) + caddyfile.write_text("{\n admin off\n auto_https off\n}\n" + + f"https://127.0.0.1:18443 {{\n tls {cert} {key}\n reverse_proxy 127.0.0.1:17655\n}}\n" + + f"https://127.0.0.1:18444 {{\n tls {cert} {key}\n reverse_proxy 127.0.0.1:17656\n}}\n") + command(["systemctl", "restart", "caddy"]) + spec = importlib.util.spec_from_file_location("dispatcher", SCRIPTS / "dispatch-demo-runtime.py") + dispatcher = importlib.util.module_from_spec(spec); spec.loader.exec_module(dispatcher) + original = (SCRIPTS / "demo-runtime-transaction.py").read_bytes() + # Only the cohost endpoint is rebound. Transaction, paths, locking, + # systemd, HTTP/TLS, estate, clocks and watches remain real and unchanged. + endpoint = b'https://relay.pulserelay.pro/health' + assert original.count(endpoint) == 1 + source = original.replace(endpoint, b'https://127.0.0.1:18444/health') + result["original_engine_sha256"] = hashlib.sha256(original).hexdigest() + result["fixture_engine_sha256"] = hashlib.sha256(source).hexdigest() + result["bootstrap_sha256"] = hashlib.sha256(dispatcher.BOOTSTRAP.encode()).hexdigest() + result["driver_sha256"] = hashlib.sha256(Path(__file__).read_bytes()).hexdigest() + result["tools"] = {"systemd": command(["systemctl", "--version"]).decode().splitlines()[0], + "caddy": command(["caddy", "version"]).decode().strip()} + profile = {name: "2" for name in ("nodes", "vms_per_node", "lxcs_per_node", "docker_hosts", "docker_containers", "generic_hosts", "k8s_clusters", "k8s_nodes", "k8s_pods", "k8s_deployments")} + profile.update(seed_duration="2h", sample_interval="5m", update_interval="15s") + + def submit(version, fault=0): + install = fixture_installer(version, fault) + request = {"mode": "update", "hostname": socket.gethostname(), "local_url": "http://127.0.0.1:17655", + "public_url": "https://127.0.0.1:18443/api/health", "version": "v" + version, + "profile": profile, "control_sha": os.environ["GITHUB_SHA"], + "run_id": os.environ["GITHUB_RUN_ID"], "run_attempt": os.environ["GITHUB_RUN_ATTEMPT"], + "installer_sha256": hashlib.sha256(install).hexdigest()} + payload = {"request": request, "source": base64.b64encode(source).decode(), "installer": base64.b64encode(install).decode()} + identity = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()).hexdigest() + proc = subprocess.Popen(["python3", "-c", dispatcher.BOOTSTRAP], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + processes.append(proc); proc.stdin.write(json.dumps(payload).encode()); proc.stdin.close() + return proc, ROOT / "attempts" / identity, payload + + # Readiness is real, including authenticated TLS through actual Caddy. + result["stage"] = "native-listener-readiness" + wait(lambda: subprocess.run(["curl", "--disable", "--silent", "--fail", "https://127.0.0.1:18443/api/health"], capture_output=True).returncode == 0, 60) + proc, attempt, payload = submit("1.0.1") + result["stage"] = "changed-executable-forward" + proc.wait(timeout=900) + receipt = json.loads((attempt / "receipt.json").read_text()); result["cases"].append(receipt) + assert proc.returncode == 0 and receipt["status"] == "committed" and receipt["forward"]["elapsed_seconds"] >= 300 + assert Path("/etc/pulse/persistent-marker").read_text() == "candidate data 1.0.1" + # A later candidate fails at 55s; lose the observer while the owned + # systemd child continues, then TERM during actual full recovery. + proc, attempt, payload = submit("1.0.2", 55) + result["stage"] = "delayed-fault-observer-loss" + wait(lambda: (attempt / "receipt.json").exists() and json.loads((attempt / "receipt.json").read_text())["status"] == "applying", 60) + proc.terminate(); proc.wait(timeout=30) + command(["systemctl", "is-active", "pulse-demo-" + attempt.name[:32]]) + wait(lambda: json.loads((attempt / "receipt.json").read_text())["status"] == "recovering", 180) + result["stage"] = "term-during-owned-recovery" + command(["systemctl", "kill", "--kill-whom=main", "--signal=TERM", "pulse-demo-" + attempt.name[:32]]) + replay = subprocess.run(["python3", "-c", dispatcher.BOOTSTRAP], input=json.dumps(payload).encode(), capture_output=True, timeout=900) + receipt = json.loads((attempt / "receipt.json").read_text()); result["cases"].append(receipt) + assert replay.returncode == 1 and receipt["status"] == "rolled_back" and receipt["recovery"]["elapsed_seconds"] >= 300 + assert receipt["failure"] in {"new-service-crash", "sustained-health"} + assert Path("/etc/pulse/persistent-marker").read_text() == "candidate data 1.0.1" + version = json.loads(command(["curl", "--disable", "--silent", "--fail", "http://127.0.0.1:17655/api/version"]))["version"] + assert version == "1.0.1" + result["stage"] = "complete" + result["passed"] = True + except Exception as error: + result["passed"] = False + result["failure_type"] = type(error).__name__ + result["failure_code"] = str(error) if isinstance(error, RuntimeError) else "fixture-observation-or-assertion" + finally: + if admitted: + try: + for proc in processes: + if proc.poll() is None: + proc.terminate(); proc.wait(timeout=30) + # Stop only retained children belonging to this fresh fixture. + if (ROOT / "attempts").exists(): + for attempt in (ROOT / "attempts").iterdir(): + subprocess.run(["systemctl", "stop", "pulse-demo-" + attempt.name[:32]], capture_output=True, timeout=1300) + command(["systemctl", "disable", "--now", "pulse", "pulse-relay"]) + command(["systemctl", "stop", "caddy"]) + if original_caddy is not None: + caddyfile.write_bytes(original_caddy) + for path in OWNED: + if path.is_dir(): shutil.rmtree(path) + elif path.exists(): path.unlink() + for path in (Path("/etc/caddy/demo-native.crt"), Path("/etc/caddy/demo-native.key")): + if path.exists(): path.unlink() + command(["systemctl", "daemon-reload"]); command(["update-ca-certificates", "--fresh"]) + result["cleanup_complete"] = not any(path.exists() for path in OWNED) + except Exception as cleanup: + result["cleanup_failure_type"] = type(cleanup).__name__ + result["passed"] = False + RESULT.write_text(json.dumps(result, indent=2) + "\n") + RESULT.chmod(0o644) + return 0 if result.get("passed") and result["cleanup_complete"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/tests/test_demo_runtime_transaction.py b/.github/scripts/tests/test_demo_runtime_transaction.py index 6050a380c8..c2fab87420 100644 --- a/.github/scripts/tests/test_demo_runtime_transaction.py +++ b/.github/scripts/tests/test_demo_runtime_transaction.py @@ -22,6 +22,7 @@ def load(name, filename): engine = load("demo_transaction", "demo-runtime-transaction.py") dispatcher = load("demo_dispatcher", "dispatch-demo-runtime.py") +native = load("demo_native", "tests/demo_runtime_native.py") def request(mode="recover"): @@ -181,6 +182,13 @@ class TransactionTest(unittest.TestCase): self.assertEqual(receipt["status"], "committed") self.assertEqual(receipt["forward"]["elapsed_seconds"], 300) self.assertEqual(receipt["forward"]["samples"], 61) + self.assertEqual(receipt["baseline_services"]["pulse"]["MainPID"], "100") + self.assertEqual(receipt["forward"]["initial_services"], receipt["forward"]["last_services"]) + self.assertEqual(receipt["forward"]["observed_version"], "6.4.5") + self.assertEqual(self.host.install_count, 0) + for name in ("binary", "unit", "dropins"): + self.assertEqual(engine.estate_hash({name: self.paths[name]}), + engine.estate_hash({name: self.attempt / "snapshot" / name})) self.assertEqual((self.paths["data"] / "alerts/events.db").read_bytes(), b"synthetic opaque persistent file") self.assertTrue((self.attempt / "snapshot/data/persisted-history").exists()) self.assertNotIn("not-logged", json.dumps(receipt)) @@ -387,6 +395,13 @@ class InputAndCommandTest(unittest.TestCase): host.check_journal(cursors) self.assertEqual(cursors, {"pulse": "d"}) + def test_available_empty_journal_window_has_no_non_json_header(self): + host = engine.Host(); cursors = {"pulse": "c"} + with patch.object(host, "command", return_value="") as command: + host.check_journal(cursors) + self.assertIn("--quiet", command.call_args.args[0]) + self.assertEqual(cursors, {"pulse": "c"}) + def test_no_journal_cursor_and_malformed_service_state_fail_closed(self): host = engine.Host() with patch.object(host, "command", return_value=""): @@ -446,5 +461,40 @@ class DispatchLifecycleTest(unittest.TestCase): self.assertEqual(json.loads((attempt / "launch.json").read_text())["state"], "uncertain") +class NativeAdmissionTest(unittest.TestCase): + def test_native_driver_refuses_wrong_context_before_any_service_command(self): + with tempfile.TemporaryDirectory() as directory: + result_path = Path(directory) / "result.json" + existing = Path(directory) / "existing-estate" + existing.mkdir() + cases = [(1000, "true", "github-hosted", []), + (0, "false", "github-hosted", []), + (0, "true", "self-hosted", []), + (0, "true", "github-hosted", [existing])] + for uid, actions, environment, estate in cases: + with self.subTest(uid=uid, actions=actions, environment=environment, occupied=bool(estate)), \ + patch.object(native, "RESULT", result_path), patch.object(native, "OWNED", estate), \ + patch.object(os, "geteuid", return_value=uid), \ + patch.dict(os.environ, {"GITHUB_ACTIONS": actions, "RUNNER_ENVIRONMENT": environment}), \ + patch.object(native.socket, "gethostname", return_value="disposable-fixture"), \ + patch.object(native, "command", side_effect=AssertionError("must not execute a service command")) as command: + self.assertEqual(native.main(), 1) + command.assert_not_called() + result = json.loads(result_path.read_text()) + self.assertFalse(result["passed"]) + self.assertFalse(result["cleanup_complete"]) + self.assertFalse(result["signed_published_installer_acceptance"]) + self.assertEqual(result["failure_code"], "not-an-empty-disposable-public-ci-runner") + self.assertTrue(existing.is_dir()) + + def test_synthetic_changed_executable_and_data_are_distinct_between_steps(self): + compile(native.binary("1.0.1"), "fixture-runtime", "exec") + compile(native.installer("1.0.1"), "fixture-installer", "exec") + self.assertNotEqual(native.binary("1.0.1"), native.binary("1.0.2", 55)) + self.assertIn(b"candidate data 1.0.1", native.installer("1.0.1")) + self.assertIn(b"candidate data 1.0.2", native.installer("1.0.2", 55)) + self.assertNotIn(b"curl", native.fixture_installer("1.0.1")) + + if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/demo-runtime-native.yml b/.github/workflows/demo-runtime-native.yml new file mode 100644 index 0000000000..a8928e5e5b --- /dev/null +++ b/.github/workflows/demo-runtime-native.yml @@ -0,0 +1,43 @@ +name: Demo transaction native acceptance +on: + pull_request: + paths: + - '.github/scripts/demo-runtime-transaction.py' + - '.github/scripts/dispatch-demo-runtime.py' + - '.github/scripts/tests/demo_runtime_native.py' + - '.github/workflows/demo-runtime-native.yml' + push: + branches: [main] + paths: + - '.github/scripts/demo-runtime-transaction.py' + - '.github/scripts/dispatch-demo-runtime.py' + - '.github/scripts/tests/demo_runtime_native.py' + - '.github/workflows/demo-runtime-native.yml' +permissions: + contents: read +concurrency: + group: demo-native-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true +jobs: + native: + runs-on: ubuntu-24.04 + timeout-minutes: 35 + steps: + - name: Check out exact source without retained credentials + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Prepare Caddy only in the disposable public CI runner + run: | + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends caddy openssl + - name: Exercise real systemd Caddy and retained child recovery + run: sudo --preserve-env=GITHUB_ACTIONS,RUNNER_ENVIRONMENT,GITHUB_SHA,GITHUB_RUN_ID,GITHUB_RUN_ATTEMPT python3 .github/scripts/tests/demo_runtime_native.py + - name: Retain sanitised terminal receipts and cleanup + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: demo-native-${{ github.run_id }} + path: demo-native-result.json + if-no-files-found: error + retention-days: 14 diff --git a/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md b/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md index af5e4322da..106e939ffc 100644 --- a/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md +++ b/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md @@ -44,6 +44,21 @@ recovery, headroom refusal, shared-lock contention, source mismatch, journal failure, cohost restart, source-bound repeated/lost submission and no-op proof. The production window is not shortened. This is **not native acceptance**. +`demo-runtime-native.yml` now carries the next executable acceptance rather +than requiring a worker to obtain system mode: exact PR/main source, a fresh +secret-free public hosted runner, actual systemd/Caddy and disposable trusted +TLS. The successful changed-executable step observes the full 300 seconds. A +different executable and data then fail at 55 seconds; the original observer +is lost, the same retained request observes its child without replay, TERM is +sent during restoration, and the old executable/data must return through a +full real-clock recovery window. Sanitised artifacts bind control/driver/ +engine/bootstrap identity, service PID/restart observations and cleanup. The +fixture rebinds only Relay's fixed health URL to its local Caddy endpoint; the +original and fixture hashes remain distinct. This is a synthetic installer, +not signed published-installer or customer acceptance. Its source exists; +**no native pass is asserted by this document**. Read the exact-head terminal +job and artifact before judging that result. + Before treating this route as restored, the exact reviewed source must return native disposable systemd/SSH-loss receipts and signed published-installer forward/reverse data/identity results. Then reconcile actual demo workflow and diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 78533e9a3f..f53a0ffd5d 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -45,6 +45,19 @@ session. These connected source controls do not establish native systemd, signed installed forward/reverse acceptance, ingress enforcement or customer recovery. See [demo transaction acceptance](../DEMO_RUNTIME_TRANSACTION.md). +The secret-free public `demo-runtime-native.yml` checks the exact PR/push source +on a fresh hosted runner, using actual systemd and Caddy with disposable TLS and +synthetic executable/data fixtures. Its two real-window cases exercise a +changed executable, a 55-second delayed synthetic failure, observer loss and +TERM during full restoration. It records original/fixture source identities, +actual bounded service observations and cleanup; a missing or failed record is +not acceptance. Only the fixed Relay health URL is rebound to that fixture. +This neither exercises a signed published installer nor proves customer or +production recovery. The driver refuses a non-empty or non-disposable estate +before service operations. Neither this driver nor worker source tests install +host policy. Production activation still requires independent review and the +operational owner's acceptance of the native result and published-source path. + ### Superseded pull request validation Build and Test and Core E2E cancel an older run for the same pull request when diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index 9cf02a0271..f3ec7681fb 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -4524,11 +4524,13 @@ ".github/scripts/resolve-demo-runtime-profile.sh", ".github/scripts/select-internal-api-shard.sh", ".github/scripts/setup-demo-ssh.sh", + ".github/scripts/tests/demo_runtime_native.py", ".github/workflows/backfill-release-assets.yml", ".github/workflows/build-and-test.yml", ".github/workflows/build-release-candidate.yml", ".github/workflows/compile-release-payload.yml", ".github/workflows/create-release.yml", + ".github/workflows/demo-runtime-native.yml", ".github/workflows/dependency-advisory-watch.yml", ".github/workflows/deploy-demo-server.yml", ".github/workflows/helm-pages.yml", @@ -5176,7 +5178,9 @@ "match_prefixes": [], "match_files": [ ".github/scripts/demo-runtime-transaction.py", - ".github/scripts/dispatch-demo-runtime.py" + ".github/scripts/dispatch-demo-runtime.py", + ".github/scripts/tests/demo_runtime_native.py", + ".github/workflows/demo-runtime-native.yml" ], "allow_same_subsystem_tests": false, "test_prefixes": [], diff --git a/scripts/installtests/demo_transaction_test.go b/scripts/installtests/demo_transaction_test.go index ae800da558..d2d2be0683 100644 --- a/scripts/installtests/demo_transaction_test.go +++ b/scripts/installtests/demo_transaction_test.go @@ -57,3 +57,37 @@ func TestDemoTransactionKeepsSignedAdmissionAndVerificationOnlyReadOnly(t *testi t.Fatal("recovery must validate and retain the actual terminal observation") } } + +func TestDemoNativeAcceptanceIsSecretFreeDisposableAndKeepsRealWindows(t *testing.T) { + data, err := os.ReadFile(repoFile(".github", "workflows", "demo-runtime-native.yml")) + if err != nil { + t.Fatal(err) + } + text := string(data) + for _, required := range []string{ + "pull_request:", "branches: [main]", "contents: read", "runs-on: ubuntu-24.04", "timeout-minutes: 35", + "persist-credentials: false", "actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1", + "demo_runtime_native.py", "if: always()", "demo-native-result.json", "if-no-files-found: error", + } { + if !strings.Contains(text, required) { + t.Fatalf("lost disposable native CI/source or terminal-receipt boundary: %s", required) + } + } + for _, forbidden := range []string{"secrets.", "pull_request_target", "self-hosted", "contents: write", "id-token: write"} { + if strings.Contains(text, forbidden) { + t.Fatalf("native fixture gained credentials, privileged trigger or non-disposable target: %s", forbidden) + } + } + driver, err := os.ReadFile(repoFile(".github", "scripts", "tests", "demo_runtime_native.py")) + if err != nil { + t.Fatal(err) + } + for _, required := range []string{"not-an-empty-disposable-public-ci-runner", "dispatcher.BOOTSTRAP", "proc.terminate()", "--signal=TERM", "original_engine_sha256", "fixture_engine_sha256", "cleanup_complete", "signed_published_installer_acceptance", "elapsed_seconds\"] >= 300"} { + if !strings.Contains(string(driver), required) { + t.Fatalf("lost native fixture observation or limitation: %s", required) + } + } + if strings.Contains(string(driver), "WINDOW =") || strings.Contains(string(driver), "time.monotonic =") || strings.Contains(string(driver), "time.sleep =") { + t.Fatal("native acceptance must not shorten production windows or replace real clocks") + } +} diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py index 5f4278134e..a933737fd4 100644 --- a/scripts/release_control/release_promotion_policy_test.py +++ b/scripts/release_control/release_promotion_policy_test.py @@ -2749,73 +2749,42 @@ class ReleasePromotionPolicyTest(unittest.TestCase): def test_stable_demo_recovery_is_fixed_current_bits_only(self) -> None: workflow = read(".github/workflows/recover-demo-server.yml") recovery = read(".github/scripts/recover-demo-runtime.sh") + transaction = read(".github/scripts/demo-runtime-transaction.py") + dispatcher = read(".github/scripts/dispatch-demo-runtime.py") - self.assertIn("workflow_dispatch:", workflow) - self.assertNotIn("workflow_call:", workflow) - self.assertNotIn("inputs:", workflow) - self.assertIn("environment: demo-stable", workflow) - self.assertIn("contents: read", workflow) - self.assertIn("cancel-in-progress: false", workflow) - self.assertIn('gh api "repos/${GITHUB_REPOSITORY}/releases/latest"', workflow) - self.assertIn("Resolve target-compatible recovery profile", workflow) - self.assertIn("resolve-demo-runtime-profile.sh", workflow) - self.assertIn(".github/scripts/recover-demo-runtime.sh", workflow) - self.assertIn("Verify public health", workflow) - self.assertIn("Verify public frontend parity", workflow) - self.assertIn("./scripts/run_demo_public_browser_smoke.sh", workflow) - self.assertIn("Compensate failed mutated recovery", workflow) - self.assertIn("Capture bounded Pulse failure diagnostics", workflow) - self.assertIn("demo-runtime-diagnostics.txt", workflow) - self.assertIn("journalctl -u pulse --since '-30 minutes'", workflow) - self.assertIn("journalctl -u pulse --since '-10 minutes'", workflow) - self.assertIn("tail -400", workflow) - self.assertIn("sudo systemctl stop pulse", workflow) - self.assertIn("Retain bounded recovery evidence", workflow) - self.assertNotIn("release-convergence.yml", workflow) - self.assertNotIn("install.sh", workflow) - self.assertNotIn("scp ", workflow) + for required in ("workflow_dispatch:", "environment: demo-stable", "contents: read", + "cancel-in-progress: false", 'gh api "repos/${GITHUB_REPOSITORY}/releases/latest"', + "Resolve target-compatible recovery profile", "resolve-demo-runtime-profile.sh", + ".github/scripts/recover-demo-runtime.sh", "Verify public health", + "Verify public frontend parity", "./scripts/run_demo_public_browser_smoke.sh", + "Capture bounded Pulse failure diagnostics", "demo-runtime-diagnostics.txt", + "journalctl -u pulse --since '-30 minutes'", "journalctl -u pulse --since '-10 minutes'", + "tail -400", "Retain bounded recovery evidence", ".forward.elapsed_seconds >= 300"): + self.assertIn(required, workflow) + for forbidden in ("workflow_call:", "inputs:", "release-convergence.yml", "install.sh", "scp ", + "Compensate failed mutated recovery", "sudo systemctl stop pulse"): + self.assertNotIn(forbidden, workflow) + self.assertIn('exec python3 .github/scripts/dispatch-demo-runtime.py recover', recovery) + self.assertIn("if mode == 'update' else b''", dispatcher) + self.assertIn("/var/lib/pulse-deploy/relay/deploy.lock", transaction) + self.assertIn('if self.request["mode"] == "update":', transaction) + self.assertIn('self.host.install(self.attempt, self.request)', transaction) + self.assertIn('raise Failure("recovery-source-mismatch")', transaction) + self.assertIn('copy_path(path, snapshot / label)', transaction) + self.assertIn('copy_path(snapshot / label, PATHS[label])', transaction) + self.assertIn('estate_hash(PATHS) != self.receipt["snapshot_sha256"]', transaction) + self.assertIn('WINDOW = 300', transaction) + self.assertIn('self.watch("recovery", original_version, cursors)', transaction) + self.assertIn('signal.signal(sig, signal.SIG_IGN)', transaction) + self.assertIn('self.save("rolled_back")', transaction) + self.assertIn('self.save("rollback_failed")', transaction) + for required in ("alerts/events.db", "alerts/alert-history.json.imported", + "alerts/alert-history.backup.json.imported", "ai_incidents.json", "snapshot_retained"): + self.assertIn(required, transaction) + self.assertNotIn('kill -QUIT', transaction + recovery + workflow) + self.assertNotIn('eval ', transaction + recovery) + self.assertNotIn("'--wait'", dispatcher) - self.assertIn('if [ "$#" -ne 16 ]', recovery) - self.assertIn('SERVICE_NAME="pulse"', recovery) - self.assertIn('RELAY_SERVICE_NAME="pulse-relay"', recovery) - self.assertIn('EXPECTED_BINARY="/opt/pulse/bin/pulse"', recovery) - self.assertIn('EXPECTED_UNIT="/etc/systemd/system/pulse.service"', recovery) - self.assertIn('set_env_value PULSE_MOCK_NODES "$MOCK_NODES"', recovery) - self.assertIn('set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE"', recovery) - self.assertIn('set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE"', recovery) - self.assertIn('set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS"', recovery) - self.assertIn('set_env_value PULSE_MOCK_DOCKER_CONTAINERS "$MOCK_DOCKER_CONTAINERS"', recovery) - self.assertIn('set_env_value PULSE_MOCK_GENERIC_HOSTS "$MOCK_GENERIC_HOSTS"', recovery) - self.assertIn('set_env_value PULSE_MOCK_K8S_CLUSTERS "$MOCK_K8S_CLUSTERS"', recovery) - self.assertIn('set_env_value PULSE_MOCK_K8S_NODES "$MOCK_K8S_NODES"', recovery) - self.assertIn('set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS"', recovery) - self.assertIn('set_env_value PULSE_MOCK_K8S_DEPLOYMENTS "$MOCK_K8S_DEPLOYMENTS"', recovery) - self.assertIn("set_env_value PULSE_MOCK_SEED_METRICS_STORE false", recovery) - self.assertIn('grep -Fxq "PULSE_MOCK_SEED_METRICS_STORE=false"', recovery) - self.assertIn('set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"', recovery) - self.assertIn('set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"', recovery) - self.assertIn('set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"', recovery) - self.assertIn('sudo systemctl restart "$SERVICE_NAME"', recovery) - self.assertIn('sudo systemctl stop "$SERVICE_NAME"', recovery) - self.assertIn('sudo kill -QUIT "$failed_pid"', recovery) - self.assertIn("clear_demo_operational_history", recovery) - self.assertIn("restore_demo_operational_history", recovery) - self.assertIn("alerts/events.db", recovery) - self.assertIn("alerts/alert-history.json.imported", recovery) - self.assertIn("alerts/alert-history.backup.json.imported", recovery) - self.assertIn("ai_incidents.json", recovery) - self.assertIn("demo-operational-history.tar", recovery) - self.assertNotIn('rm -f "/etc/pulse/', recovery) - self.assertIn('AFTER_RELAY_PID" = "$BEFORE_RELAY_PID', recovery) - self.assertIn('AFTER_BINARY_SHA" = "$BEFORE_BINARY_SHA', recovery) - self.assertIn('AFTER_UNIT_SHA" = "$BEFORE_UNIT_SHA', recovery) - self.assertIn('AFTER_DROPINS_SHA" = "$BEFORE_DROPINS_SHA', recovery) - self.assertNotIn('AFTER_CONFIG_SHA" = "$BEFORE_CONFIG_SHA', recovery) - self.assertIn("/etc/pulse/.env /etc/pulse/billing.json", recovery) - self.assertIn("restore_runtime_config", recovery) - self.assertIn("sudo cp --archive", recovery) - self.assertNotIn("eval ", recovery) - self.assertNotIn("sudo bash", recovery) def test_blocked_record_tracks_current_target_and_candidate_version(self) -> None: blocked_record_surface = { diff --git a/scripts/release_control/subsystem_lookup_test.py b/scripts/release_control/subsystem_lookup_test.py index 1fae39c773..8521b32545 100644 --- a/scripts/release_control/subsystem_lookup_test.py +++ b/scripts/release_control/subsystem_lookup_test.py @@ -102,7 +102,8 @@ def _contract_reference(contract_path: str, needle: str, runtime_path: str) -> d class SubsystemLookupTest(unittest.TestCase): def test_demo_transaction_sources_have_connected_proof_without_widening_release_metadata(self): - for path in (".github/scripts/demo-runtime-transaction.py", ".github/scripts/dispatch-demo-runtime.py"): + for path in (".github/scripts/demo-runtime-transaction.py", ".github/scripts/dispatch-demo-runtime.py", + ".github/scripts/tests/demo_runtime_native.py", ".github/workflows/demo-runtime-native.yml"): with self.subTest(path=path): result = lookup_paths([path]) self.assertEqual(result["unowned_runtime_files"], []) From 360721cc910b91b83bf39aa46cd130bf64266c1a Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:44:42 +0100 Subject: [PATCH 5/7] Bind demo cohost observation to the real Relay health route Current operational source exposes healthz and takes the shared deployment lock. Pin that route and make the disposable service return 404 for unknown paths so native acceptance cannot manufacture a healthy cohost observation. Change-source: pulse-maintainer --- .github/scripts/demo-runtime-transaction.py | 2 +- .github/scripts/tests/demo_runtime_native.py | 9 ++++++--- .github/scripts/tests/test_demo_runtime_transaction.py | 3 +++ .../v6/internal/subsystems/deployment-installability.md | 3 +++ 4 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/scripts/demo-runtime-transaction.py b/.github/scripts/demo-runtime-transaction.py index 436e74b46d..6f12dfebf2 100644 --- a/.github/scripts/demo-runtime-transaction.py +++ b/.github/scripts/demo-runtime-transaction.py @@ -27,7 +27,7 @@ PATHS = {"binary": Path("/opt/pulse/bin/pulse"), "unit": Path("/etc/systemd/system/pulse.service"), "dropins": Path("/etc/systemd/system/pulse.service.d"), "data": Path("/etc/pulse")} -RELAY_HEALTH = "https://relay.pulserelay.pro/health" +RELAY_HEALTH = "https://relay.pulserelay.pro/healthz" COUNT_KEYS = ("nodes", "vms_per_node", "lxcs_per_node", "docker_hosts", "docker_containers", "generic_hosts", "k8s_clusters", "k8s_nodes", "k8s_pods", "k8s_deployments") diff --git a/.github/scripts/tests/demo_runtime_native.py b/.github/scripts/tests/demo_runtime_native.py index b004b1cf05..0b7fc6a73b 100644 --- a/.github/scripts/tests/demo_runtime_native.py +++ b/.github/scripts/tests/demo_runtime_native.py @@ -38,11 +38,14 @@ def fail(): if %r: threading.Thread(target=fail,daemon=True).start() class Handler(http.server.BaseHTTPRequestHandler): def do_GET(self): + expected = {'/healthz'} if %r == 17656 else {'/api/health','/api/version'} + if self.path not in expected: + self.send_response(404); self.end_headers(); return body=json.dumps({'version':VERSION} if self.path=='/api/version' else {'status':'healthy'}).encode() self.send_response(200); self.end_headers(); self.wfile.write(body) def log_message(self,*args): pass http.server.HTTPServer(('127.0.0.1',%r),Handler).serve_forever() -''' % (version, fail_after, fail_after, port)).encode() +''' % (version, fail_after, fail_after, port, port)).encode() def unit(executable, user="pulse"): @@ -128,9 +131,9 @@ def main(): original = (SCRIPTS / "demo-runtime-transaction.py").read_bytes() # Only the cohost endpoint is rebound. Transaction, paths, locking, # systemd, HTTP/TLS, estate, clocks and watches remain real and unchanged. - endpoint = b'https://relay.pulserelay.pro/health' + endpoint = b'https://relay.pulserelay.pro/healthz' assert original.count(endpoint) == 1 - source = original.replace(endpoint, b'https://127.0.0.1:18444/health') + source = original.replace(endpoint, b'https://127.0.0.1:18444/healthz') result["original_engine_sha256"] = hashlib.sha256(original).hexdigest() result["fixture_engine_sha256"] = hashlib.sha256(source).hexdigest() result["bootstrap_sha256"] = hashlib.sha256(dispatcher.BOOTSTRAP.encode()).hexdigest() diff --git a/.github/scripts/tests/test_demo_runtime_transaction.py b/.github/scripts/tests/test_demo_runtime_transaction.py index c2fab87420..86bc90e60c 100644 --- a/.github/scripts/tests/test_demo_runtime_transaction.py +++ b/.github/scripts/tests/test_demo_runtime_transaction.py @@ -347,6 +347,9 @@ class TransactionTest(unittest.TestCase): class InputAndCommandTest(unittest.TestCase): + def test_cohost_check_uses_the_existing_relay_health_route(self): + self.assertEqual(engine.RELAY_HEALTH, "https://relay.pulserelay.pro/healthz") + def test_valid_current_envelope(self): engine.validate(request()) engine.validate(request("update")) diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index f53a0ffd5d..c24547706f 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -24,6 +24,9 @@ The mutating paths now share the Relay host's existing Every successful path, including a healthy no-op, needs a complete 300-second local/public/Relay health, version, service PID/restart and new-journal-crash window. Readiness is separate. Missing observations fail closed. +The fixed Relay observation is its existing `/healthz` endpoint, not a guessed +new route; disposable acceptance also rejects an unknown route rather than +responding healthy to every URL. Before installation or runtime-profile mutation, stop only Pulse and durably capture its executable, unit/drop-ins and entire persistent data estate. Refuse From 7ec246381fdf0a58c8b695a0982e34e04cb6545f Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:45:34 +0100 Subject: [PATCH 6/7] Check retained demo profile inside its owning transaction The broader release-policy check still expected the removed shell configuration step. Assert its real transaction entry and runtime profile/entitlement preservation while keeping activation, exact stable tag, network and public browser checks unchanged. Contract-Neutral: Align verification with the already implemented demo transaction; no runtime or authority change. Change-source: pulse-maintainer --- .../release_promotion_policy_test.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/scripts/release_control/release_promotion_policy_test.py b/scripts/release_control/release_promotion_policy_test.py index a933737fd4..e27d53e1b9 100644 --- a/scripts/release_control/release_promotion_policy_test.py +++ b/scripts/release_control/release_promotion_policy_test.py @@ -2589,7 +2589,9 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("DEMO_EXPECTED_HOSTNAME", demo) self.assertIn("Verify target host identity", demo) self.assertIn("Demo environment points at host $REMOTE_HOSTNAME but expected $DEMO_EXPECTED_HOSTNAME.", demo) - self.assertIn("Restore demo runtime configuration", demo) + self.assertIn("Apply guarded demo transaction", demo) + self.assertIn("dispatch-demo-runtime.py", demo) + self.assertIn("Retain guarded demo transaction evidence", demo) self.assertIn("Resolve target-compatible demo runtime profile", demo) self.assertIn("mockEagerHistoryPVEGuestLimit", demo_profile) self.assertIn("UpdateMetricCohort", demo_profile) @@ -2608,19 +2610,17 @@ class ReleasePromotionPolicyTest(unittest.TestCase): self.assertIn("MOCK_SEED_DURATION=2h", demo_profile) self.assertIn("MOCK_SAMPLE_INTERVAL=5m", demo_profile) self.assertIn("MOCK_UPDATE_INTERVAL=15s", demo_profile) - self.assertIn("resolve_config_dir", demo) - self.assertIn("set_env_value DEMO_MODE true", demo) - self.assertIn("set_env_value PULSE_MOCK_MODE true", demo) - self.assertIn('set_env_value PULSE_MOCK_NODES "$MOCK_NODES"', demo) - self.assertIn("set_env_value PULSE_MOCK_SEED_METRICS_STORE false", demo) - self.assertIn('set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"', demo) - self.assertIn('set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"', demo) - self.assertIn('set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"', demo) - self.assertIn("ensure_demo_fixture_entitlement", demo) - self.assertIn('"demo_fixtures"', demo) - self.assertIn("del(.integrity)", demo) - self.assertIn("Demo fixture entitlement ensured in governed demo billing state.", demo) - self.assertIn("Demo service restarted with governed demo runtime configuration.", demo) + # Profile application moved into the owned transaction; keep its + # runtime settings and entitlement, not the superseded shell snippets. + transaction = read(".github/scripts/demo-runtime-transaction.py") + for required in ('"DEMO_MODE": "true"', '"PULSE_MOCK_MODE": "true"', + '"PULSE_MOCK_SEED_METRICS_STORE": "false"', + '"seed_duration": "trends_seed_duration"', '"sample_interval": "trends_sample_interval"', + '"PULSE_MOCK_" + key.upper()', '["demo_fixtures"]', 'billing.pop("integrity", None)', + 'self.host.profile(self.request, not healthy_baseline)', 'self.host.start()'): + self.assertIn(required, transaction) + self.assertIn("MOCK_NODES: ${{ needs.resolve.outputs.mock_nodes }}", demo) + self.assertIn("MOCK_UPDATE_INTERVAL: ${{ needs.resolve.outputs.mock_update_interval }}", demo) self.assertIn("/api/license/runtime-capabilities", demo) self.assertIn("Mock mode enabled", demo) self.assertIn("Demo server mock mode did not enable after entitlement sync", demo) From ee451f8c8c23dc64319dfee281b22d8d679f4d3e Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 02:23:58 +0100 Subject: [PATCH 7/7] Restore canonical ordering for demo registry entries Candidate preflight rejected three path lists that used ASCII or insertion order instead of the registry's case-insensitive order. Sort only those lists and exercise the committed registry in the audit suite so fixture-only passes cannot miss the same mistake. Contract-Neutral: Canonical registry path-list ordering and regression coverage only; ownership, verification requirements and runtime behaviour are unchanged. Change-source: pulse-maintainer --- .../v6/internal/subsystems/registry.json | 26 +++++++++---------- .../release_control/registry_audit_test.py | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index f3ec7681fb..b58adc709b 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -4548,9 +4548,6 @@ ".github/workflows/retry-release-convergence.yml", ".github/workflows/update-demo-server.yml", ".github/workflows/validate-release-assets.yml", - "Dockerfile", - "Makefile", - "VERSION", "cmd/pulse-control-plane/main.go", "cmd/pulse-control-plane/provider_msp.go", "cmd/pulse-control-plane/provider_msp_backup.go", @@ -4560,12 +4557,13 @@ "cmd/pulse-control-plane/provider_msp_recover.go", "cmd/pulse-control-plane/provider_msp_status.go", "docker-compose.yml", + "Dockerfile", "docs/MSP.md", - "docs/RELEASE_NOTES.md", - "docs/UPGRADE_v6.md", "docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md", "docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md", "docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md", + "docs/RELEASE_NOTES.md", + "docs/UPGRADE_v6.md", "frontend-modern/package-lock.json", "frontend-modern/package.json", "frontend-modern/public/docs/MSP.md", @@ -4585,6 +4583,7 @@ "internal/cloudcp/provider_msp_backup.go", "internal/cloudcp/provider_msp_recovery.go", "internal/cloudcp/tenant_runtime_rollout.go", + "Makefile", "package-lock.json", "package.json", "pulse-enterprise:.github/workflows/build-pro-release.yml", @@ -4659,16 +4658,17 @@ "scripts/uninstall-sensor-proxy.sh", "scripts/validate-release.sh", "scripts/verify-github-release-integrity.sh", + "tests/integration/playwright.config.ts", "tests/integration/QUICK_START.md", "tests/integration/README.md", - "tests/integration/playwright.config.ts", "tests/integration/scripts/bootstrap-hosted-mobile-onboarding.mjs", "tests/integration/scripts/hosted-mobile-token-runtime.mjs", "tests/integration/scripts/hosted-tenant-runtime.mjs", "tests/integration/scripts/managed-dev-runtime.mjs", "tests/integration/scripts/relay-mobile-token-helper.go", "tests/integration/tests/helpers.ts", - "tests/integration/tests/runtime-defaults.ts" + "tests/integration/tests/runtime-defaults.ts", + "VERSION" ], "release_cycle_artifact_globs": [ "docs/RELEASE_NOTES.md", @@ -4775,12 +4775,11 @@ ".github/workflows/retry-release-convergence.yml", ".github/workflows/update-demo-server.yml", ".github/workflows/validate-release-assets.yml", - "VERSION", - "docs/RELEASE_NOTES.md", - "docs/UPGRADE_v6.md", "docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md", "docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md", "docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md", + "docs/RELEASE_NOTES.md", + "docs/UPGRADE_v6.md", "frontend-modern/public/docs/UPGRADE_v6.md", "frontend-modern/src/components/UpdateBanner.tsx", "frontend-modern/src/components/WhatsNewCard.tsx", @@ -4808,7 +4807,8 @@ "scripts/trigger-release-dry-run.sh", "scripts/trigger-release.sh", "scripts/trigger-stable-patch.sh", - "scripts/verify-github-release-integrity.sh" + "scripts/verify-github-release-integrity.sh", + "VERSION" ], "allow_same_subsystem_tests": false, "test_prefixes": [], @@ -5185,8 +5185,8 @@ "allow_same_subsystem_tests": false, "test_prefixes": [], "exact_files": [ - "scripts/installtests/demo_transaction_test.go", - ".github/scripts/tests/test_demo_runtime_transaction.py" + ".github/scripts/tests/test_demo_runtime_transaction.py", + "scripts/installtests/demo_transaction_test.go" ] } ], diff --git a/scripts/release_control/registry_audit_test.py b/scripts/release_control/registry_audit_test.py index 1118e6c6fb..803a075b08 100644 --- a/scripts/release_control/registry_audit_test.py +++ b/scripts/release_control/registry_audit_test.py @@ -47,6 +47,32 @@ class RegistryAuditTest(unittest.TestCase): args = parse_args(["--check", "--repo-scope", "pulse"]) self.assertEqual(args.repo_scope, ["pulse"]) + def test_live_registry_path_lists_are_casefold_sorted(self) -> None: + # Fixture-only audit tests cannot catch ordering mistakes in the + # committed registry. Check its real path lists using the same + # case-insensitive contract as candidate preflight, not ASCII order. + registry = registry_audit.load_registry_payload() + for subsystem in registry["subsystems"]: + lists = { + field: subsystem[field] + for field in ("owned_prefixes", "owned_files") + } + verification = subsystem["verification"] + lists.update({ + f"verification.{field}": verification[field] + for field in ("test_prefixes", "exact_files") + }) + for policy in verification["path_policies"]: + lists.update({ + f"{policy['id']}.{field}": policy[field] + for field in ( + "match_prefixes", "match_files", "test_prefixes", "exact_files" + ) + }) + for field, paths in lists.items(): + with self.subTest(subsystem=subsystem["id"], field=field): + self.assertEqual(paths, sorted(paths, key=str.casefold)) + def test_trust_gate_regression_proofs_are_registered_to_exact_policies(self) -> None: registry = registry_audit.load_registry_payload() rules = {rule["id"]: rule for rule in registry["subsystems"]}