From f56e28e6afd2c4586fbaf3db761a488c9615ca6b Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Sat, 3 Oct 2026 01:05:16 +0100 Subject: [PATCH] Own demo update and recovery through sustained host observation Compose the existing signed stable demo paths with the Relay host lock, durable systemd ownership, full persistent estate capture and five-minute forward and recovery watches. Preserve uncertain attempts and failed rollback, refuse destructive headroom pruning, and keep native installed acceptance separate from connected source controls. Change-source: pulse-maintainer --- .github/scripts/demo-runtime-transaction.py | 504 ++++++++++++++++++ .github/scripts/dispatch-demo-runtime.py | 113 ++++ .github/scripts/recover-demo-runtime.sh | 395 +------------- .../tests/test_demo_runtime_transaction.py | 443 +++++++++++++++ .github/workflows/recover-demo-server.yml | 34 +- .github/workflows/update-demo-server.yml | 326 +---------- .../v6/internal/DEMO_RUNTIME_TRANSACTION.md | 60 +++ .../subsystems/deployment-installability.md | 32 ++ .../v6/internal/subsystems/registry.json | 28 +- .../installtests/build_release_assets_test.go | 43 +- scripts/installtests/demo_transaction_test.go | 56 ++ 11 files changed, 1278 insertions(+), 756 deletions(-) create mode 100644 .github/scripts/demo-runtime-transaction.py create mode 100644 .github/scripts/dispatch-demo-runtime.py create mode 100644 .github/scripts/tests/test_demo_runtime_transaction.py create mode 100644 docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md create mode 100644 scripts/installtests/demo_transaction_test.go diff --git a/.github/scripts/demo-runtime-transaction.py b/.github/scripts/demo-runtime-transaction.py new file mode 100644 index 0000000000..60193e378e --- /dev/null +++ b/.github/scripts/demo-runtime-transaction.py @@ -0,0 +1,504 @@ +#!/usr/bin/env python3 +"""Demo-only host transaction. Called by the reviewed CI dispatcher, not a worker tool.""" +from __future__ import annotations + +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import signal +import socket +import stat +import subprocess +import sys +import time +from urllib.parse import urlsplit + +WINDOW = 300 +INTERVAL = 5 +READY_TIMEOUT = 60 +SERVICES = ("pulse", "pulse-relay", "caddy") +ROOT = Path("/var/lib/pulse-deploy/demo") +LOCK = Path("/var/lib/pulse-deploy/relay/deploy.lock") +PATHS = {"binary": Path("/opt/pulse/bin/pulse"), + "unit": Path("/etc/systemd/system/pulse.service"), + "dropins": Path("/etc/systemd/system/pulse.service.d"), + "data": Path("/etc/pulse")} +RELAY_HEALTH = "https://relay.pulserelay.pro/health" +COUNT_KEYS = ("nodes", "vms_per_node", "lxcs_per_node", "docker_hosts", + "docker_containers", "generic_hosts", "k8s_clusters", "k8s_nodes", + "k8s_pods", "k8s_deployments") +DURATION_KEYS = ("seed_duration", "sample_interval", "update_interval") +# Only generated demo operational memory is reset on the already-unhealthy +# recovery route. Its complete original estate remains in the private snapshot. +DEMO_HISTORY = ("ai_incidents.json", "ai_incidents.json.tmp", + "alerts/alert-history.json", "alerts/alert-history.backup.json", + "alerts/alert-history.json.imported", "alerts/alert-history.backup.json.imported", + "alerts/events.db", "alerts/events.db-shm", "alerts/events.db-wal") +CRASH = re.compile(r"\bpanic\b|\bfatal error\b|segmentation fault|core dumped|watchdog timeout", re.I) +TERMINAL = {"committed", "healthy_noop", "rolled_back", "refused"} + + +class Failure(Exception): + """Only fixed stage names, never command output or private response bodies.""" + + +def atomic_json(path, value): + tmp = path.with_suffix(".tmp") + with open(tmp, "w", encoding="utf-8") as stream: + os.chmod(tmp, 0o600) + json.dump(value, stream, sort_keys=True) + stream.flush() + os.fsync(stream.fileno()) + os.replace(tmp, path) + fd = os.open(path.parent, os.O_DIRECTORY) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def validate(request): + if set(request) != {"mode", "hostname", "local_url", "public_url", "version", "profile", "control_sha", "run_id", "run_attempt", "installer_sha256"}: + raise Failure("request-shape") + if request["mode"] not in {"update", "recover"}: + raise Failure("request-mode") + for field, pattern in (("version", r"v[0-9]+\.[0-9]+\.[0-9]+"), + ("hostname", r"[a-zA-Z0-9][a-zA-Z0-9.-]{0,252}"), + ("control_sha", r"[0-9a-f]{40}"), + ("run_id", r"[1-9][0-9]*"), ("run_attempt", r"[1-9][0-9]*")): + if not isinstance(request[field], str) or not re.fullmatch(pattern, request[field]): + raise Failure("request-identity") + for field, public in (("local_url", False), ("public_url", True)): + try: + url = urlsplit(request[field]) + valid = (url.scheme == ("https" if public else "http") + and url.hostname and not url.username and not url.password + and not url.query and not url.fragment + and (url.path == "/api/health" if public else url.path in {"", "/"}) + and (public or url.hostname in {"localhost", "127.0.0.1", "::1"}) + and 0 < (url.port or (443 if public else 80)) < 65536) + except (TypeError, ValueError): + valid = False + if not valid: + raise Failure("request-url") + profile = request["profile"] + if not isinstance(profile, dict) or set(profile) != set(COUNT_KEYS + DURATION_KEYS): + raise Failure("request-profile") + for name, value in profile.items(): + pattern = r"[1-9][0-9]{0,3}" if name in COUNT_KEYS else r"[1-9][0-9]{0,3}[smhd]" + if not isinstance(value, str) or not re.fullmatch(pattern, value): + raise Failure("request-profile") + digest = request["installer_sha256"] + if (request["mode"] == "update" and not re.fullmatch(r"[0-9a-f]{64}", digest)) or (request["mode"] == "recover" and digest != ""): + raise Failure("request-installer") + + +def copy_path(src, dst): + """Copy without following symlinks, retaining ownership as well as modes.""" + info = src.lstat() + if stat.S_ISLNK(info.st_mode): + dst.symlink_to(os.readlink(src)) + elif stat.S_ISDIR(info.st_mode): + dst.mkdir(mode=0o700) + for child in src.iterdir(): + copy_path(child, dst / child.name) + shutil.copystat(src, dst, follow_symlinks=False) + elif stat.S_ISREG(info.st_mode): + shutil.copy2(src, dst, follow_symlinks=False) + else: + raise Failure("snapshot-special-file") + os.chown(dst, info.st_uid, info.st_gid, follow_symlinks=False) + if not stat.S_ISLNK(info.st_mode): + fd = os.open(dst, os.O_RDONLY | (os.O_DIRECTORY if stat.S_ISDIR(info.st_mode) else 0)) + try: + os.fsync(fd) + finally: + os.close(fd) + + +def remove_path(path): + if path.is_symlink() or path.is_file(): + path.unlink() + elif path.is_dir(): + shutil.rmtree(path) + + +def estate_hash(paths): + rows = [] + for label, root in sorted(paths.items()): + exists = root.exists() or root.is_symlink() + if not exists: + rows.append((label, "absent")) + continue + for path in [root] + (sorted(root.rglob("*")) if root.is_dir() and not root.is_symlink() else []): + info = path.lstat() + kind = stat.S_IFMT(info.st_mode) + content = os.readlink(path) if stat.S_ISLNK(info.st_mode) else "" + if stat.S_ISREG(info.st_mode): + digest = hashlib.sha256() + with open(path, "rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + content = digest.hexdigest() + rows.append((label, str(path.relative_to(root)), kind, stat.S_IMODE(info.st_mode), info.st_uid, info.st_gid, content)) + return hashlib.sha256(json.dumps(rows, sort_keys=True).encode()).hexdigest() + + +class Host: + now = staticmethod(time.monotonic) + sleep = staticmethod(time.sleep) + + def command(self, argv, *, timeout=20, env=None): + try: + result = subprocess.run(argv, capture_output=True, timeout=timeout, env=env, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise Failure("command-unavailable") from exc + if result.returncode or len(result.stdout) > 1024 * 1024: + raise Failure("command-failed") + return result.stdout.decode("utf-8", errors="strict") + + def identity(self, hostname): + if socket.gethostname() != hostname or os.geteuid() != 0: + raise Failure("host-identity") + unit = self.command(["systemctl", "show", "pulse", "--property=FragmentPath,User,Group,ExecStart,DropInPaths", "--no-pager"]) + fields = dict(line.split("=", 1) for line in unit.splitlines() if "=" in line) + if (fields.get("FragmentPath") != str(PATHS["unit"]) + or fields.get("User") != "pulse" or fields.get("Group") != "pulse" + or str(PATHS["binary"]) not in fields.get("ExecStart", "")): + raise Failure("service-identity") + for name in ("binary", "unit", "data"): + if PATHS[name].is_symlink() or not PATHS[name].exists(): + raise Failure("estate-identity") + if PATHS["dropins"].is_symlink(): + raise Failure("dropin-identity") + for path in fields.get("DropInPaths", "").split(): + if Path(path).parent != PATHS["dropins"] or Path(path).is_symlink(): + raise Failure("unsupported-dropin") + if self.command(["systemctl", "is-enabled", "pulse"]).strip() != "enabled": + raise Failure("service-disabled") + + def state(self): + states = {} + for service in SERVICES: + text = self.command(["systemctl", "show", service, "--property=ActiveState,SubState,MainPID,NRestarts", "--no-pager"]) + fields = dict(line.split("=", 1) for line in text.splitlines() if "=" in line) + if (set(fields) != {"ActiveState", "SubState", "MainPID", "NRestarts"} + or not fields["MainPID"].isdigit() or not fields["NRestarts"].isdigit()): + raise Failure("service-observation") + states[service] = fields + return states + + def status(self, url): + # No redirects, response bodies, credential arguments or ambient curl config. + try: + result = self.command(["curl", "--disable", "--silent", "--connect-timeout", "3", "--max-time", "8", "--output", "/dev/null", "--write-out", "%{http_code}", "--", url], timeout=10) + except Failure: + return "000" + return result if re.fullmatch(r"[1-5][0-9]{2}", result) else "000" + + def version(self, url): + try: + body = self.command(["curl", "--disable", "--silent", "--fail", "--connect-timeout", "3", "--max-time", "8", "--", url.rstrip("/") + "/api/version"], timeout=10) + if len(body) > 8192: + raise Failure("version-observation") + value = json.loads(body)["version"] + if not isinstance(value, str) or not re.fullmatch(r"v?[0-9]+\.[0-9]+\.[0-9]+", value): + raise Failure("version-observation") + return value.lstrip("v") + except (ValueError, KeyError, TypeError) as exc: + raise Failure("version-observation") from exc + + def cursors(self): + cursors = {} + for service in SERVICES: + text = self.command(["journalctl", "-u", service, "-n", "1", "--output=json", "--show-cursor", "--no-pager"]) + matches = re.findall(r"^-- cursor: ([^\s]+)$", text, re.M) + if len(matches) != 1: + raise Failure("journal-cursor") + cursors[service] = matches[0] + return cursors + + def check_journal(self, cursors): + for service, cursor in cursors.items(): + text = self.command(["journalctl", "-u", service, "--after-cursor", cursor, "--output=json", "--no-pager"]) + for line in text.splitlines(): + try: + entry = json.loads(line) + if not isinstance(entry.get("MESSAGE"), str) or not isinstance(entry.get("__CURSOR"), str): + raise Failure("journal-observation") + except (ValueError, TypeError, AttributeError) as exc: + raise Failure("journal-observation") from exc + if CRASH.search(entry["MESSAGE"]): + raise Failure("new-service-crash") + cursors[service] = entry["__CURSOR"] + + def stop(self): + self.command(["systemctl", "stop", "pulse"], timeout=90) + + def start(self): + self.command(["systemctl", "daemon-reload"]) + self.command(["systemctl", "start", "pulse"], timeout=90) + + def install(self, attempt, request): + installer = attempt / "installer.sh" + if hashlib.sha256(installer.read_bytes()).hexdigest() != request["installer_sha256"]: + raise Failure("installer-identity") + self.command(["bash", str(installer), "--version", request["version"]], timeout=900, + env={"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "HOME": "/root", "PULSE_SERVICE_NAME": "pulse"}) + + def profile_values(self, request): + values = {"DEMO_MODE": "true", "PULSE_MOCK_MODE": "true", "PULSE_MOCK_RANDOM_METRICS": "true", + "PULSE_MOCK_STOPPED_PERCENT": "6", "PULSE_MOCK_SEED_METRICS_STORE": "false"} + for key, value in request["profile"].items(): + key = {"seed_duration": "trends_seed_duration", "sample_interval": "trends_sample_interval"}.get(key, key) + values["PULSE_MOCK_" + key.upper()] = value + return values + + def profile_matches(self, request): + text = (PATHS["data"] / ".env").read_text().splitlines() + values = self.profile_values(request) + return (all(text.count(key + "=" + value) == 1 for key, value in values.items()) + and "demo_fixtures" in json.loads((PATHS["data"] / "billing.json").read_text()).get("capabilities", [])) + + def profile(self, request, unhealthy): + env_file = PATHS["data"] / ".env" + billing_file = PATHS["data"] / "billing.json" + if env_file.is_symlink() or billing_file.is_symlink(): + raise Failure("configuration-symlink") + values = self.profile_values(request) + text = env_file.read_text() + for key, value in values.items(): + pattern = re.compile(r"^[ \t]*" + re.escape(key) + r"=.*$", re.M) + text = pattern.sub(key + "=" + value, text) if pattern.search(text) else text.rstrip("\n") + "\n" + key + "=" + value + "\n" + env_file.write_text(text) + os.chmod(env_file, 0o600) + billing = json.loads(billing_file.read_text()) + if not isinstance(billing, dict) or not isinstance(billing.get("capabilities", []), list): + raise Failure("billing-shape") + billing["capabilities"] = sorted(set(billing.get("capabilities", []) + ["demo_fixtures"])) + billing.pop("integrity", None) + billing_file.write_text(json.dumps(billing)) + os.chmod(billing_file, 0o600) + if unhealthy: + for relative in DEMO_HISTORY: + path = PATHS["data"] / relative + # Never follow a writable parent symlink to a path outside the estate. + if any(p.is_symlink() for p in path.parents if p != PATHS["data"].parent): + raise Failure("history-symlink") + if path.is_symlink(): + raise Failure("history-symlink") + if path.is_file(): + path.unlink() + + +def active(state): + return state["ActiveState"] == "active" and state["SubState"] == "running" and int(state["MainPID"]) > 0 + + +class Transaction: + def __init__(self, host, request, attempt): + self.host, self.request, self.attempt = host, request, attempt + self.receipt = {"schema_version": 2, "status": "intent", "mutated": False, + "control_sha": request["control_sha"], "run_id": request["run_id"], + "run_attempt": request["run_attempt"], "mode": request["mode"], + "expected_version": request["version"], "installer_sha256": request["installer_sha256"], + "forward": {}, "recovery": {}, "recovery_required": False} + + def save(self, status): + self.receipt["status"] = status + atomic_json(self.attempt / "receipt.json", self.receipt) + + def urls(self): + return (self.request["local_url"].rstrip("/") + "/api/health", self.request["public_url"], RELAY_HEALTH) + + def healthy(self, states=None): + return all(active(s) for s in (states or self.host.state()).values()) and all(self.host.status(url) == "200" for url in self.urls()) + + def ready(self, cursors): + deadline = self.host.now() + READY_TIMEOUT + while True: + self.host.check_journal(cursors) + if self.healthy(): + return + if self.host.now() >= deadline: + raise Failure("listener-readiness") + self.host.sleep(INTERVAL) + + def watch(self, phase, expected_version, cursors): + baseline = self.host.state() + started = self.host.now() + evidence = self.receipt[phase] + evidence.update({"required_seconds": WINDOW, "samples": 0, "elapsed_seconds": 0}) + while True: + self.host.check_journal(cursors) + state = self.host.state() + if state != baseline or not self.healthy(state): + raise Failure("sustained-health") + if self.host.version(self.request["local_url"]) != expected_version: + raise Failure("runtime-version") + evidence["samples"] += 1 + elapsed = self.host.now() - started + evidence["elapsed_seconds"] = elapsed + if elapsed >= WINDOW: + return + self.host.sleep(min(INTERVAL, WINDOW - elapsed)) + + def capture(self): + snapshot = self.attempt / "snapshot" + size = sum(path.lstat().st_size for root in PATHS.values() if root.exists() + for path in ([root] + (list(root.rglob("*")) if root.is_dir() else [])) + if path.is_file() and not path.is_symlink()) + # Both our independent estate and the signed installer's own backup need + # space. Refuse shortage; never delete backups or databases for headroom. + if any(shutil.disk_usage(parent).free < 2 * size + 64 * 1024 * 1024 + for parent in (self.attempt, PATHS["data"].parent)): + raise Failure("snapshot-headroom") + snapshot.mkdir(mode=0o700) + manifest = {} + for label, path in PATHS.items(): + manifest[label] = path.exists() or path.is_symlink() + if manifest[label]: + copy_path(path, snapshot / label) + atomic_json(snapshot / "paths.json", manifest) + digest = estate_hash(PATHS) + self.receipt["snapshot_sha256"] = digest + self.receipt["snapshot_retained"] = True + return digest + + def restore(self): + snapshot = self.attempt / "snapshot" + for label, exists in json.loads((snapshot / "paths.json").read_text()).items(): + remove_path(PATHS[label]) + if exists: + copy_path(snapshot / label, PATHS[label]) + if estate_hash(PATHS) != self.receipt["snapshot_sha256"]: + raise Failure("restored-estate-identity") + + def run(self): + old_handlers = {} + healthy_baseline = False + stopped = False + original_version = "" + states = {} + + def cancel(_signum, _frame): + raise Failure("cancelled-forward") + + try: + validate(self.request) + self.host.identity(self.request["hostname"]) + for entry in ROOT.glob("attempts/*"): + if entry == self.attempt: + continue + receipt = entry / "receipt.json" + if not receipt.exists() or json.loads(receipt.read_text()).get("status") not in TERMINAL: + raise Failure("prior-recovery-required") + states = self.host.state() + if not all(active(states[name]) for name in ("pulse-relay", "caddy")) or self.host.status(RELAY_HEALTH) != "200": + raise Failure("cohost-unhealthy") + healthy_baseline = self.healthy(states) + if healthy_baseline: + original_version = self.host.version(self.request["local_url"]) + if self.request["mode"] == "recover" and original_version != self.request["version"].lstrip("v"): + raise Failure("recovery-source-mismatch") + elif self.request["mode"] == "update": + raise Failure("update-baseline-unhealthy") + self.receipt["healthy_baseline"] = healthy_baseline + # The no-op is observed too; an immediate healthy sample is not success. + if self.request["mode"] == "recover" and healthy_baseline and self.host.profile_matches(self.request): + self.watch("forward", original_version, self.host.cursors()) + self.save("healthy_noop") + return 0 + old_handlers = {sig: signal.getsignal(sig) for sig in (signal.SIGTERM, signal.SIGINT, signal.SIGHUP)} + for sig in old_handlers: + signal.signal(sig, cancel) + self.save("capturing") + stopped = True + self.host.stop() + self.capture() + self.save("applying") + self.receipt["mutated"] = True + cursors = self.host.cursors() + if self.request["mode"] == "update": + self.host.install(self.attempt, self.request) + # The installer may start the unit; finish profile work quiescent. + self.host.stop() + self.host.profile(self.request, not healthy_baseline) + self.host.start() + self.ready(cursors) + self.watch("forward", self.request["version"].lstrip("v"), cursors) + # No unrelated Relay/Caddy restart is owned by this transaction. + final = self.host.state() + for service in ("pulse-relay", "caddy"): + if final[service] != states[service]: + raise Failure("cohost-identity-changed") + self.save("committed") + return 0 + except Exception as exc: + self.receipt["failure"] = str(exc) if isinstance(exc, Failure) else "transaction-observation" + # Owned restoration, observation and the terminal receipt survive cancellation. + for sig in old_handlers: + signal.signal(sig, signal.SIG_IGN) + if stopped: + try: + self.save("recovering") + self.host.stop() + if "snapshot_sha256" in self.receipt: + self.restore() + if not healthy_baseline: + self.receipt["recovery_required"] = True + self.receipt["rollback"] = "unavailable-unhealthy-baseline" + self.save("recovery_required") + else: + cursors = self.host.cursors() + self.host.start() + self.ready(cursors) + self.watch("recovery", original_version, cursors) + final = self.host.state() + if any(final[name] != states[name] for name in ("pulse-relay", "caddy")): + raise Failure("cohost-identity-changed") + self.receipt["rollback"] = "verified" + self.save("rolled_back") + except Exception as recovery: + self.receipt["rollback"] = "failed" + self.receipt["recovery_failure"] = str(recovery) if isinstance(recovery, Failure) else "recovery-observation" + self.receipt["recovery_required"] = True + self.save("rollback_failed") + else: + self.save("refused") + return 1 + finally: + for sig, handler in old_handlers.items(): + signal.signal(sig, handler) + + +def main(): + # Attempt paths are generated by the fixed CI bootstrap, not CLI overrides. + if len(sys.argv) != 2: + return 2 + attempt = Path(sys.argv[1]) + if attempt.parent != ROOT / "attempts" or not re.fullmatch(r"[0-9a-f]{64}", attempt.name) or attempt.is_symlink(): + return 2 + request = json.loads((attempt / "request.json").read_text()) + transaction = Transaction(Host(), request, attempt) + fd = None + try: + LOCK.parent.mkdir(mode=0o700, parents=True, exist_ok=True) + fd = os.open(LOCK, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + return transaction.run() + except (OSError, ValueError): + transaction.receipt["failure"] = "host-lock-unavailable" + transaction.save("refused") + return 1 + finally: + if fd is not None: + os.close(fd) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/dispatch-demo-runtime.py b/.github/scripts/dispatch-demo-runtime.py new file mode 100644 index 0000000000..499602fd17 --- /dev/null +++ b/.github/scripts/dispatch-demo-runtime.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""The two existing demo CI paths share one retained, independently owned child.""" +from __future__ import annotations + +import base64 +import hashlib +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys + +BOOTSTRAP = r''' +import base64, fcntl, hashlib, json, os, pathlib, subprocess, sys, time +root = pathlib.Path('/var/lib/pulse-deploy/demo') +payload = json.loads(sys.stdin.read(512 * 1024)) +assert os.geteuid() == 0 and set(payload) == {'request', 'source', 'installer'} +source = base64.b64decode(payload['source'], validate=True) +installer = base64.b64decode(payload['installer'], validate=True) +assert len(source) < 128 * 1024 and len(installer) < 256 * 1024 +assert (hashlib.sha256(installer).hexdigest() == payload['request']['installer_sha256']) if installer else (payload['request']['installer_sha256'] == '') +identity = hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(',', ':')).encode()).hexdigest() +root.mkdir(mode=0o700, parents=True, exist_ok=True) +assert not root.is_symlink() and root.stat().st_uid == 0 +os.chmod(root, 0o700) +(root / 'attempts').mkdir(mode=0o700, exist_ok=True) +assert not (root / 'attempts').is_symlink() +fd = os.open(root / 'submit.lock', os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600) +fcntl.flock(fd, fcntl.LOCK_EX) +attempt = root / 'attempts' / identity +if not attempt.exists(): + attempt.mkdir(mode=0o700) + for name, data in [('transaction.py', source), ('installer.sh', installer), + ('request.json', json.dumps(payload['request']).encode()), + ('intent.json', json.dumps({'schema_version': 1, 'identity': identity, 'state': 'intent'}).encode())]: + with open(attempt / name, 'xb') as stream: + os.chmod(attempt / name, 0o600) + stream.write(data); stream.flush(); os.fsync(stream.fileno()) + # Intent is durable before starting the unit. An uncertain launch is never replayed. + dfd = os.open(attempt, os.O_DIRECTORY); os.fsync(dfd); os.close(dfd) + try: + child = subprocess.run(['systemd-run', '--quiet', '--collect', + '--unit=pulse-demo-' + identity[:32], + '--property=RuntimeMaxSec=45min', + '--property=TimeoutStopSec=20min', '--property=KillMode=mixed', + '/usr/bin/python3', str(attempt / 'transaction.py'), str(attempt)], + capture_output=True, timeout=30) + launch = {'schema_version': 1, 'state': 'submitted' if child.returncode == 0 else 'uncertain', + 'unit': 'pulse-demo-' + identity[:32]} + except (OSError, subprocess.TimeoutExpired): + launch = {'schema_version': 1, 'state': 'uncertain'} + with open(attempt / 'launch.json', 'x') as stream: + os.chmod(attempt / 'launch.json', 0o600) + json.dump(launch, stream); stream.flush(); os.fsync(stream.fileno()) +os.close(fd) +# Closing or losing this observer never stops the independently owned unit. +deadline = time.monotonic() + 4000 +terminal = {'committed', 'healthy_noop', 'rolled_back', 'rollback_failed', 'recovery_required', 'refused'} +while time.monotonic() < deadline: + try: + receipt = json.loads((attempt / 'receipt.json').read_text()) + except (FileNotFoundError, ValueError): + receipt = None + if receipt and receipt.get('status') in terminal: + receipt['request_identity'] = identity + print(json.dumps(receipt)) + sys.exit(0 if receipt['status'] in {'committed', 'healthy_noop'} else 1) + launch = attempt / 'launch.json' + if launch.exists() and json.loads(launch.read_text()).get('state') == 'uncertain': + break + time.sleep(5) +print(json.dumps({'schema_version': 2, 'status': 'uncertain', 'request_identity': identity, + 'failure': 'terminal-receipt-unavailable', 'recovery_required': True})) +sys.exit(1) +''' + + +def payload(mode, environ=os.environ): + profile = {key: environ['MOCK_' + key.upper()] for key in + ('nodes', 'vms_per_node', 'lxcs_per_node', 'docker_hosts', 'docker_containers', + 'generic_hosts', 'k8s_clusters', 'k8s_nodes', 'k8s_pods', 'k8s_deployments', + 'seed_duration', 'sample_interval', 'update_interval')} + installer = Path('/tmp/pulse-install.sh').read_bytes() if mode == 'update' else b'' + request = {'mode': mode, 'hostname': environ['DEMO_EXPECTED_HOSTNAME'], + 'local_url': environ['DEMO_LOCAL_BASE_URL'], 'public_url': environ['DEMO_PUBLIC_HEALTH_URL'], + 'version': environ['EXPECTED_VERSION'], 'profile': profile, + 'control_sha': environ['GITHUB_SHA'], 'run_id': environ['GITHUB_RUN_ID'], + 'run_attempt': environ['GITHUB_RUN_ATTEMPT'], + 'installer_sha256': hashlib.sha256(installer).hexdigest() if installer else ''} + return {'request': request, 'source': base64.b64encode(Path(__file__).with_name('demo-runtime-transaction.py').read_bytes()).decode(), + 'installer': base64.b64encode(installer).decode()} + + +def main(): + if len(sys.argv) != 2 or sys.argv[1] not in {'update', 'recover'}: + return 2 + # Same project CI SSH identity and pinned host verification as before. This + # is not an installed worker dispatcher or a new project credential. + command = ['ssh', '-i', str(Path.home() / '.ssh/id_ed25519'), '-o', 'IdentitiesOnly=yes', + '-o', 'StrictHostKeyChecking=yes', '-o', 'UserKnownHostsFile=' + str(Path.home() / '.ssh/known_hosts'), + os.environ['DEMO_SERVER_USER'] + '@' + os.environ['DEMO_SERVER_HOST'], + 'sudo python3 -c ' + shlex.quote(BOOTSTRAP)] + try: + result = subprocess.run(command, input=json.dumps(payload(sys.argv[1])).encode(), timeout=4100, check=False) + except (OSError, subprocess.TimeoutExpired): + print(json.dumps({'schema_version': 2, 'status': 'uncertain', 'failure': 'observer-unavailable', 'recovery_required': True})) + return 1 + return result.returncode + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/scripts/recover-demo-runtime.sh b/.github/scripts/recover-demo-runtime.sh index 132c5cf48f..81612502a7 100755 --- a/.github/scripts/recover-demo-runtime.sh +++ b/.github/scripts/recover-demo-runtime.sh @@ -1,395 +1,4 @@ #!/usr/bin/env bash set -euo pipefail - -if [ "$#" -ne 16 ]; then - echo "usage: recover-demo-runtime.sh EXPECTED_HOSTNAME LOCAL_BASE_URL EXPECTED_VERSION MOCK_NODES MOCK_VMS_PER_NODE MOCK_LXCS_PER_NODE MOCK_DOCKER_HOSTS MOCK_DOCKER_CONTAINERS MOCK_GENERIC_HOSTS MOCK_K8S_CLUSTERS MOCK_K8S_NODES MOCK_K8S_PODS MOCK_K8S_DEPLOYMENTS MOCK_SEED_DURATION MOCK_SAMPLE_INTERVAL MOCK_UPDATE_INTERVAL" >&2 - exit 2 -fi - -EXPECTED_HOSTNAME="$1" -LOCAL_BASE_URL="${2%/}" -EXPECTED_VERSION="${3#v}" -MOCK_NODES="$4" -MOCK_VMS_PER_NODE="$5" -MOCK_LXCS_PER_NODE="$6" -MOCK_DOCKER_HOSTS="$7" -MOCK_DOCKER_CONTAINERS="$8" -MOCK_GENERIC_HOSTS="$9" -MOCK_K8S_CLUSTERS="${10}" -MOCK_K8S_NODES="${11}" -MOCK_K8S_PODS="${12}" -MOCK_K8S_DEPLOYMENTS="${13}" -MOCK_SEED_DURATION="${14}" -MOCK_SAMPLE_INTERVAL="${15}" -MOCK_UPDATE_INTERVAL="${16}" -SERVICE_NAME="pulse" -RELAY_SERVICE_NAME="pulse-relay" -EXPECTED_BINARY="/opt/pulse/bin/pulse" -EXPECTED_UNIT="/etc/systemd/system/pulse.service" -MUTATED=false -BACKUP_DIR="" -HISTORY_BACKUP_ARCHIVE="" -DEMO_HISTORY_PATHS=( - ai_incidents.json - ai_incidents.json.tmp - alerts/alert-history.json - alerts/alert-history.backup.json - alerts/alert-history.json.imported - alerts/alert-history.backup.json.imported - alerts/events.db - alerts/events.db-shm - alerts/events.db-wal -) - -for fixture_count in \ - "$MOCK_NODES" \ - "$MOCK_VMS_PER_NODE" \ - "$MOCK_LXCS_PER_NODE" \ - "$MOCK_DOCKER_HOSTS" \ - "$MOCK_DOCKER_CONTAINERS" \ - "$MOCK_GENERIC_HOSTS" \ - "$MOCK_K8S_CLUSTERS" \ - "$MOCK_K8S_NODES" \ - "$MOCK_K8S_PODS" \ - "$MOCK_K8S_DEPLOYMENTS"; do - [[ "$fixture_count" =~ ^[1-9][0-9]*$ ]] -done -[[ "$MOCK_SEED_DURATION" =~ ^[1-9][0-9]*[smhd]$ ]] -[[ "$MOCK_SAMPLE_INTERVAL" =~ ^[1-9][0-9]*[smh]$ ]] -[[ "$MOCK_UPDATE_INTERVAL" =~ ^[1-9][0-9]*[smh]$ ]] - -log() { - printf '[demo-recovery] %s\n' "$*" >&2 -} - -unit_hash() { - sudo sha256sum "$EXPECTED_UNIT" | awk '{print $1}' -} - -binary_hash() { - sudo sha256sum "$EXPECTED_BINARY" | awk '{print $1}' -} - -dropin_manifest_hash() { - local paths path - paths="$(systemctl show "$SERVICE_NAME" --property=DropInPaths --value)" - if [ -z "$paths" ]; then - printf '%s' 'none' | sha256sum | awk '{print $1}' - return - fi - for path in $paths; do - sudo test -f "$path" - printf '%s\t' "$path" - sudo sha256sum "$path" | awk '{print $1}' - done | sort | sha256sum | awk '{print $1}' -} - -runtime_config_hash() { - local path - for path in /etc/pulse/.env /etc/pulse/billing.json; do - sudo test -f "$path" - printf '%s\t' "$path" - sudo sha256sum "$path" | awk '{print $1}' - done | sha256sum | awk '{print $1}' -} - -set_env_value() { - local key="$1" - local value="$2" - local env_file="/etc/pulse/.env" - if sudo grep -Eq "^[[:space:]]*${key}=" "$env_file"; then - sudo sed -i "s|^[[:space:]]*${key}=.*|${key}=${value}|" "$env_file" - else - printf '\n%s=%s\n' "$key" "$value" | sudo tee -a "$env_file" >/dev/null - fi -} - -runtime_profile_matches() { - sudo grep -Fxq "PULSE_MOCK_NODES=${MOCK_NODES}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_VMS_PER_NODE=${MOCK_VMS_PER_NODE}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_LXCS_PER_NODE=${MOCK_LXCS_PER_NODE}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_DOCKER_HOSTS=${MOCK_DOCKER_HOSTS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_DOCKER_CONTAINERS=${MOCK_DOCKER_CONTAINERS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_GENERIC_HOSTS=${MOCK_GENERIC_HOSTS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_K8S_CLUSTERS=${MOCK_K8S_CLUSTERS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_K8S_NODES=${MOCK_K8S_NODES}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_K8S_PODS=${MOCK_K8S_PODS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_K8S_DEPLOYMENTS=${MOCK_K8S_DEPLOYMENTS}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_SEED_METRICS_STORE=false" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_TRENDS_SEED_DURATION=${MOCK_SEED_DURATION}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_TRENDS_SAMPLE_INTERVAL=${MOCK_SAMPLE_INTERVAL}" /etc/pulse/.env && - sudo grep -Fxq "PULSE_MOCK_UPDATE_INTERVAL=${MOCK_UPDATE_INTERVAL}" /etc/pulse/.env -} - -cleanup_config_backup() { - [ -n "$BACKUP_DIR" ] || return 0 - sudo unlink "$BACKUP_DIR/runtime.env" 2>/dev/null || true - sudo unlink "$BACKUP_DIR/billing.json" 2>/dev/null || true - if [ -n "$HISTORY_BACKUP_ARCHIVE" ]; then - sudo unlink "$HISTORY_BACKUP_ARCHIVE" 2>/dev/null || true - HISTORY_BACKUP_ARCHIVE="" - fi - rmdir "$BACKUP_DIR" 2>/dev/null || true - BACKUP_DIR="" -} - -clear_demo_operational_history() { - local rel_path - local -a existing=() - for rel_path in "${DEMO_HISTORY_PATHS[@]}"; do - if sudo test -f "/etc/pulse/${rel_path}"; then - existing+=("$rel_path") - fi - done - ((${#existing[@]} > 0)) || return 0 - - HISTORY_BACKUP_ARCHIVE="$BACKUP_DIR/demo-operational-history.tar" - sudo tar --create --preserve-permissions \ - --file "$HISTORY_BACKUP_ARCHIVE" --directory /etc/pulse -- "${existing[@]}" - for rel_path in "${existing[@]}"; do - sudo unlink "/etc/pulse/${rel_path}" - done -} - -restore_demo_operational_history() { - local rel_path - [ -n "$HISTORY_BACKUP_ARCHIVE" ] || return 0 - sudo test -f "$HISTORY_BACKUP_ARCHIVE" || return 0 - for rel_path in "${DEMO_HISTORY_PATHS[@]}"; do - sudo unlink "/etc/pulse/${rel_path}" 2>/dev/null || true - done - sudo tar --extract --preserve-permissions \ - --file "$HISTORY_BACKUP_ARCHIVE" --directory /etc/pulse -} - -restore_runtime_config() { - [ -n "$BACKUP_DIR" ] || return 0 - sudo cp --archive "$BACKUP_DIR/runtime.env" /etc/pulse/.env - sudo cp --archive "$BACKUP_DIR/billing.json" /etc/pulse/billing.json -} - -local_version() { - curl --connect-timeout 3 --max-time 8 -fsS \ - "${LOCAL_BASE_URL}/api/version" | jq -er '.version' -} - -local_healthy() { - curl --connect-timeout 3 --max-time 8 -fsS \ - "${LOCAL_BASE_URL}/api/health" >/dev/null && local_version >/dev/null -} - -emit_evidence() { - local status="$1" - local before_pid="$2" - local after_pid="$3" - local before_relay_pid="$4" - local after_relay_pid="$5" - local before_binary_sha="$6" - local after_binary_sha="$7" - local before_unit_sha="$8" - local after_unit_sha="$9" - local before_dropins_sha="${10}" - local after_dropins_sha="${11}" - local version="${12}" - local before_config_sha="${13}" - local after_config_sha="${14}" - jq -n \ - --arg status "$status" \ - --argjson mutated "$MUTATED" \ - --arg hostname "$(hostname)" \ - --arg service "$SERVICE_NAME" \ - --arg before_pid "$before_pid" \ - --arg after_pid "$after_pid" \ - --arg relay_service "$RELAY_SERVICE_NAME" \ - --arg before_relay_pid "$before_relay_pid" \ - --arg after_relay_pid "$after_relay_pid" \ - --arg binary "$EXPECTED_BINARY" \ - --arg before_binary_sha256 "$before_binary_sha" \ - --arg after_binary_sha256 "$after_binary_sha" \ - --arg unit "$EXPECTED_UNIT" \ - --arg before_unit_sha256 "$before_unit_sha" \ - --arg after_unit_sha256 "$after_unit_sha" \ - --arg before_dropins_sha256 "$before_dropins_sha" \ - --arg after_dropins_sha256 "$after_dropins_sha" \ - --arg before_runtime_config_sha256 "$before_config_sha" \ - --arg after_runtime_config_sha256 "$after_config_sha" \ - --arg version "$version" \ - --arg expected_version "$EXPECTED_VERSION" \ - --arg mock_nodes "$MOCK_NODES" \ - --arg mock_vms_per_node "$MOCK_VMS_PER_NODE" \ - --arg mock_lxcs_per_node "$MOCK_LXCS_PER_NODE" \ - --arg mock_docker_hosts "$MOCK_DOCKER_HOSTS" \ - --arg mock_docker_containers "$MOCK_DOCKER_CONTAINERS" \ - --arg mock_generic_hosts "$MOCK_GENERIC_HOSTS" \ - --arg mock_k8s_clusters "$MOCK_K8S_CLUSTERS" \ - --arg mock_k8s_nodes "$MOCK_K8S_NODES" \ - --arg mock_k8s_pods "$MOCK_K8S_PODS" \ - --arg mock_k8s_deployments "$MOCK_K8S_DEPLOYMENTS" \ - --arg mock_seed_duration "$MOCK_SEED_DURATION" \ - --arg mock_sample_interval "$MOCK_SAMPLE_INTERVAL" \ - --arg mock_update_interval "$MOCK_UPDATE_INTERVAL" \ - '{schema_version: 1, status: $status, mutated: $mutated, - hostname: $hostname, service: $service, - before_pid: $before_pid, after_pid: $after_pid, - relay_service: $relay_service, - before_relay_pid: $before_relay_pid, - after_relay_pid: $after_relay_pid, - binary: $binary, - before_binary_sha256: $before_binary_sha256, - after_binary_sha256: $after_binary_sha256, - unit: $unit, - before_unit_sha256: $before_unit_sha256, - after_unit_sha256: $after_unit_sha256, - before_dropins_sha256: $before_dropins_sha256, - after_dropins_sha256: $after_dropins_sha256, - before_runtime_config_sha256: $before_runtime_config_sha256, - after_runtime_config_sha256: $after_runtime_config_sha256, - version: $version, expected_version: $expected_version, - runtime_profile: {mock_nodes: $mock_nodes, - mock_vms_per_node: $mock_vms_per_node, - mock_lxcs_per_node: $mock_lxcs_per_node, - mock_docker_hosts: $mock_docker_hosts, - mock_docker_containers: $mock_docker_containers, - mock_generic_hosts: $mock_generic_hosts, - mock_k8s_clusters: $mock_k8s_clusters, - mock_k8s_nodes: $mock_k8s_nodes, - mock_k8s_pods: $mock_k8s_pods, - mock_k8s_deployments: $mock_k8s_deployments, - mock_seed_duration: $mock_seed_duration, - mock_sample_interval: $mock_sample_interval, - mock_update_interval: $mock_update_interval}}' -} - -[ "$(hostname)" = "$EXPECTED_HOSTNAME" ] || { - log "target hostname does not match the governed demo environment" - exit 1 -} -[ "$(systemctl show "$SERVICE_NAME" --property=FragmentPath --value)" = "$EXPECTED_UNIT" ] -[ "$(systemctl show "$SERVICE_NAME" --property=User --value)" = "pulse" ] -[ "$(systemctl show "$SERVICE_NAME" --property=Group --value)" = "pulse" ] -[ "$(systemctl is-enabled "$SERVICE_NAME")" = "enabled" ] -[ "$(systemctl is-active "$RELAY_SERVICE_NAME")" = "active" ] -systemctl show "$SERVICE_NAME" --property=ExecStart --value | grep -Fq "$EXPECTED_BINARY" -sudo test -x "$EXPECTED_BINARY" -sudo test -f "$EXPECTED_UNIT" - -BEFORE_PID="$(systemctl show "$SERVICE_NAME" --property=MainPID --value)" -BEFORE_RELAY_PID="$(systemctl show "$RELAY_SERVICE_NAME" --property=MainPID --value)" -BEFORE_BINARY_SHA="$(binary_hash)" -BEFORE_UNIT_SHA="$(unit_hash)" -BEFORE_DROPINS_SHA="$(dropin_manifest_hash)" -BEFORE_CONFIG_SHA="$(runtime_config_hash)" - -if local_healthy && runtime_profile_matches; then - VERSION="$(local_version)" - [ "${VERSION#v}" = "$EXPECTED_VERSION" ] - emit_evidence healthy_noop "$BEFORE_PID" "$BEFORE_PID" \ - "$BEFORE_RELAY_PID" "$BEFORE_RELAY_PID" \ - "$BEFORE_BINARY_SHA" "$BEFORE_BINARY_SHA" \ - "$BEFORE_UNIT_SHA" "$BEFORE_UNIT_SHA" \ - "$BEFORE_DROPINS_SHA" "$BEFORE_DROPINS_SHA" "$VERSION" \ - "$BEFORE_CONFIG_SHA" "$BEFORE_CONFIG_SHA" - exit 0 -fi - -rollback_on_error() { - local rc=$? - local failed_pid - trap - ERR - if [ "$MUTATED" = true ]; then - failed_pid="$(systemctl show "$SERVICE_NAME" --property=MainPID --value || true)" - if [[ "$failed_pid" =~ ^[1-9][0-9]*$ ]] && sudo kill -0 "$failed_pid" 2>/dev/null; then - # The HTTP listener never opened, so pprof is unavailable. SIGQUIT gives - # one bounded Go goroutine dump in the service journal before the already - # failed process is stopped; it does not widen recovery to any other unit. - log "capturing failed Pulse startup goroutines before compensation" - sudo kill -QUIT "$failed_pid" || true - sleep 2 - fi - log "recovery validation failed; compensating by stopping only pulse.service" - sudo systemctl stop "$SERVICE_NAME" || true - restore_runtime_config || true - restore_demo_operational_history || true - fi - emit_evidence compensated_unavailable "$BEFORE_PID" \ - "$(systemctl show "$SERVICE_NAME" --property=MainPID --value || true)" \ - "$BEFORE_RELAY_PID" \ - "$(systemctl show "$RELAY_SERVICE_NAME" --property=MainPID --value || true)" \ - "$BEFORE_BINARY_SHA" "$(binary_hash || true)" \ - "$BEFORE_UNIT_SHA" "$(unit_hash || true)" \ - "$BEFORE_DROPINS_SHA" "$(dropin_manifest_hash || true)" "" \ - "$BEFORE_CONFIG_SHA" "$(runtime_config_hash || true)" - cleanup_config_backup - exit "$rc" -} -BACKUP_DIR="$(mktemp -d)" -sudo cp --archive /etc/pulse/.env "$BACKUP_DIR/runtime.env" -sudo cp --archive /etc/pulse/billing.json "$BACKUP_DIR/billing.json" -trap rollback_on_error ERR - -log "applying the target-compatible demo runtime profile and restarting only pulse.service" -MUTATED=true -set_env_value PULSE_MOCK_NODES "$MOCK_NODES" -set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE" -set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE" -set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS" -set_env_value PULSE_MOCK_DOCKER_CONTAINERS "$MOCK_DOCKER_CONTAINERS" -set_env_value PULSE_MOCK_GENERIC_HOSTS "$MOCK_GENERIC_HOSTS" -set_env_value PULSE_MOCK_K8S_CLUSTERS "$MOCK_K8S_CLUSTERS" -set_env_value PULSE_MOCK_K8S_NODES "$MOCK_K8S_NODES" -set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS" -set_env_value PULSE_MOCK_K8S_DEPLOYMENTS "$MOCK_K8S_DEPLOYMENTS" -# Persistent mock-store backfill is reserved for the isolated local-dev data -# directory. The governed demo already has persistent runtime data, so a stale -# opt-in here can turn startup into an unbounded SQLite backfill. -set_env_value PULSE_MOCK_SEED_METRICS_STORE false -set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION" -set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL" -set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL" -sudo chown pulse:pulse /etc/pulse/.env -sudo chmod 600 /etc/pulse/.env -# Alert lifecycle and AI incident history on this host are generated demo -# operational memory. A failed 50-node bootstrap can leave thousands of -# lifecycle events whose v6.4 replay queues one serialized incident save per -# event before HTTP opens. Back up and clear only those fixed demo-history -# files, including the retired JSON sources that repopulate a cleared event -# log; compensation restores them if the bounded profile still fails. -clear_demo_operational_history -sudo systemctl restart "$SERVICE_NAME" - -for _ in $(seq 1 12); do - if [ "$(systemctl is-active "$SERVICE_NAME" || true)" = "active" ] && local_healthy; then - break - fi - sleep 5 -done - -[ "$(systemctl is-active "$SERVICE_NAME")" = "active" ] -[ "$(systemctl show "$SERVICE_NAME" --property=SubState --value)" = "running" ] -local_healthy -runtime_profile_matches - -AFTER_PID="$(systemctl show "$SERVICE_NAME" --property=MainPID --value)" -AFTER_RELAY_PID="$(systemctl show "$RELAY_SERVICE_NAME" --property=MainPID --value)" -AFTER_BINARY_SHA="$(binary_hash)" -AFTER_UNIT_SHA="$(unit_hash)" -AFTER_DROPINS_SHA="$(dropin_manifest_hash)" -AFTER_CONFIG_SHA="$(runtime_config_hash)" -VERSION="$(local_version)" - -[ "$AFTER_PID" -gt 0 ] -[ "$AFTER_PID" != "$BEFORE_PID" ] -[ "$AFTER_RELAY_PID" = "$BEFORE_RELAY_PID" ] -[ "$AFTER_BINARY_SHA" = "$BEFORE_BINARY_SHA" ] -[ "$AFTER_UNIT_SHA" = "$BEFORE_UNIT_SHA" ] -[ "$AFTER_DROPINS_SHA" = "$BEFORE_DROPINS_SHA" ] -[ "${VERSION#v}" = "$EXPECTED_VERSION" ] - -trap - ERR -cleanup_config_backup -emit_evidence recovered "$BEFORE_PID" "$AFTER_PID" \ - "$BEFORE_RELAY_PID" "$AFTER_RELAY_PID" \ - "$BEFORE_BINARY_SHA" "$AFTER_BINARY_SHA" \ - "$BEFORE_UNIT_SHA" "$AFTER_UNIT_SHA" \ - "$BEFORE_DROPINS_SHA" "$AFTER_DROPINS_SHA" "$VERSION" \ - "$BEFORE_CONFIG_SHA" "$AFTER_CONFIG_SHA" +# CI observer only; remote systemd owns mutation, restoration and both watches. +exec python3 .github/scripts/dispatch-demo-runtime.py recover diff --git a/.github/scripts/tests/test_demo_runtime_transaction.py b/.github/scripts/tests/test_demo_runtime_transaction.py new file mode 100644 index 0000000000..36bb810f44 --- /dev/null +++ b/.github/scripts/tests/test_demo_runtime_transaction.py @@ -0,0 +1,443 @@ +import copy +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import signal +import subprocess +import tempfile +import unittest +from unittest.mock import patch + + +SCRIPTS = Path(__file__).resolve().parents[1] + + +def load(name, filename): + spec = importlib.util.spec_from_file_location(name, SCRIPTS / filename) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +engine = load("demo_transaction", "demo-runtime-transaction.py") +dispatcher = load("demo_dispatcher", "dispatch-demo-runtime.py") + + +def request(mode="recover"): + return {"mode": mode, "hostname": "fixture", "local_url": "http://127.0.0.1:7655", + "public_url": "https://demo.example/api/health", "version": "v6.4.5", + "control_sha": "a" * 40, "run_id": "123", "run_attempt": "1", + "installer_sha256": hashlib.sha256(b"signed-installer-fixture").hexdigest() if mode == "update" else "", + "profile": {**{name: "2" for name in engine.COUNT_KEYS}, + "seed_duration": "2h", "sample_interval": "5m", "update_interval": "15s"}} + + +class FixtureHost(engine.Host): + """Real filesystem estate with a virtual *unchanged* 300-second clock. + + Commands/services/journal/HTTP are adapters, not native or installed proof. + """ + def __init__(self): + self.clock = 0 + self.pid = 100 + self.running = True + self.runtime_version = "6.4.5" + self.starts = 0 + self.stops = 0 + self.install_count = 0 + self.fail_at = None + self.failed = False + self.rollback_failure = False + self.unhealthy = False + self.journal_failure = False + self.ready_until = 0 + self.cancel_at = None + self.cancel_in_recovery = False + self.co_host_change = False + self.co_pid = 200 + self.transaction = None + + def now(self): + return self.clock + + def sleep(self, seconds): + self.clock += seconds + + def identity(self, hostname): + if hostname != "fixture": + raise engine.Failure("host-identity") + + def state(self): + state = {"ActiveState": "active", "SubState": "running", "MainPID": str(self.pid), "NRestarts": "0"} + result = {"pulse": state, "pulse-relay": {**state, "MainPID": "200"}, "caddy": {**state, "MainPID": str(self.co_pid)}} + if not self.running: + result["pulse"] = {**state, "ActiveState": "inactive", "SubState": "dead", "MainPID": "0"} + return result + + def status(self, url): + if url == engine.RELAY_HEALTH: + return "200" + if self.clock < self.ready_until or self.unhealthy: + return "503" + if self.failed and self.rollback_failure: + return "503" + return "200" if self.running else "000" + + def version(self, url): + return self.runtime_version + + def cursors(self): + return {name: str(self.clock) for name in engine.SERVICES} + + def check_journal(self, cursors): + phase = self.transaction.receipt["status"] if self.transaction else "" + if self.journal_failure: + raise engine.Failure("journal-observation") + if self.co_host_change and self.clock >= 55: + self.co_pid = 201 + if self.cancel_at is not None and self.clock >= self.cancel_at and not self.failed: + self.failed = True + handler = signal.getsignal(signal.SIGTERM) + handler(signal.SIGTERM, None) + if self.cancel_in_recovery and phase == "recovering" and self.clock >= 350: + # Exercise the actual installed handler decision; do not signal the + # source-proof executor or kill another process to prove it. + assert signal.getsignal(signal.SIGTERM) == signal.SIG_IGN + self.cancel_in_recovery = False + if self.fail_at is not None and self.clock >= self.fail_at and not self.failed: + self.failed = True + raise engine.Failure("new-service-crash") + + def stop(self): + self.stops += 1 + self.running = False + + def start(self): + self.starts += 1 + self.pid += 1 + self.running = True + if self.transaction.receipt["status"] == "recovering": + self.runtime_version = "6.4.5" + self.unhealthy = False + + def install(self, attempt, req): + if hashlib.sha256((attempt / "installer.sh").read_bytes()).hexdigest() != req["installer_sha256"]: + raise engine.Failure("installer-identity") + self.install_count += 1 + self.runtime_version = req["version"].lstrip("v") + engine.PATHS["binary"].write_bytes(b"new executable") + engine.PATHS["unit"].write_bytes(b"new unit") + (engine.PATHS["data"] / "persisted-history").write_bytes(b"forward history") + + def profile(self, req, unhealthy): + super().profile(req, unhealthy) + if self.unhealthy: + self.unhealthy = False + + +class TransactionTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + root = Path(self.tmp.name) + self.paths = {name: root / "estate" / name for name in engine.PATHS} + for name, path in self.paths.items(): + path.parent.mkdir(parents=True, exist_ok=True) + if name in {"dropins", "data"}: + path.mkdir() + else: + path.write_bytes(b"old " + name.encode()) + path.chmod(0o755 if name == "binary" else 0o644) + (self.paths["dropins"] / "limits.conf").write_bytes(b"old limits") + (self.paths["data"] / ".env").write_text("PRIVATE_TOKEN=not-logged\n") + (self.paths["data"] / "billing.json").write_text('{"capabilities":[],"integrity":"private"}') + (self.paths["data"] / "persisted-history").write_bytes(b"original history") + (self.paths["data"] / "alerts").mkdir() + (self.paths["data"] / "alerts/events.db").write_bytes(b"synthetic opaque persistent file") + self.root = root / "control" + self.attempt = self.root / "attempts" / ("a" * 64) + self.attempt.mkdir(parents=True) + (self.attempt / "installer.sh").write_bytes(b"signed-installer-fixture") + self.addCleanup(patch.stopall) + patch.object(engine, "ROOT", self.root).start() + patch.object(engine, "PATHS", self.paths).start() + self.before = engine.estate_hash(self.paths) + self.host = FixtureHost() + + def run_transaction(self, mode="recover", version=None): + req = request(mode) + if version: + req["version"] = version + tx = engine.Transaction(self.host, req, self.attempt) + self.host.transaction = tx + result = tx.run() + self.assertEqual(tx.receipt, json.loads((self.attempt / "receipt.json").read_text())) + return result, tx.receipt + + def test_healthy_profile_change_keeps_data_and_watches_complete_window(self): + code, receipt = self.run_transaction() + self.assertEqual(code, 0) + self.assertEqual(receipt["status"], "committed") + self.assertEqual(receipt["forward"]["elapsed_seconds"], 300) + self.assertEqual(receipt["forward"]["samples"], 61) + self.assertEqual((self.paths["data"] / "alerts/events.db").read_bytes(), b"synthetic opaque persistent file") + self.assertTrue((self.attempt / "snapshot/data/persisted-history").exists()) + self.assertNotIn("not-logged", json.dumps(receipt)) + + def test_healthy_noop_requires_profile_and_full_watch_without_restart(self): + self.host.profile(request(), False) + code, receipt = self.run_transaction() + self.assertEqual(code, 0) + self.assertEqual(receipt["status"], "healthy_noop") + self.assertEqual(receipt["forward"]["elapsed_seconds"], 300) + self.assertFalse(receipt["mutated"]) + self.assertEqual(self.host.stops, 0) + + def test_update_restores_complete_changed_executable_unit_and_data_after_late_failure(self): + self.host.fail_at = 55 + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "rolled_back") + self.assertEqual(receipt["failure"], "new-service-crash") + self.assertEqual(receipt["recovery"]["elapsed_seconds"], 300) + self.assertEqual(engine.estate_hash(self.paths), self.before) + self.assertEqual(self.host.runtime_version, "6.4.5") + + def test_failure_at_299_seconds_still_collects_entire_recovery_and_cancellation(self): + self.host.fail_at = 299 + self.host.cancel_in_recovery = True + previous = signal.getsignal(signal.SIGTERM) + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["recovery"]["elapsed_seconds"], 300) + self.assertGreaterEqual(self.host.clock, 600) + self.assertFalse(self.host.cancel_in_recovery) + self.assertEqual(signal.getsignal(signal.SIGTERM), previous) + self.assertEqual(engine.estate_hash(self.paths), self.before) + + def test_forward_cancel_owns_full_recovery(self): + self.host.cancel_at = 55 + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["failure"], "cancelled-forward") + self.assertEqual(receipt["status"], "rolled_back") + self.assertEqual(receipt["recovery"]["elapsed_seconds"], 300) + + def test_failed_rollback_retains_estate_and_blocks_next_transaction(self): + self.host.fail_at = 55 + self.host.rollback_failure = True + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "rollback_failed") + self.assertTrue(receipt["snapshot_retained"]) + next_attempt = self.root / "attempts" / ("b" * 64) + next_attempt.mkdir() + self.attempt = next_attempt + self.host.rollback_failure = False + code, receipt = self.run_transaction() + self.assertEqual(receipt["failure"], "prior-recovery-required") + self.assertFalse(receipt["mutated"]) + + def test_unhealthy_recovery_retains_original_generated_history(self): + self.host.unhealthy = True + code, receipt = self.run_transaction() + self.assertEqual(code, 0) + self.assertFalse(receipt["healthy_baseline"]) + self.assertFalse((self.paths["data"] / "alerts/events.db").exists()) + self.assertEqual((self.attempt / "snapshot/data/alerts/events.db").read_bytes(), b"synthetic opaque persistent file") + self.assertEqual(receipt["forward"]["elapsed_seconds"], 300) + + def test_failed_unhealthy_recovery_is_not_a_healthy_rollback(self): + self.host.unhealthy = True + self.host.fail_at = 55 + code, receipt = self.run_transaction() + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "recovery_required") + self.assertEqual(receipt["rollback"], "unavailable-unhealthy-baseline") + self.assertEqual(engine.estate_hash(self.paths), self.before) + self.assertFalse(self.host.running) + + def test_update_refuses_unhealthy_baseline_without_stopping_or_pruning(self): + self.host.unhealthy = True + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["failure"], "update-baseline-unhealthy") + self.assertEqual(self.host.stops, 0) + self.assertEqual(engine.estate_hash(self.paths), self.before) + + def test_recovery_refuses_different_healthy_source(self): + code, receipt = self.run_transaction(version="v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["failure"], "recovery-source-mismatch") + self.assertEqual(self.host.stops, 0) + + def test_listener_readiness_does_not_spend_the_watch_window(self): + self.host.ready_until = 30 + code, receipt = self.run_transaction() + self.assertEqual(code, 0) + self.assertEqual(receipt["forward"]["elapsed_seconds"], 300) + self.assertGreaterEqual(self.host.clock, 330) + self.assertEqual(self.host.starts, 1) + + def test_missing_journal_fails_closed_and_retains_failure(self): + self.host.journal_failure = True + code, receipt = self.run_transaction() + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "rollback_failed") + self.assertEqual(receipt["recovery_failure"], "journal-observation") + + def test_cohost_restart_is_not_silently_accepted_as_recovery(self): + self.host.co_host_change = True + code, receipt = self.run_transaction("update", "v6.4.6") + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "rollback_failed") + self.assertEqual(receipt["recovery_failure"], "cohost-identity-changed") + + def test_snapshot_headroom_failure_never_deletes_persistent_files(self): + with patch.object(engine.shutil, "disk_usage", return_value=type("Usage", (), {"free": 0})()): + code, receipt = self.run_transaction() + self.assertEqual(code, 1) + self.assertEqual(receipt["status"], "rolled_back") + self.assertFalse(receipt["mutated"]) + self.assertEqual(receipt["failure"], "snapshot-headroom") + self.assertEqual(engine.estate_hash(self.paths), self.before) + + def test_linked_history_cannot_remove_files_outside_estate(self): + (self.paths["data"] / "alerts/events.db").unlink() + outside = self.root / "private-file" + outside.write_text("keep") + (self.paths["data"] / "alerts/events.db").symlink_to(outside) + self.host.unhealthy = True + code, receipt = self.run_transaction() + self.assertEqual(code, 1) + self.assertEqual(receipt["failure"], "history-symlink") + self.assertEqual(outside.read_text(), "keep") + + def test_shared_host_lock_refusal_is_terminal_and_has_no_mutation(self): + lock = self.root / "host.lock" + fd = os.open(lock, os.O_CREAT | os.O_RDWR, 0o600) + self.addCleanup(os.close, fd) + engine.fcntl.flock(fd, engine.fcntl.LOCK_EX | engine.fcntl.LOCK_NB) + (self.attempt / "request.json").write_text(json.dumps(request())) + with patch.object(engine, "LOCK", lock), patch.object(engine.sys, "argv", ["transaction", str(self.attempt)]): + self.assertEqual(engine.main(), 1) + receipt = json.loads((self.attempt / "receipt.json").read_text()) + self.assertEqual(receipt["status"], "refused") + self.assertFalse(receipt["mutated"]) + self.assertEqual(engine.estate_hash(self.paths), self.before) + + +class InputAndCommandTest(unittest.TestCase): + def test_valid_current_envelope(self): + engine.validate(request()) + engine.validate(request("update")) + + def test_no_candidate_prerelease_arbitrary_service_url_or_profile(self): + cases = [("mode", "shell"), ("version", "v6.4.6-rc.1"), ("control_sha", "main"), + ("local_url", "http://other-host:7655"), ("public_url", "http://demo.example/api/health"), + ("public_url", "https://user:secret@demo.example/api/health"), + ("public_url", "https://demo.example/api/health?token=secret"), + ("installer_sha256", "arbitrary")] + for name, value in cases: + with self.subTest(name=name, value=value): + req = request(); req[name] = value + with self.assertRaises(engine.Failure): + engine.validate(req) + req = request(); req["service"] = "license" + with self.assertRaises(engine.Failure): + engine.validate(req) + req = request(); req["profile"]["nodes"] = "2; restart caddy" + with self.assertRaises(engine.Failure): + engine.validate(req) + + def test_http_observation_rejects_redirect_and_transport_failure(self): + host = engine.Host() + with patch.object(host, "command", return_value="302") as command: + self.assertEqual(host.status("https://demo.example/api/health"), "302") + self.assertNotIn("--location", command.call_args.args[0]) + self.assertIn("--disable", command.call_args.args[0]) + self.assertIn("/dev/null", command.call_args.args[0]) + with patch.object(host, "command", side_effect=engine.Failure("command-failed")): + self.assertEqual(host.status("https://demo.example/api/health"), "000") + + def test_malformed_or_unavailable_journal_is_not_no_crash(self): + host = engine.Host() + for text in ["not-json", '{"MESSAGE":"fine"}', '{"MESSAGE":[],"__CURSOR":"c"}']: + with self.subTest(text=text), patch.object(host, "command", return_value=text): + with self.assertRaises(engine.Failure): + host.check_journal({"pulse": "c"}) + with patch.object(host, "command", return_value='{"MESSAGE":"panic: fixture","__CURSOR":"d"}'): + with self.assertRaisesRegex(engine.Failure, "new-service-crash"): + host.check_journal({"pulse": "c"}) + + def test_journal_cursor_advances_without_returning_private_message(self): + host = engine.Host(); cursors = {"pulse": "c"} + with patch.object(host, "command", return_value='{"MESSAGE":"private normal message","__CURSOR":"d"}'): + host.check_journal(cursors) + self.assertEqual(cursors, {"pulse": "d"}) + + def test_no_journal_cursor_and_malformed_service_state_fail_closed(self): + host = engine.Host() + with patch.object(host, "command", return_value=""): + with self.assertRaises(engine.Failure): + host.cursors() + with self.assertRaises(engine.Failure): + host.state() + + +class DispatchLifecycleTest(unittest.TestCase): + def test_observer_budget_covers_child_and_recovery_and_no_wait_owned_by_ssh(self): + source = dispatcher.BOOTSTRAP + self.assertIn("RuntimeMaxSec=45min", source) + self.assertIn("TimeoutStopSec=20min", source) + self.assertIn("KillMode=mixed", source) + self.assertIn("+ 4000", source) + self.assertGreater(4000, 45 * 60 + 20 * 60) + self.assertNotIn("'--wait'", source) + self.assertNotIn("systemctl', 'stop'", source) + + def test_bootstrap_retains_intent_and_repeated_or_uncertain_launch_does_not_replay(self): + # Execute the real bootstrap with only its filesystem/root/systemd/clock + # adapters rebound, including a lost submission response. No host tools. + for uncertain in (False, True): + with self.subTest(uncertain=uncertain), tempfile.TemporaryDirectory() as directory: + root = Path(directory) / "state" + source = dispatcher.BOOTSTRAP.replace("pathlib.Path('/var/lib/pulse-deploy/demo')", repr(root)) + # repr(Path) is not a Python literal: bind just the fixed path. + source = dispatcher.BOOTSTRAP.replace("'/var/lib/pulse-deploy/demo'", repr(str(root))) + source = source.replace("root.stat().st_uid == 0", "root.stat().st_uid == os.getuid()") + clock = [0] + calls = [] + payload = {"request": request(), "source": "c291cmNl", "installer": ""} + + def launch(argv, **kwargs): + calls.append(argv) + attempt = next((root / "attempts").iterdir()) + if uncertain: + raise subprocess.TimeoutExpired("systemd-run", 30) + (attempt / "receipt.json").write_text(json.dumps({"status": "committed"})) + return subprocess.CompletedProcess(argv, 0) + + class Input: + def read(self, _limit): + return json.dumps(payload) + + with patch.object(os, "geteuid", return_value=0), patch.object(subprocess, "run", side_effect=launch), \ + patch.object(dispatcher.sys, "stdin", Input()), patch("time.monotonic", side_effect=lambda: clock[0]), \ + patch("time.sleep", side_effect=lambda seconds: clock.__setitem__(0, clock[0] + seconds)), patch("builtins.print"): + for _ in range(2): + with self.assertRaises(SystemExit): + exec(compile(source, "fixed-bootstrap", "exec"), {}) + self.assertEqual(len(calls), 1) + attempt = next((root / "attempts").iterdir()) + self.assertTrue((attempt / "intent.json").exists()) + self.assertEqual((attempt / "request.json").stat().st_mode & 0o777, 0o600) + self.assertEqual(attempt.stat().st_mode & 0o777, 0o700) + if uncertain: + self.assertEqual(json.loads((attempt / "launch.json").read_text())["state"], "uncertain") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/recover-demo-server.yml b/.github/workflows/recover-demo-server.yml index c482081c59..b0dc92f2c0 100644 --- a/.github/workflows/recover-demo-server.yml +++ b/.github/workflows/recover-demo-server.yml @@ -15,7 +15,7 @@ concurrency: jobs: recover: runs-on: ubuntu-24.04 - timeout-minutes: 45 + timeout-minutes: 90 environment: demo-stable env: DEMO_EXPECTED_HOSTNAME: ${{ vars.DEMO_EXPECTED_HOSTNAME }} @@ -117,16 +117,8 @@ jobs: MOCK_UPDATE_INTERVAL: ${{ steps.runtime_profile.outputs.mock_update_interval }} run: | set -euo pipefail - ssh -i ~/.ssh/id_ed25519 \ - -o IdentitiesOnly=yes \ - -o StrictHostKeyChecking=yes \ - -o UserKnownHostsFile=~/.ssh/known_hosts \ - "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "bash -s -- $(printf '%q ' "$DEMO_EXPECTED_HOSTNAME" "$DEMO_LOCAL_BASE_URL" "$EXPECTED_VERSION" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_SAMPLE_INTERVAL" "$MOCK_UPDATE_INTERVAL")" \ - < .github/scripts/recover-demo-runtime.sh \ - > demo-runtime-recovery.json - jq -e '.schema_version == 1 and (.status == "recovered" or .status == "healthy_noop")' \ - demo-runtime-recovery.json + bash .github/scripts/recover-demo-runtime.sh > demo-runtime-recovery.json + jq -e '.schema_version == 2 and (.status == "committed" or .status == "healthy_noop") and .forward.elapsed_seconds >= 300' demo-runtime-recovery.json echo "mutated=$(jq -r '.mutated' demo-runtime-recovery.json)" >> "$GITHUB_OUTPUT" - name: Capture bounded Pulse failure diagnostics @@ -199,22 +191,9 @@ jobs: export PULSE_DEMO_AUTH_PASS="${DEMO_AUTH_PASS:-demo}" ./scripts/run_demo_public_browser_smoke.sh - - name: Compensate failed mutated recovery - if: failure() && steps.recovery.outputs.mutated == 'true' - env: - DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} - DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} - run: | - set -euo pipefail - BEFORE="$(ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "systemctl show pulse --property=ActiveState,SubState,MainPID --value | paste -sd, -")" - ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "sudo systemctl stop pulse" - AFTER="$(ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "systemctl show pulse --property=ActiveState,SubState,MainPID --value | paste -sd, -")" - jq -n --arg before "$BEFORE" --arg after "$AFTER" \ - '{schema_version: 1, status: "compensated_unavailable", before: $before, after: $after}' \ - > demo-runtime-rollback.json + # The retained systemd transaction already owns compensation. A later + # browser/network failure is evidence to reconcile, not authority to stop + # an unrelated or already-committed Pulse process in a second SSH session. - name: Retain bounded recovery evidence if: always() @@ -223,7 +202,6 @@ jobs: name: stable-demo-runtime-recovery-${{ github.run_id }} path: | demo-runtime-recovery.json - demo-runtime-rollback.json demo-runtime-diagnostics.txt if-no-files-found: warn retention-days: 14 diff --git a/.github/workflows/update-demo-server.yml b/.github/workflows/update-demo-server.yml index 996f7bda56..214115ebbe 100644 --- a/.github/workflows/update-demo-server.yml +++ b/.github/workflows/update-demo-server.yml @@ -189,7 +189,7 @@ jobs: needs: resolve if: needs.resolve.outputs.skip != 'true' runs-on: ubuntu-24.04 - timeout-minutes: 60 + timeout-minutes: 90 environment: ${{ needs.resolve.outputs.environment_name }} env: DEMO_EXPECTED_HOSTNAME: ${{ vars.DEMO_EXPECTED_HOSTNAME }} @@ -450,309 +450,41 @@ jobs: echo "::error::Verification-only check found a demo version mismatch; refusing to update the host." exit 1 - - name: Prepare demo host storage - if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true' - env: - DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} - DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} - WORKFLOW_OUTPUT_1: ${{ steps.config.outputs.service_name }} - run: | - set -euo pipefail - SERVICE_NAME="${WORKFLOW_OUTPUT_1}" - REMOTE_SCRIPT=$(cat <<'EOF' - set -euo pipefail - SERVICE_NAME="$1" - CONFIG_DIR="/etc/${SERVICE_NAME:-pulse}" - BACKUP_PARENT="$(dirname "$CONFIG_DIR")" - BACKUP_BASENAME="$(basename "$CONFIG_DIR")" - KEEP_BACKUPS=2 - BACKUP_MARGIN_KB=$((64 * 1024)) - - echo "Demo host filesystem before cleanup:" - df -h "$BACKUP_PARENT" /tmp || true - - mapfile -t old_backups < <( - sudo find "$BACKUP_PARENT" -maxdepth 1 -type d -name "${BACKUP_BASENAME}.backup.*" -printf '%T@ %p\n' 2>/dev/null \ - | sort -nr \ - | awk -v keep="$KEEP_BACKUPS" 'NR > keep {sub(/^[^ ]+ /, ""); print}' - ) - if ((${#old_backups[@]} > 0)); then - printf 'Removing %d old demo backup directorie(s).\n' "${#old_backups[@]}" - printf '%s\0' "${old_backups[@]}" | sudo xargs -0 rm -rf -- - fi - - if [[ -d "$CONFIG_DIR" ]]; then - config_kb="$(sudo du -sk "$CONFIG_DIR" | awk 'NR == 1 {print $1}')" - free_kb="$(df -Pk "$BACKUP_PARENT" | awk 'NR == 2 {print $4}')" - required_kb=$((config_kb + BACKUP_MARGIN_KB)) - if ((free_kb < required_kb)); then - mapfile -t remaining_backups < <( - sudo find "$BACKUP_PARENT" -maxdepth 1 -type d -name "${BACKUP_BASENAME}.backup.*" -printf '%T@ %p\n' 2>/dev/null \ - | sort -n \ - | awk '{sub(/^[^ ]+ /, ""); print}' - ) - for backup_dir in "${remaining_backups[@]}"; do - printf 'Removing demo backup to restore install headroom: %s\n' "$backup_dir" - sudo rm -rf -- "$backup_dir" - free_kb="$(df -Pk "$BACKUP_PARENT" | awk 'NR == 2 {print $4}')" - if ((free_kb >= required_kb)); then - break - fi - done - fi - pruned_volatile=false - if ((free_kb < required_kb)); then - echo "Pruning demo volatile runtime stores to restore install headroom." - if command -v systemctl >/dev/null 2>&1; then - sudo systemctl stop "$SERVICE_NAME" || true - fi - mapfile -d '' volatile_paths < <( - sudo find "$CONFIG_DIR" -xdev -type f \( \ - -name "metrics.db" -o \ - -name "metrics.db-shm" -o \ - -name "metrics.db-wal" -o \ - -name "notification_queue.db" -o \ - -name "notification_queue.db-shm" -o \ - -name "notification_queue.db-wal" -o \ - -name "deploy.db" -o \ - -name "deploy.db-shm" -o \ - -name "deploy.db-wal" \ - \) -print0 2>/dev/null || true - ) - for volatile_path in "${volatile_paths[@]}"; do - printf 'Removing demo volatile store: %s\n' "$(basename "$volatile_path")" - sudo rm -f -- "$volatile_path" - pruned_volatile=true - done - config_kb="$(sudo du -sk "$CONFIG_DIR" | awk 'NR == 1 {print $1}')" - free_kb="$(df -Pk "$BACKUP_PARENT" | awk 'NR == 2 {print $4}')" - required_kb=$((config_kb + BACKUP_MARGIN_KB)) - fi - if ((free_kb < required_kb)); then - echo "::error::Demo host does not have enough free space to back up $CONFIG_DIR before install." - echo "Config: ${config_kb}KB, available: ${free_kb}KB, required: ${required_kb}KB" - sudo find "$BACKUP_PARENT" -maxdepth 1 -type d -name "${BACKUP_BASENAME}.backup.*" -printf '%TY-%Tm-%Td %TH:%TM %p\n' 2>/dev/null | sort || true - if [[ "$pruned_volatile" == "true" ]] && command -v systemctl >/dev/null 2>&1; then - sudo systemctl start "$SERVICE_NAME" || true - fi - exit 1 - fi - fi - - echo "Demo host filesystem after cleanup:" - df -h "$BACKUP_PARENT" /tmp || true - EOF - ) - ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" "bash -s -- $(printf '%q ' "$SERVICE_NAME")" <<<"$REMOTE_SCRIPT" - - - name: Upload tagged installer - if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true' - env: - DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} - DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} - run: | - set -euo pipefail - scp -i ~/.ssh/id_ed25519 /tmp/pulse-install.sh "$DEMO_SERVER_USER@$DEMO_SERVER_HOST:/tmp/pulse-install.sh" - - - name: Update demo server - if: inputs.verify_only != true && steps.current.outputs.skip_current != 'true' - env: - DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} - DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} - WORKFLOW_OUTPUT_1: ${{ needs.resolve.outputs.tag }} - WORKFLOW_OUTPUT_2: ${{ steps.config.outputs.service_name }} - run: | - set -euo pipefail - TAG="${WORKFLOW_OUTPUT_1}" - SERVICE_NAME="${WORKFLOW_OUTPUT_2}" - REMOTE_SCRIPT=$(cat <<'EOF' - set -euo pipefail - TAG="$1" - SERVICE_NAME="$2" - INSTALLER_ENV=() - if [ -n "$SERVICE_NAME" ]; then - INSTALLER_ENV+=("PULSE_SERVICE_NAME=$SERVICE_NAME") - fi - sudo env "${INSTALLER_ENV[@]}" bash /tmp/pulse-install.sh --version "$TAG" - rm -f /tmp/pulse-install.sh - EOF - ) - ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "bash -s -- $(printf '%q ' "$TAG" "$SERVICE_NAME")" <<<"$REMOTE_SCRIPT" - - - name: Restore demo runtime configuration + - name: Apply guarded demo transaction if: inputs.verify_only != true env: DEMO_SERVER_HOST: ${{ secrets.DEMO_SERVER_HOST }} DEMO_SERVER_USER: ${{ secrets.DEMO_SERVER_USER }} - WORKFLOW_OUTPUT_1: ${{ steps.config.outputs.service_name }} - WORKFLOW_OUTPUT_2: ${{ needs.resolve.outputs.mock_nodes }} - WORKFLOW_OUTPUT_3: ${{ needs.resolve.outputs.mock_vms_per_node }} - WORKFLOW_OUTPUT_4: ${{ needs.resolve.outputs.mock_lxcs_per_node }} - WORKFLOW_OUTPUT_5: ${{ needs.resolve.outputs.mock_docker_hosts }} - WORKFLOW_OUTPUT_6: ${{ needs.resolve.outputs.mock_docker_containers }} - WORKFLOW_OUTPUT_7: ${{ needs.resolve.outputs.mock_generic_hosts }} - WORKFLOW_OUTPUT_8: ${{ needs.resolve.outputs.mock_k8s_clusters }} - WORKFLOW_OUTPUT_9: ${{ needs.resolve.outputs.mock_k8s_nodes }} - WORKFLOW_OUTPUT_10: ${{ needs.resolve.outputs.mock_k8s_pods }} - WORKFLOW_OUTPUT_11: ${{ needs.resolve.outputs.mock_k8s_deployments }} - WORKFLOW_OUTPUT_12: ${{ needs.resolve.outputs.mock_seed_duration }} - WORKFLOW_OUTPUT_13: ${{ needs.resolve.outputs.mock_sample_interval }} - WORKFLOW_OUTPUT_14: ${{ needs.resolve.outputs.mock_update_interval }} + EXPECTED_VERSION: ${{ needs.resolve.outputs.tag }} + CURRENT_MATCHES: ${{ steps.current.outputs.skip_current }} + MOCK_NODES: ${{ needs.resolve.outputs.mock_nodes }} + MOCK_VMS_PER_NODE: ${{ needs.resolve.outputs.mock_vms_per_node }} + MOCK_LXCS_PER_NODE: ${{ needs.resolve.outputs.mock_lxcs_per_node }} + MOCK_DOCKER_HOSTS: ${{ needs.resolve.outputs.mock_docker_hosts }} + MOCK_DOCKER_CONTAINERS: ${{ needs.resolve.outputs.mock_docker_containers }} + MOCK_GENERIC_HOSTS: ${{ needs.resolve.outputs.mock_generic_hosts }} + MOCK_K8S_CLUSTERS: ${{ needs.resolve.outputs.mock_k8s_clusters }} + MOCK_K8S_NODES: ${{ needs.resolve.outputs.mock_k8s_nodes }} + MOCK_K8S_PODS: ${{ needs.resolve.outputs.mock_k8s_pods }} + MOCK_K8S_DEPLOYMENTS: ${{ needs.resolve.outputs.mock_k8s_deployments }} + MOCK_SEED_DURATION: ${{ needs.resolve.outputs.mock_seed_duration }} + MOCK_SAMPLE_INTERVAL: ${{ needs.resolve.outputs.mock_sample_interval }} + MOCK_UPDATE_INTERVAL: ${{ needs.resolve.outputs.mock_update_interval }} run: | set -euo pipefail - SERVICE_NAME="${WORKFLOW_OUTPUT_1}" - MOCK_NODES="${WORKFLOW_OUTPUT_2}" - MOCK_VMS_PER_NODE="${WORKFLOW_OUTPUT_3}" - MOCK_LXCS_PER_NODE="${WORKFLOW_OUTPUT_4}" - MOCK_DOCKER_HOSTS="${WORKFLOW_OUTPUT_5}" - MOCK_DOCKER_CONTAINERS="${WORKFLOW_OUTPUT_6}" - MOCK_GENERIC_HOSTS="${WORKFLOW_OUTPUT_7}" - MOCK_K8S_CLUSTERS="${WORKFLOW_OUTPUT_8}" - MOCK_K8S_NODES="${WORKFLOW_OUTPUT_9}" - MOCK_K8S_PODS="${WORKFLOW_OUTPUT_10}" - MOCK_K8S_DEPLOYMENTS="${WORKFLOW_OUTPUT_11}" - MOCK_SEED_DURATION="${WORKFLOW_OUTPUT_12}" - MOCK_SAMPLE_INTERVAL="${WORKFLOW_OUTPUT_13}" - MOCK_UPDATE_INTERVAL="${WORKFLOW_OUTPUT_14}" - REMOTE_SCRIPT=$(cat <<'EOF' - set -euo pipefail - SERVICE_NAME="${1:-pulse}" - DEMO_LOCAL_BASE_URL="$2" - MOCK_NODES="$3" - MOCK_VMS_PER_NODE="$4" - MOCK_LXCS_PER_NODE="$5" - MOCK_DOCKER_HOSTS="$6" - MOCK_DOCKER_CONTAINERS="$7" - MOCK_GENERIC_HOSTS="$8" - MOCK_K8S_CLUSTERS="$9" - MOCK_K8S_NODES="${10}" - MOCK_K8S_PODS="${11}" - MOCK_K8S_DEPLOYMENTS="${12}" - MOCK_SEED_DURATION="${13}" - MOCK_SAMPLE_INTERVAL="${14}" - MOCK_UPDATE_INTERVAL="${15}" - CONFIG_DIR="/etc/${SERVICE_NAME:-pulse}" - SERVICE_USER="$(systemctl show "$SERVICE_NAME" --property=User --value 2>/dev/null || true)" - SERVICE_GROUP="$(systemctl show "$SERVICE_NAME" --property=Group --value 2>/dev/null || true)" - SERVICE_USER="${SERVICE_USER:-pulse}" - SERVICE_GROUP="${SERVICE_GROUP:-$SERVICE_USER}" + MODE=update + if [ "$CURRENT_MATCHES" = "true" ]; then MODE=recover; fi + python3 .github/scripts/dispatch-demo-runtime.py "$MODE" > demo-runtime-transaction.json + jq -e '.schema_version == 2 and (.status == "committed" or .status == "healthy_noop") and .forward.elapsed_seconds >= 300' demo-runtime-transaction.json - resolve_config_dir() { - local fallback="$CONFIG_DIR" - local unit_env="" - local fragment="" - unit_env="$(systemctl show "$SERVICE_NAME" --property=Environment --value 2>/dev/null || true)" - for entry in $unit_env; do - case "$entry" in - PULSE_DATA_DIR=*) - printf '%s\n' "${entry#PULSE_DATA_DIR=}" - return 0 - ;; - esac - done - - fragment="$(systemctl show "$SERVICE_NAME" --property=FragmentPath --value 2>/dev/null || true)" - if [ -n "$fragment" ] && sudo test -r "$fragment"; then - local unit_data_dir - unit_data_dir="$(sudo sed -n -E 's/^Environment="?PULSE_DATA_DIR=([^"]*)"?$/\1/p' "$fragment" | tail -1)" - if [ -n "$unit_data_dir" ]; then - printf '%s\n' "$unit_data_dir" - return 0 - fi - fi - - printf '%s\n' "$fallback" - } - - CONFIG_DIR="$(resolve_config_dir)" - ENV_FILE="$CONFIG_DIR/.env" - - set_env_value() { - local key="$1" - local value="$2" - sudo mkdir -p "$CONFIG_DIR" - if sudo test -f "$ENV_FILE"; then - if sudo grep -Eq "^[[:space:]]*${key}=" "$ENV_FILE"; then - sudo sed -i "s|^[[:space:]]*${key}=.*|${key}=${value}|" "$ENV_FILE" - else - printf '\n%s=%s\n' "$key" "$value" | sudo tee -a "$ENV_FILE" >/dev/null - fi - else - printf '# Pulse demo runtime configuration\n%s=%s\n' "$key" "$value" | sudo tee "$ENV_FILE" >/dev/null - fi - } - - ensure_demo_fixture_entitlement() { - local billing_file="$CONFIG_DIR/billing.json" - local tmp_file - tmp_file="$(mktemp)" - sudo mkdir -p "$CONFIG_DIR" - - if sudo test -s "$billing_file" && sudo jq ' - .capabilities = (((.capabilities // []) + ["demo_fixtures"]) | unique) - | .limits = (.limits // {}) - | .meters_enabled = (.meters_enabled // []) - | .plan_version = (if (.plan_version // "") == "" then "community" else .plan_version end) - | .subscription_state = (if (.subscription_state // "") == "" then "active" else .subscription_state end) - | del(.integrity) - ' "$billing_file" > "$tmp_file"; then - : - else - cat > "$tmp_file" <<'JSON' - {"capabilities":["demo_fixtures"],"limits":{},"meters_enabled":[],"plan_version":"community","subscription_state":"active"} - JSON - fi - - sudo tee "$billing_file" < "$tmp_file" >/dev/null - rm -f "$tmp_file" - sudo chown "$SERVICE_USER:$SERVICE_GROUP" "$billing_file" || sudo chown "$SERVICE_USER" "$billing_file" || true - sudo chmod 600 "$billing_file" - echo "Demo fixture entitlement ensured in governed demo billing state." - } - - set_env_value DEMO_MODE true - set_env_value PULSE_MOCK_MODE true - set_env_value PULSE_MOCK_NODES "$MOCK_NODES" - set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE" - set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE" - set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS" - set_env_value PULSE_MOCK_DOCKER_CONTAINERS "$MOCK_DOCKER_CONTAINERS" - set_env_value PULSE_MOCK_GENERIC_HOSTS "$MOCK_GENERIC_HOSTS" - set_env_value PULSE_MOCK_K8S_CLUSTERS "$MOCK_K8S_CLUSTERS" - set_env_value PULSE_MOCK_K8S_NODES "$MOCK_K8S_NODES" - set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS" - set_env_value PULSE_MOCK_K8S_DEPLOYMENTS "$MOCK_K8S_DEPLOYMENTS" - set_env_value PULSE_MOCK_RANDOM_METRICS true - set_env_value PULSE_MOCK_STOPPED_PERCENT 6 - # Never carry the isolated local-dev metrics-store opt-in onto the - # persistent public demo data directory. - set_env_value PULSE_MOCK_SEED_METRICS_STORE false - set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION" - set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL" - set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL" - - ensure_demo_fixture_entitlement - - sudo chown "$SERVICE_USER:$SERVICE_GROUP" "$ENV_FILE" || sudo chown "$SERVICE_USER" "$ENV_FILE" || true - sudo chmod 600 "$ENV_FILE" - sudo systemctl daemon-reload - sudo systemctl restart "$SERVICE_NAME" - - for attempt in $(seq 1 30); do - if curl -fsS "${DEMO_LOCAL_BASE_URL}/api/health" >/dev/null; then - echo "Demo service restarted with governed demo runtime configuration." - exit 0 - fi - sleep 2 - done - - echo "::error::Demo service did not become healthy after restoring demo runtime configuration." - exit 1 - EOF - ) - ssh -i ~/.ssh/id_ed25519 "$DEMO_SERVER_USER@$DEMO_SERVER_HOST" \ - "bash -s -- $(printf '%q ' "$SERVICE_NAME" "$DEMO_LOCAL_BASE_URL" "$MOCK_NODES" "$MOCK_VMS_PER_NODE" "$MOCK_LXCS_PER_NODE" "$MOCK_DOCKER_HOSTS" "$MOCK_DOCKER_CONTAINERS" "$MOCK_GENERIC_HOSTS" "$MOCK_K8S_CLUSTERS" "$MOCK_K8S_NODES" "$MOCK_K8S_PODS" "$MOCK_K8S_DEPLOYMENTS" "$MOCK_SEED_DURATION" "$MOCK_SAMPLE_INTERVAL" "$MOCK_UPDATE_INTERVAL")" <<<"$REMOTE_SCRIPT" + - name: Retain guarded demo transaction evidence + if: always() && inputs.verify_only != true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: stable-demo-transaction-${{ github.run_id }} + path: demo-runtime-transaction.json + if-no-files-found: warn + retention-days: 14 - name: Verify demo runtime env: diff --git a/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md b/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md new file mode 100644 index 0000000000..af5e4322da --- /dev/null +++ b/docs/release-control/v6/internal/DEMO_RUNTIME_TRANSACTION.md @@ -0,0 +1,60 @@ +# Stable demo transaction and acceptance + +This continues the existing update/recovery route. It was built for a single +demo workflow and first-success health; the actual host also runs Relay and +Caddy. An immediate healthy sample does not cover a delayed failure, and a +workflow queue does not exclude another repository's host deployment. + +## Implemented boundary + +- Both existing mutating workflows submit the same reviewed Python engine. + Update still requires the immutable stable activation and customer-promotion + lease, exact tagged installer and trusted release signing key. Verification + only never submits it. Recovery never installs a different executable. +- The independently owned systemd child holds the existing Relay deployment + file lock through capture, installation/profile work, forward observation, + complete restoration and recovery observation. It does not restart Relay or + Caddy, upgrade a package, lift the Caddy hold or activate frozen commercial copy. +- The complete quiescent Pulse executable/unit/drop-in/data estate is retained + privately before mutation. No backup or database is deleted for space. The + unhealthy demo-only route retains the original generated operational history + before its existing bounded-profile reset. No snapshot is automatically erased. +- Readiness has a separate 60-second limit. Forward, healthy-noop and restored + paths require 300 seconds of actual local/public/Relay HTTP 200, expected + version, stable service PID/restarts and available new-journal observations. + Transport errors, redirects, missing state/journals and new crashes fail. + Recovery uses fresh cursors; a retained forward failure remains a failure. +- Failed/unhealthy/uncertain recovery remains owned and blocks subsequent + mutation. A prior failed baseline is not a healthy rollback. No crash dump is + induced. Receipt fields contain fixed diagnostics and identities, never raw + journal/HTTP bodies, environment values or snapshot contents. +- Source-bound intent precedes systemd submission. An identical request reads + its receipt and cannot start another child after a lost response. SSH loss + does not terminate the child. TERM/INT/HUP cannot abort owned recovery. The + 45-minute runtime, 20-minute stop, 4,000-second remote observer, 4,100-second + client and 90-minute workflow budgets cover both observation windows. + +## Available validation and remaining result + +`TestDemoTransactionConnectedRecoveryControls` executes the real engine and +bootstrap using a real private filesystem estate and virtual-clock command, +service, journal and HTTP adapters. Tests retain delayed 55/299-second failure, +changed executable/unit/data rollback, cancellation, failed and unhealthy +recovery, headroom refusal, shared-lock contention, source mismatch, journal +failure, cohost restart, source-bound repeated/lost submission and no-op proof. +The production window is not shortened. This is **not native acceptance**. + +Before treating this route as restored, the exact reviewed source must return +native disposable systemd/SSH-loss receipts and signed published-installer +forward/reverse data/identity results. Then reconcile actual demo workflow and +public browser readback through the release owner; source tests do not establish +an installed recovery. Delivery owns that continuation and the remaining ingress +migration. Unrestricted root/Tailscale administration can still bypass a source +transaction; this candidate is not host enforcement. Workers neither install +policy/credentials nor replay a successful publication to test it. + +Retained failed/unknown estates require the operational owner's reconciliation; +do not remove a lock, intent or snapshot merely to permit a fresh sample. A +later frontend/network failure is not authority for a second SSH session to +stop a committed or unrelated service. Keep the signed packet, failure and +native/browser acceptance distinct in the release judgment. diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index efa5d4e8c8..78533e9a3f 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -15,6 +15,36 @@ ## Purpose +### Demo transactions share the service host, not just a workflow queue + +Stable demo update and recovery keep their existing publication/activation, +exact-tag installer signing key, Tailscale, pinned SSH and environment boundaries. +The mutating paths now share the Relay host's existing +`/var/lib/pulse-deploy/relay/deploy.lock` and one retained systemd transaction. +Every successful path, including a healthy no-op, needs a complete 300-second +local/public/Relay health, version, service PID/restart and new-journal-crash +window. Readiness is separate. Missing observations fail closed. + +Before installation or runtime-profile mutation, stop only Pulse and durably +capture its executable, unit/drop-ins and entire persistent data estate. Refuse +insufficient space rather than prune backups/databases. A failed healthy change +restores that complete estate and observes another 300 seconds; the original +operation still fails. An unhealthy recovery baseline is never claimed as a +healthy rollback. Retain its original generated demo history, all snapshots and +failed/uncertain terminal evidence privately. An unresolved estate blocks later +mutation. No source-worker production capability, package upgrade, Relay copy +activation or credential is added. + +SSH observes, but never owns or cancels, the systemd child. Durable intent +precedes submission; the exact request identity is not replayed after a lost +response. Forward cancellation initiates restoration; recovery ignores +TERM/INT/HUP until its watch and terminal receipt are retained. Child, stop, +observer and workflow deadlines cover both full windows. A later browser or +network check must not stop a committed or unrelated service in another SSH +session. These connected source controls do not establish native systemd, +signed installed forward/reverse acceptance, ingress enforcement or customer +recovery. See [demo transaction acceptance](../DEMO_RUNTIME_TRANSACTION.md). + ### Superseded pull request validation Build and Test and Core E2E cancel an older run for the same pull request when @@ -732,6 +762,8 @@ release-latency optimization. 23. `.github/workflows/update-demo-server.yml` 23a. `.github/workflows/recover-demo-server.yml` 23b. `.github/scripts/recover-demo-runtime.sh` +23c. `.github/scripts/demo-runtime-transaction.py` +23d. `.github/scripts/dispatch-demo-runtime.py` 23c. `.github/scripts/resolve-demo-runtime-profile.sh` 24. `.github/workflows/validate-release-assets.yml` 25. `.github/workflows/install-sh-smoke.yml` diff --git a/docs/release-control/v6/internal/subsystems/registry.json b/docs/release-control/v6/internal/subsystems/registry.json index 5b59559345..69c3b883c3 100644 --- a/docs/release-control/v6/internal/subsystems/registry.json +++ b/docs/release-control/v6/internal/subsystems/registry.json @@ -4515,6 +4515,8 @@ ".dockerignore", ".github/ISSUE_TEMPLATE/v6_rc_feedback.yml", ".github/scripts/check-demo-reachability.sh", + ".github/scripts/demo-runtime-transaction.py", + ".github/scripts/dispatch-demo-runtime.py", ".github/scripts/internal-api-test-seconds.txt", ".github/scripts/record-internal-api-test-seconds.py", ".github/scripts/recover-demo-runtime.sh", @@ -4544,6 +4546,9 @@ ".github/workflows/retry-release-convergence.yml", ".github/workflows/update-demo-server.yml", ".github/workflows/validate-release-assets.yml", + "Dockerfile", + "Makefile", + "VERSION", "cmd/pulse-control-plane/main.go", "cmd/pulse-control-plane/provider_msp.go", "cmd/pulse-control-plane/provider_msp_backup.go", @@ -4553,13 +4558,12 @@ "cmd/pulse-control-plane/provider_msp_recover.go", "cmd/pulse-control-plane/provider_msp_status.go", "docker-compose.yml", - "Dockerfile", "docs/MSP.md", + "docs/RELEASE_NOTES.md", + "docs/UPGRADE_v6.md", "docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md", "docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md", "docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md", - "docs/RELEASE_NOTES.md", - "docs/UPGRADE_v6.md", "frontend-modern/package-lock.json", "frontend-modern/package.json", "frontend-modern/public/docs/MSP.md", @@ -4579,7 +4583,6 @@ "internal/cloudcp/provider_msp_backup.go", "internal/cloudcp/provider_msp_recovery.go", "internal/cloudcp/tenant_runtime_rollout.go", - "Makefile", "package-lock.json", "package.json", "pulse-enterprise:.github/workflows/build-pro-release.yml", @@ -4654,17 +4657,16 @@ "scripts/uninstall-sensor-proxy.sh", "scripts/validate-release.sh", "scripts/verify-github-release-integrity.sh", - "tests/integration/playwright.config.ts", "tests/integration/QUICK_START.md", "tests/integration/README.md", + "tests/integration/playwright.config.ts", "tests/integration/scripts/bootstrap-hosted-mobile-onboarding.mjs", "tests/integration/scripts/hosted-mobile-token-runtime.mjs", "tests/integration/scripts/hosted-tenant-runtime.mjs", "tests/integration/scripts/managed-dev-runtime.mjs", "tests/integration/scripts/relay-mobile-token-helper.go", "tests/integration/tests/helpers.ts", - "tests/integration/tests/runtime-defaults.ts", - "VERSION" + "tests/integration/tests/runtime-defaults.ts" ], "release_cycle_artifact_globs": [ "docs/RELEASE_NOTES.md", @@ -4749,6 +4751,8 @@ "match_files": [ ".github/ISSUE_TEMPLATE/v6_rc_feedback.yml", ".github/scripts/check-demo-reachability.sh", + ".github/scripts/demo-runtime-transaction.py", + ".github/scripts/dispatch-demo-runtime.py", ".github/scripts/recover-demo-runtime.sh", ".github/scripts/resolve-demo-runtime-profile.sh", ".github/scripts/setup-demo-ssh.sh", @@ -4771,11 +4775,12 @@ ".github/workflows/retry-release-convergence.yml", ".github/workflows/update-demo-server.yml", ".github/workflows/validate-release-assets.yml", + "VERSION", + "docs/RELEASE_NOTES.md", + "docs/UPGRADE_v6.md", "docs/release-control/v6/internal/PRE_RELEASE_CHECKLIST.md", "docs/release-control/v6/internal/RC_TO_GA_REHEARSAL_TEMPLATE.md", "docs/release-control/v6/internal/RELEASE_PROMOTION_POLICY.md", - "docs/RELEASE_NOTES.md", - "docs/UPGRADE_v6.md", "frontend-modern/public/docs/UPGRADE_v6.md", "frontend-modern/src/components/UpdateBanner.tsx", "frontend-modern/src/components/WhatsNewCard.tsx", @@ -4803,17 +4808,18 @@ "scripts/trigger-release-dry-run.sh", "scripts/trigger-release.sh", "scripts/trigger-stable-patch.sh", - "scripts/verify-github-release-integrity.sh", - "VERSION" + "scripts/verify-github-release-integrity.sh" ], "allow_same_subsystem_tests": false, "test_prefixes": [], "exact_files": [ + ".github/scripts/tests/test_demo_runtime_transaction.py", "frontend-modern/src/components/__tests__/whatsNewModel.test.ts", "frontend-modern/src/utils/__tests__/docsLinks.test.ts", "pulse-enterprise:scripts/validate-pro-release-line_test.sh", "pulse-pro:scripts/tests/test_validate_paid_runtime_distribution.py", "scripts/installtests/build_release_assets_test.go", + "scripts/installtests/demo_transaction_test.go", "scripts/release_control/internal/record_rc_to_ga_rehearsal_test.py", "scripts/release_control/internal/release_preflight_test.py", "scripts/release_control/mobile_release_gate_test.py", diff --git a/scripts/installtests/build_release_assets_test.go b/scripts/installtests/build_release_assets_test.go index 94688451c0..fff9fd54fd 100644 --- a/scripts/installtests/build_release_assets_test.go +++ b/scripts/installtests/build_release_assets_test.go @@ -2306,7 +2306,11 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { if err != nil { t.Fatalf("read demo runtime profile resolver: %v", err) } - workflow := string(workflowBytes) + "\n" + string(profileBytes) + transactionBytes, err := os.ReadFile(repoFile(".github", "scripts", "demo-runtime-transaction.py")) + if err != nil { + t.Fatal(err) + } + workflow := string(workflowBytes) + "\n" + string(profileBytes) + "\n" + string(transactionBytes) required := []string{ `- name: Tailscale`, `uses: tailscale/github-action@780049a30b6ff5c378a9e7b389d15ece7a204888 # v4.1.3`, @@ -2319,7 +2323,6 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { `workflow_call:`, `verify_only:`, `Waiting for activated release assets to be available`, - `bash /tmp/pulse-install.sh --version "$TAG"`, `Refuse mutation during verification-only checks`, `uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0`, `go run ./scripts/release_update_key.go public-key-ssh`, @@ -2327,15 +2330,10 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { `Verify target host identity`, `bash .github/scripts/setup-demo-ssh.sh`, `Demo environment points at host $REMOTE_HOSTNAME but expected $DEMO_EXPECTED_HOSTNAME.`, - `Prepare demo host storage`, - `KEEP_BACKUPS=2`, - `Removing demo backup to restore install headroom: %s`, - `Pruning demo volatile runtime stores to restore install headroom.`, - `sudo find "$CONFIG_DIR" -xdev -type f`, - `-name "metrics.db"`, - `Removing demo volatile store: %s`, - `Demo host does not have enough free space to back up $CONFIG_DIR before install.`, - `Restore demo runtime configuration`, + `Apply guarded demo transaction`, + `python3 .github/scripts/dispatch-demo-runtime.py "$MODE"`, + `Retain guarded demo transaction evidence`, + `.forward.elapsed_seconds >= 300`, `Resolve target-compatible demo runtime profile`, `git grep -q 'mockEagerHistoryPVEGuestLimit'`, `git grep -q 'UpdateMetricCohort'`, @@ -2361,22 +2359,13 @@ func TestUpdateDemoWorkflowUsesGovernedNetworkPath(t *testing.T) { `MOCK_SEED_DURATION=2h`, `MOCK_SAMPLE_INTERVAL=5m`, `MOCK_UPDATE_INTERVAL=15s`, - `resolve_config_dir`, - `set_env_value DEMO_MODE true`, - `set_env_value PULSE_MOCK_MODE true`, - `set_env_value PULSE_MOCK_NODES "$MOCK_NODES"`, - `set_env_value PULSE_MOCK_VMS_PER_NODE "$MOCK_VMS_PER_NODE"`, - `set_env_value PULSE_MOCK_LXCS_PER_NODE "$MOCK_LXCS_PER_NODE"`, - `set_env_value PULSE_MOCK_DOCKER_HOSTS "$MOCK_DOCKER_HOSTS"`, - `set_env_value PULSE_MOCK_K8S_PODS "$MOCK_K8S_PODS"`, - `set_env_value PULSE_MOCK_SEED_METRICS_STORE false`, - `set_env_value PULSE_MOCK_TRENDS_SEED_DURATION "$MOCK_SEED_DURATION"`, - `set_env_value PULSE_MOCK_TRENDS_SAMPLE_INTERVAL "$MOCK_SAMPLE_INTERVAL"`, - `set_env_value PULSE_MOCK_UPDATE_INTERVAL "$MOCK_UPDATE_INTERVAL"`, - `ensure_demo_fixture_entitlement`, - `"demo_fixtures"`, - `del(.integrity)`, - `Demo fixture entitlement ensured in governed demo billing state.`, + `WINDOW = 300`, + `/var/lib/pulse-deploy/relay/deploy.lock`, + `snapshot-headroom`, + `restored-estate-identity`, + `demo_fixtures`, + `PULSE_MOCK_SEED_METRICS_STORE`, + `installer-identity`, `/api/license/runtime-capabilities`, `Mock mode enabled`, `Demo server mock mode did not enable after entitlement sync`, diff --git a/scripts/installtests/demo_transaction_test.go b/scripts/installtests/demo_transaction_test.go new file mode 100644 index 0000000000..f53ed0abc0 --- /dev/null +++ b/scripts/installtests/demo_transaction_test.go @@ -0,0 +1,56 @@ +package installtests + +import ( + "os" + "os/exec" + "strings" + "testing" +) + +func TestDemoTransactionConnectedRecoveryControls(t *testing.T) { + cmd := exec.Command("python3", repoFile(".github", "scripts", "tests", "test_demo_runtime_transaction.py")) + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("demo connected transaction controls failed: %v\n%s", err, output) + } + if !strings.Contains(string(output), "\nOK\n") { + t.Fatalf("demo controls did not return a complete unittest verdict: %s", output) + } +} + +func TestDemoTransactionKeepsSignedAdmissionAndVerificationOnlyReadOnly(t *testing.T) { + workflow, err := os.ReadFile(repoFile(".github", "workflows", "update-demo-server.yml")) + if err != nil { + t.Fatal(err) + } + text := string(workflow) + start := strings.Index(text, "- name: Apply guarded demo transaction") + end := strings.Index(text[start:], "- name: Retain guarded demo transaction evidence") + if start < 0 || end < 0 { + t.Fatal("missing retained demo transaction entry") + } + entry := text[start : start+end] + if !strings.Contains(entry, "if: inputs.verify_only != true") { + t.Fatal("verification-only invocation must never submit a transaction") + } + for _, forbidden := range []string{"Pruning demo volatile runtime stores", "Removing demo backup to restore install headroom", "sudo systemctl restart"} { + if strings.Contains(text, forbidden) { + t.Fatalf("unguarded demo mutation remains: %s", forbidden) + } + } + for _, required := range []string{"Require exact committed activation marker for mutation", "release-activation.json", "Stable demo mutation refuses mutable, inactive, or prerelease tag", "PINNED_RELEASE_SSH_PUBLIC_KEY", "git show \"refs/tags/${TAG}:install.sh\"", "Verify public browser smoke"} { + if !strings.Contains(text, required) { + t.Fatalf("lost signed/admission/customer verification boundary: %s", required) + } + } + recovery, err := os.ReadFile(repoFile(".github", "workflows", "recover-demo-server.yml")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(recovery), "sudo systemctl stop pulse") || strings.Contains(string(recovery), "Compensate failed mutated recovery") { + t.Fatal("later workflow failure must not stop a committed or unrelated service") + } + if !strings.Contains(string(recovery), ".forward.elapsed_seconds >= 300") || !strings.Contains(string(recovery), "if: always()") { + t.Fatal("recovery must validate and retain the actual terminal observation") + } +}