Files
pulse/scripts/installtests/auto_update_systemd.py
T
pulse-triage[bot] e1cd6843e9 Preserve native updater failures across cleanup
Keep the original failed transaction command distinct from any later cleanup command. Retain fixed operation labels and require portable controls for command, timeout, assertion, cleanup-only and successful receipts, without changing update transactions or disclosing command output.

Change-source: pulse-maintainer
2026-10-14 09:16:19 +01:00

529 lines
27 KiB
Python

#!/usr/bin/env python3
"""Current-source updater transactions on a disposable GitHub-hosted systemd VM.
The two real, real-embed binaries differ only in synthetic version ldflags.
Installers and archives use an ephemeral test signer, never a release key.
Only download transport and /tmp allocation are redirected to private fixtures;
production signature, replacement, service observation and recovery functions run.
This is not a published-release/LXC, timer-sandbox or browser acceptance test.
"""
import argparse
import hashlib
import json
import os
from pathlib import Path
import re
import shutil
import signal
import stat
import subprocess
import tarfile
import time
import urllib.error
import urllib.request
UNIT = "pulse-source-updater"
TAGS = ("v0.0.1", "v0.0.2")
PORT = 17655
PARSE_MARKER = "# Parse command line arguments\n"
def digest(path):
with Path(path).open("rb") as stream:
return hashlib.file_digest(stream, "sha256").hexdigest()
def installer_definitions(source):
if source.count(PARSE_MARKER) != 1:
raise ValueError("installer definition boundary changed")
definitions, entry = source.split(PARSE_MARKER)
if not entry.startswith("while [[ $# -gt 0 ]]; do\n"):
raise ValueError("installer entry boundary changed")
return definitions
def hosted_admission(env, uid, source):
if (uid != 0 or env.get("GITHUB_ACTIONS") != "true"
or env.get("RUNNER_ENVIRONMENT") != "github-hosted"
or env.get("GITHUB_REPOSITORY") != "rcourtman/Pulse"
or not re.fullmatch(r"[0-9a-f]{40}", source)
or any(Path(p).exists() for p in ("/.dockerenv", "/opt/pulse", "/etc/pulse"))):
raise ValueError("native updater proof requires a fresh public hosted VM")
# A closed adapter: missing fixtures fail, never fall back to online curl.
TRANSPORT = r'''#!/usr/bin/python3
import os, pathlib, shutil, subprocess, sys
root = pathlib.Path(os.environ["PULSE_NATIVE_ROOT"])
args = sys.argv[1:]
if pathlib.Path(sys.argv[0]).name == "mktemp":
if args and args[-1].startswith("/tmp/pulse-"):
args[-1] = str(root / "tmp" / pathlib.Path(args[-1]).name)
os.execv("/usr/bin/mktemp", ["mktemp", *args])
# Admit the production downloader's exact bounded HTTPS envelope, not an
# arbitrary curl option parser. The short form is used by isolated controls.
flags = ["--disable", "-fsSL", "--connect-timeout", "5", "--max-time", "60",
"--proto", "=https", "--proto-redir", "=https"]
if args[:len(flags)] == flags:
args = args[len(flags):]
elif args[:1] == ["-fsSL"]:
args = args[1:]
else:
sys.exit(97)
if len(args) == 1:
url, out = args[0], None
elif len(args) == 3 and args[1] == "-o":
url, out = args[0], pathlib.Path(args[2])
else:
sys.exit(97)
allowed = {"https://github.com/rcourtman/Pulse/releases/latest/download/install.sh": "installer"}
for tag in ("v0.0.1", "v0.0.2"):
allowed["https://github.com/rcourtman/Pulse/releases/download/" + tag + "/install.sh"] = "installer"
allowed.update({url + ".sshsig": name + ".sshsig" for url, name in list(allowed.items())})
name = allowed.get(url)
if name is None:
sys.exit(97)
file = root / name
if out is None:
sys.stdout.buffer.write(file.read_bytes())
else:
shutil.copyfile(file, out)
'''
INSTALLER = r'''#!/bin/bash
set -euo pipefail
[[ $# == 2 && "$1" == --version ]]
case "$2" in v0.0.1|v0.0.2) ;; *) exit 97 ;; esac
source "$PULSE_NATIVE_ROOT/installer-definitions"
PINNED_RELEASE_SSH_PUBLIC_KEY=$(cat "$PULSE_NATIVE_ROOT/signer/key.pub")
printf 'attempted\n' >> "$PULSE_NATIVE_ROOT/installer-calls"
install_pulse_archive "$PULSE_NATIVE_ROOT/archives/pulse-$2-linux-amd64.tar.gz" "$2"
# Fail after real signed replacement, not before the interesting transaction.
case "${PULSE_NATIVE_FAULT:-}" in installer|restore) exit 23 ;; esac
if [[ "$PULSE_WAS_ACTIVE" == true ]]; then
systemctl start "$SERVICE_NAME"
fi
'''
UPDATER = r'''
set -euo pipefail
source "$PULSE_NATIVE_UPDATER"
PINNED_RELEASE_SSH_PUBLIC_KEY=$(cat "$PULSE_NATIVE_ROOT/signer/key.pub")
# One explicit restore-copy failure tests retained backup/no unsafe restart.
if [[ "${PULSE_NATIVE_FAULT:-}" == restore ]]; then
cp() {
if [[ "${*: -1}" == *.rollback.* ]]; then return 71; fi
command cp "$@"
}
fi
status=0
perform_update "$PULSE_NATIVE_TARGET" || status=$?
# Exercise a subsequent function return: the transaction's trap must be gone.
is_prerelease_tag v0.0.3-rc.1
exit "$status"
'''
class NativeLab:
def __init__(self, root, repo, before, after, source):
self.root, self.repo = Path(root), Path(repo)
self.binaries = dict(zip(TAGS, (Path(before), Path(after))))
self.receipt = {"schema_version": 1, "source_sha": source,
"scope": "current-source-synthetic-signed-systemd-transaction",
"phases": [], "cleanup": {"complete": False}}
self.env = {"PATH": str(self.root / "tools") + ":/usr/sbin:/usr/bin:/sbin:/bin",
"LANG": "C.UTF-8", "LC_ALL": "C", "HOME": str(self.root),
"PULSE_NATIVE_ROOT": str(self.root),
"PULSE_NATIVE_UPDATER": str(self.repo / "scripts/pulse-auto-update.sh"),
"PULSE_SERVICE_NAME": UNIT, "PULSE_INSTALL_DIR": str(self.root / "install"),
"PULSE_CONFIG_DIR": str(self.root / "data"),
"PULSE_BINARY_LINK_PATH": str(self.root / "pulse"),
"PULSE_UPDATE_HELPER_PATH": str(self.root / "update"),
"PULSE_PROFILE_PATH": str(self.root / "profile"),
"PULSE_BASHRC_PATH": str(self.root / "bashrc"), "FRONTEND_PORT": str(PORT)}
self.unit = Path("/etc/systemd/system") / (UNIT + ".service")
self.owned_unit = False
self.owned_root = False
self.owned_user = False
self.operation = "prepare-admission"
def run(self, *args, timeout=120, ok=(0,)):
process = subprocess.Popen(args, env=self.env, stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
start_new_session=True)
try:
output, _ = process.communicate(timeout=timeout)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGTERM)
try:
process.communicate(timeout=2)
except subprocess.TimeoutExpired:
os.killpg(process.pid, signal.SIGKILL)
process.communicate(timeout=2)
self.receipt["command_failure"] = {"command": Path(args[0]).name,
"deadline_exceeded": True,
"observed_exit": process.returncode,
"operation": self.operation}
raise RuntimeError("native command deadline exceeded") from None
# Private in-memory diagnostic for confined controls, never a receipt field.
self._last_output = output
result = subprocess.CompletedProcess(args, process.returncode, output)
if result.returncode not in ok:
self.receipt["command_failure"] = {"command": Path(args[0]).name,
"observed_exit": result.returncode,
"operation": self.operation,
"output_bytes": len(output),
"output_sha256": hashlib.sha256(output).hexdigest()}
raise RuntimeError(f"native command failed ({result.returncode}): {Path(args[0]).name}")
return result
def shell(self, script, **env):
script = "set -euo pipefail\n" + script
old = self.env.copy()
try:
self.env.update(env)
return self.run("/bin/bash", "-c", script)
finally:
self.env = old
def state(self):
result = self.run("systemctl", "show", UNIT + ".service", "--no-pager",
"--property=LoadState,ActiveState,UnitFileState,MainPID")
pairs = dict(line.split("=", 1) for line in result.stdout.decode().splitlines())
if set(pairs) != {"LoadState", "ActiveState", "UnitFileState", "MainPID"}:
raise RuntimeError("incomplete native manager observation")
return pairs
def prepare(self):
self.operation = "prepare-admission"
for path in (self.unit, Path("/run/systemd/system") / self.unit.name):
if path.exists() or path.is_symlink():
raise ValueError("native fixture unit already exists")
# Own the fresh scratch/HOME before launching any child. A platform
# runtime may initialise HOME even for a read-only manager observation.
# mkdir remains exclusive: never adopt or clean a pre-existing path.
self.root.mkdir(mode=0o755) # A pulse-owned service must traverse it.
self.owned_root = True
if self.state()["LoadState"] != "not-found":
raise ValueError("native fixture unit is already known to systemd")
for name in ("tools", "tmp", "archives", "signer"):
(self.root / name).mkdir(mode=0o700 if name == "signer" else 0o755)
self.operation = "prepare-source"
source = (self.repo / "install.sh").read_text()
(self.root / "installer-definitions").write_text(installer_definitions(source))
self.receipt["sources"] = {p: digest(self.repo / p) for p in
("install.sh", "scripts/pulse-auto-update.sh",
"scripts/installtests/auto_update_systemd.py")}
self.receipt["binaries"] = {tag: digest(binary) for tag, binary in self.binaries.items()}
if len(set(self.receipt["binaries"].values())) != 2:
raise ValueError("same executable bytes prove no replacement")
for tag, binary in self.binaries.items():
self.operation = "verify-binary-" + tag
if self.run(str(binary), "--version").stdout.decode().splitlines()[0] != "Pulse " + tag:
raise ValueError("synthetic binary identity mismatch")
self.create_signer()
for tag, binary in self.binaries.items():
self.operation = "prepare-archive-" + tag
version = self.root / "VERSION"
version.write_text(tag + "\n")
archive = self.root / "archives" / f"pulse-{tag}-linux-amd64.tar.gz"
with tarfile.open(archive, "w:gz") as tar:
tar.add(binary, arcname="bin/pulse")
tar.add(version, arcname="VERSION")
self.sign(archive)
self.operation = "prepare-installer"
(self.root / "installer").write_text(INSTALLER)
self.sign(self.root / "installer")
self.operation = "prepare-transport"
for name in ("curl", "mktemp"):
path = self.root / "tools" / name
path.write_text(TRANSPORT)
path.chmod(0o755)
# Establish the baseline using real source staging and unit/helper writers.
self.operation = "baseline-account-admission"
self.owned_user = self.run("id", "-u", "pulse", ok=(0, 1)).returncode == 1
self.owned_unit = True # Any partial writer output is ours to clean up.
self.operation = "baseline-install"
self.shell('source "$PULSE_NATIVE_ROOT/installer-definitions"; '
'PINNED_RELEASE_SSH_PUBLIC_KEY=$(cat "$PULSE_NATIVE_ROOT/signer/key.pub"); '
'create_user; setup_directories; '
'install_pulse_archive "$PULSE_NATIVE_ROOT/archives/pulse-v0.0.1-linux-amd64.tar.gz" v0.0.1; '
'install_systemd_service; setup_update_command')
self.operation = "baseline-configure"
data = self.root / "data"
(data / ".env").write_text("PULSE_TELEMETRY=false\nBIND_ADDRESS=127.0.0.1\n")
(data / "system.json").write_text('{"autoUpdateEnabled":false,"updateChannel":"stable"}\n')
(data / "native-persistence-sentinel").write_text("original persistent data\n")
self.operation = "baseline-start"
self.run("systemctl", "start", self.unit.name)
self.operation = "baseline-readback"
self.assert_runtime(TAGS[0], "active")
time.sleep(2) # Let asynchronous initial persistence settle once.
self.operation = "baseline-persistence"
files = (".env", "system.json", ".encryption.key", "native-persistence-sentinel")
self.baseline_data = {p: digest(data / p) for p in files}
self.baseline_mode = (self.root / "install/bin/pulse").stat().st_mode & 0o777
def create_signer(self):
# The directory is exclusive and private to this attempt. Establish
# the generated key's mode explicitly before signing: the hosted
# receipt observed 0644 and OpenSSH refused the first archive. Never
# repair/adopt an existing key or relax the real verifier's checks.
key = self.root / "signer/key"
if key.exists() or key.is_symlink():
raise ValueError("fixture signer already exists")
self.signer_command("generate-ephemeral-key", "-q", "-t", "ed25519", "-N", "", "-f", str(key))
info = key.lstat()
if not stat.S_ISREG(info.st_mode) or info.st_uid != os.geteuid():
raise ValueError("fixture signer is not an owned regular file")
key.chmod(0o600)
if stat.S_IMODE(key.lstat().st_mode) != 0o600:
raise ValueError("fixture signer is not owner-only")
def signer_command(self, operation, *arguments):
"""Retain non-secret facts about a failed, fixed ephemeral-signer step.
Do not publish command output or key bytes. In particular, an exit and
digest alone cannot distinguish generation from signing, or establish
why a hosted OpenSSH invocation failed before the first native phase.
Diagnostic labels below mean observed text, not a causal diagnosis.
"""
self.operation = operation
try:
return self.run("ssh-keygen", *arguments)
except RuntimeError:
failure = self.receipt.get("command_failure")
if failure is None:
# A confined control may stop before command execution. Do
# not manufacture an executed-command observation for it.
raise
failure["operation"] = operation
try:
info = (self.root / "signer/key").lstat()
failure["signer_key_metadata"] = {
"present": True, "regular_file": stat.S_ISREG(info.st_mode),
"mode": format(stat.S_IMODE(info.st_mode), "04o"),
"owned_by_executor": info.st_uid == os.geteuid(),
}
except FileNotFoundError:
failure["signer_key_metadata"] = {"present": False}
except OSError:
failure["signer_key_metadata"] = {"observation": "unavailable"}
output = getattr(self, "_last_output", b"")
# A timeout has no completed output. Never label an earlier
# command's output as this command's diagnostic.
if failure.get("output_sha256") == hashlib.sha256(output).hexdigest():
markers = {
"private-key-permissions": (b"UNPROTECTED PRIVATE KEY FILE", b"bad permissions"),
"key-load-failure": (b"Load key", b"Couldn't load public key"),
"missing-file": (b"No such file or directory",),
"unsupported-operation": (b"Unsupported operation", b"unknown option"),
}
failure["observed_diagnostics"] = [
label for label, texts in markers.items() if any(text in output for text in texts)
]
raise
def sign(self, path):
name = Path(path).name
operation = {"installer": "sign-installer",
**{f"pulse-{tag}-linux-amd64.tar.gz": "sign-archive-" + tag
for tag in TAGS}}.get(name, "sign-confined-fixture")
self.signer_command(operation, "-Y", "sign", "-f", str(self.root / "signer/key"),
"-n", "pulse-install", str(path))
Path(str(path) + ".sig").rename(str(path) + ".sshsig")
def assert_runtime(self, tag, active):
binary = self.root / "install/bin/pulse"
expected_hash = self.receipt["binaries"][tag]
if digest(binary) != expected_hash:
raise RuntimeError("installed executable differs from authenticated fixture")
if (self.root / "install/VERSION").read_text() != tag + "\n":
raise RuntimeError("version sidecar differs")
state = self.state()
if state["ActiveState"] != active:
raise RuntimeError("prior activity was not restored")
if active == "active":
deadline = time.monotonic() + 30
while True:
try:
observations = {}
for endpoint in ("health", "version"):
with urllib.request.urlopen(f"http://127.0.0.1:{PORT}/api/{endpoint}", timeout=2) as response:
body = response.read(16385)
if len(body) > 16384:
raise RuntimeError("native API observation too large")
observations[endpoint] = json.loads(body)
if observations["version"].get("version", "").lstrip("v") != tag.lstrip("v"):
raise RuntimeError("served version differs")
if observations["health"].get("status") != "healthy":
raise RuntimeError("native application is not healthy")
break
except urllib.error.HTTPError:
# Do not reinterpret refusal as a restart gap or retry it.
raise RuntimeError("native API request refused") from None
except OSError:
if time.monotonic() >= deadline:
raise RuntimeError("native application readiness deadline") from None
time.sleep(0.25)
# Type=simple start can return before exec. Bind the observed
# process after application readiness, not to that launch race.
state = self.state()
if (state["LoadState"] != "loaded" or state["ActiveState"] != "active"
or not re.fullmatch(r"[1-9][0-9]*", state["MainPID"])):
raise RuntimeError("served application has no active managed process")
if digest(Path("/proc") / state["MainPID"] / "exe") != expected_hash:
raise RuntimeError("running executable differs from authenticated fixture")
# A process or destination can change during the byte read. Do not
# accept a stale PID observation, or two equally wrong executables,
# as independent proof of the source transaction's final state.
observed = self.state()
if observed != state:
raise RuntimeError("managed process state changed during executable readback")
if digest(binary) != expected_hash:
raise RuntimeError("installed executable changed during runtime readback")
state = observed
elif state["MainPID"] != "0":
raise RuntimeError("inactive fixture retains a managed process")
return state
def phase(self, name, tag, expected_exit=0, active="active", fault="", retain=False, manual=False):
self.operation = name
before_state = self.state()
self.env.update(PULSE_NATIVE_TARGET=tag, PULSE_NATIVE_FAULT=fault)
result = self.run("/bin/bash", "-c", '"$PULSE_UPDATE_HELPER_PATH" --version "$PULSE_NATIVE_TARGET"'
if manual else UPDATER, ok=(expected_exit,))
(self.root / (name + ".log")).write_bytes(result.stdout)
expected_tag = TAGS[1] if retain else (TAGS[0] if expected_exit else tag)
state = self.assert_runtime(expected_tag, active)
if state["LoadState"] != before_state["LoadState"] or state["UnitFileState"] != before_state["UnitFileState"]:
raise RuntimeError("unit mask/enablement changed")
if {p: digest(self.root / "data" / p) for p in self.baseline_data} != self.baseline_data:
raise RuntimeError("persistent state changed")
backups = list((self.root / "tmp").glob("pulse-backup.*"))
if len(backups) != (1 if retain else 0):
raise RuntimeError("rollback backup disposition differs")
if retain and digest(backups[0] / "pulse-bin") != self.receipt["binaries"][TAGS[0]]:
raise RuntimeError("retained rollback backup differs from baseline")
if not retain and list((self.root / "install/bin").glob("*.rollback.*")):
raise RuntimeError("rollback staging leaked")
if digest(self.root / "install/bin/pulse") == self.receipt["binaries"][TAGS[0]]:
if (self.root / "install/bin/pulse").stat().st_mode & 0o777 != self.baseline_mode:
raise RuntimeError("baseline executable mode not restored")
self.receipt["phases"].append({"name": name, "exit": result.returncode, "passed": True,
"installed_tag": expected_tag, "state": state,
"persistent_hashes_unchanged": True, "backup_retained": retain,
"output_sha256": hashlib.sha256(result.stdout).hexdigest()})
print("NATIVE_UPDATER_PHASE=" + name, flush=True)
def exercise(self):
self.phase("forward", TAGS[1])
self.phase("manual-reverse", TAGS[0], manual=True)
self.phase("failed-active", TAGS[1], 1, fault="installer")
self.operation = "prepare-inactive"
self.run("systemctl", "stop", self.unit.name)
self.phase("failed-inactive", TAGS[1], 1, active="inactive", fault="installer")
# A real /dev/null mask, not a manager double returning 'masked'.
self.operation = "prepare-masked"
saved_unit = self.unit.read_bytes()
self.unit.unlink()
self.run("systemctl", "mask", self.unit.name)
self.run("systemctl", "daemon-reload")
self.phase("failed-masked", TAGS[1], 1, active="inactive", fault="installer")
self.operation = "restore-unmasked"
self.run("systemctl", "unmask", self.unit.name)
self.unit.write_bytes(saved_unit)
self.run("systemctl", "daemon-reload")
self.run("systemctl", "start", self.unit.name)
self.operation = "prepare-bad-signature"
installer = self.root / "installer"
original = installer.read_bytes()
calls = (self.root / "installer-calls").read_bytes()
installer.write_bytes(original + b"# tampered after signing\n")
try:
self.phase("bad-signature", TAGS[1], 1)
if (self.root / "installer-calls").read_bytes() != calls:
raise RuntimeError("invalid signature reached installer")
finally:
installer.write_bytes(original)
# Native stop/byte replacement precede a deliberate failed restore copy.
self.phase("incomplete-restore", TAGS[1], 1, active="inactive", fault="restore", retain=True)
def cleanup(self):
self.operation = "cleanup"
if self.owned_unit:
state = self.state()
if state["LoadState"] != "not-found":
self.run("systemctl", "stop", self.unit.name)
state = self.state()
if state["ActiveState"] != "inactive" or state["MainPID"] != "0":
raise RuntimeError("fixture service did not stop")
# A failed writer may have left a file not yet known to the manager.
self.unit.unlink(missing_ok=True)
self.run("systemctl", "daemon-reload")
self.run("systemctl", "reset-failed", self.unit.name, ok=(0, 1))
if self.state()["LoadState"] != "not-found" or self.state()["MainPID"] != "0":
raise RuntimeError("fixture unit/process remained")
if self.owned_user:
if self.run("id", "-u", "pulse", ok=(0, 1)).returncode == 0:
self.run("userdel", "pulse")
if self.run("id", "-u", "pulse", ok=(0, 1)).returncode != 1:
raise RuntimeError("owned fixture account remained")
if self.owned_root:
# Delete only this invocation's owned signer and scratch directories.
shutil.rmtree(self.root)
self.receipt["cleanup"] = {"complete": True, "owned_unit_removed": self.owned_unit,
"owned_account_removed": self.owned_user,
"owned_fixture_removed": self.owned_root,
"fixture_files_absent": not self.root.exists()}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--before", required=True)
parser.add_argument("--after", required=True)
parser.add_argument("--receipt", required=True)
args = parser.parse_args()
source = os.environ.get("EXPECTED_SOURCE_SHA", "")
hosted_admission(os.environ, os.geteuid(), source)
runner_temp = Path(os.environ["RUNNER_TEMP"]).resolve()
receipt = Path(args.receipt).resolve()
if receipt.parent != runner_temp or receipt.exists():
raise ValueError("receipt must be a new direct runner-temp file")
# /home is hidden by the real unit's ProtectHome; /tmp and /var/tmp are
# private under PrivateTmp. Preserve both protections, use a fixed /opt lab.
root = Path("/opt/pulse-source-native")
lab = NativeLab(root, Path(__file__).resolve().parents[2], args.before, args.after, source)
failed = False
try:
lab.prepare()
lab.exercise()
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error:
failed = True
lab.receipt["failure"] = {"type": type(error).__name__, "message": str(error),
"operation": lab.operation}
finally:
# Cleanup runs its own commands. Preserve the original transaction
# observation before any later failure can replace it or be mistaken
# for that transaction. A logical assertion has no command observation.
transaction_failure = lab.receipt.pop("command_failure", None)
try:
lab.cleanup()
except (OSError, RuntimeError, subprocess.SubprocessError) as error:
failed = True
lab.receipt["cleanup"] = {"complete": False, "error_type": type(error).__name__}
cleanup_failure = lab.receipt.pop("command_failure", None)
if cleanup_failure is not None:
lab.receipt["cleanup"]["command_failure"] = cleanup_failure
if transaction_failure is not None:
lab.receipt["command_failure"] = transaction_failure
receipt.write_text(json.dumps(lab.receipt, indent=2) + "\n")
if failed or len(lab.receipt["phases"]) != 7:
raise SystemExit("Native updater proof incomplete; inspect the bounded receipt.")
print("NATIVE_UPDATER_COMPLETE=" + source)
if __name__ == "__main__":
main()