mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-25 21:20:00 +00:00
Preserve native updater failures across cleanup
Keep the original failed transaction command distinct from any later cleanup command. Retain fixed operation labels and require portable controls for command, timeout, assertion, cleanup-only and successful receipts, without changing update transactions or disclosing command output. Change-source: pulse-maintainer
This commit is contained in:
pulse-triage[bot]
@@ -369,6 +369,23 @@ cleanup removes only objects created by this attempt. Compiler and root-command
|
||||
failures stay gating under the existing job's deadline; a seven-day closed receipt
|
||||
contains source/binary/output hashes, all phases and cleanup, never signer keys,
|
||||
persistent contents or raw application/installer logs.
|
||||
|
||||
Failed commands and assertions name their fixed preparation or transaction
|
||||
operation. The original transaction's `command_failure` is preserved before
|
||||
cleanup runs, and a cleanup command failure is retained separately under
|
||||
`cleanup.command_failure`. A successful cleanup must not label the earlier
|
||||
transaction failure as its own; an assertion or cleanup-only failure must not
|
||||
invent an executed transaction command. Timeout facts remain distinct from a
|
||||
later numeric cleanup exit, with no stale output hash attached. Neither these
|
||||
labels nor completed owned cleanup diagnose why an update failed or accept a
|
||||
native phase. `test_primary_command_survives_successful_or_failed_cleanup`,
|
||||
`test_primary_timeout_is_not_replaced_by_cleanup_exit`,
|
||||
`test_assertion_and_cleanup_only_errors_do_not_invent_primary_command` and
|
||||
`test_successful_receipt_retains_no_failure_record` drive the actual receipt
|
||||
producer with real confined failing/timeout children and no native manager;
|
||||
the registered Go harness requires every portable verdict. Raw command output,
|
||||
private fixture contents, argv and paths remain outside these observations.
|
||||
|
||||
The private fixture disables telemetry and binds the existing service to loopback;
|
||||
it exposes no test server on the hosted VM's external interface.
|
||||
Before compiling the native pair, CI requires the exact checkout head and no
|
||||
|
||||
@@ -146,6 +146,7 @@ class NativeLab:
|
||||
self.owned_unit = False
|
||||
self.owned_root = False
|
||||
self.owned_user = False
|
||||
self.operation = "prepare-admission"
|
||||
|
||||
def run(self, *args, timeout=120, ok=(0,)):
|
||||
process = subprocess.Popen(args, env=self.env, stdin=subprocess.DEVNULL,
|
||||
@@ -162,7 +163,8 @@ class NativeLab:
|
||||
process.communicate(timeout=2)
|
||||
self.receipt["command_failure"] = {"command": Path(args[0]).name,
|
||||
"deadline_exceeded": True,
|
||||
"observed_exit": process.returncode}
|
||||
"observed_exit": process.returncode,
|
||||
"operation": self.operation}
|
||||
raise RuntimeError("native command deadline exceeded") from None
|
||||
# Private in-memory diagnostic for confined controls, never a receipt field.
|
||||
self._last_output = output
|
||||
@@ -170,6 +172,7 @@ class NativeLab:
|
||||
if result.returncode not in ok:
|
||||
self.receipt["command_failure"] = {"command": Path(args[0]).name,
|
||||
"observed_exit": result.returncode,
|
||||
"operation": self.operation,
|
||||
"output_bytes": len(output),
|
||||
"output_sha256": hashlib.sha256(output).hexdigest()}
|
||||
raise RuntimeError(f"native command failed ({result.returncode}): {Path(args[0]).name}")
|
||||
@@ -193,6 +196,7 @@ class NativeLab:
|
||||
return pairs
|
||||
|
||||
def prepare(self):
|
||||
self.operation = "prepare-admission"
|
||||
for path in (self.unit, Path("/run/systemd/system") / self.unit.name):
|
||||
if path.exists() or path.is_symlink():
|
||||
raise ValueError("native fixture unit already exists")
|
||||
@@ -205,6 +209,7 @@ class NativeLab:
|
||||
raise ValueError("native fixture unit is already known to systemd")
|
||||
for name in ("tools", "tmp", "archives", "signer"):
|
||||
(self.root / name).mkdir(mode=0o700 if name == "signer" else 0o755)
|
||||
self.operation = "prepare-source"
|
||||
source = (self.repo / "install.sh").read_text()
|
||||
(self.root / "installer-definitions").write_text(installer_definitions(source))
|
||||
self.receipt["sources"] = {p: digest(self.repo / p) for p in
|
||||
@@ -214,10 +219,12 @@ class NativeLab:
|
||||
if len(set(self.receipt["binaries"].values())) != 2:
|
||||
raise ValueError("same executable bytes prove no replacement")
|
||||
for tag, binary in self.binaries.items():
|
||||
self.operation = "verify-binary-" + tag
|
||||
if self.run(str(binary), "--version").stdout.decode().splitlines()[0] != "Pulse " + tag:
|
||||
raise ValueError("synthetic binary identity mismatch")
|
||||
self.create_signer()
|
||||
for tag, binary in self.binaries.items():
|
||||
self.operation = "prepare-archive-" + tag
|
||||
version = self.root / "VERSION"
|
||||
version.write_text(tag + "\n")
|
||||
archive = self.root / "archives" / f"pulse-{tag}-linux-amd64.tar.gz"
|
||||
@@ -225,27 +232,35 @@ class NativeLab:
|
||||
tar.add(binary, arcname="bin/pulse")
|
||||
tar.add(version, arcname="VERSION")
|
||||
self.sign(archive)
|
||||
self.operation = "prepare-installer"
|
||||
(self.root / "installer").write_text(INSTALLER)
|
||||
self.sign(self.root / "installer")
|
||||
self.operation = "prepare-transport"
|
||||
for name in ("curl", "mktemp"):
|
||||
path = self.root / "tools" / name
|
||||
path.write_text(TRANSPORT)
|
||||
path.chmod(0o755)
|
||||
# Establish the baseline using real source staging and unit/helper writers.
|
||||
self.operation = "baseline-account-admission"
|
||||
self.owned_user = self.run("id", "-u", "pulse", ok=(0, 1)).returncode == 1
|
||||
self.owned_unit = True # Any partial writer output is ours to clean up.
|
||||
self.operation = "baseline-install"
|
||||
self.shell('source "$PULSE_NATIVE_ROOT/installer-definitions"; '
|
||||
'PINNED_RELEASE_SSH_PUBLIC_KEY=$(cat "$PULSE_NATIVE_ROOT/signer/key.pub"); '
|
||||
'create_user; setup_directories; '
|
||||
'install_pulse_archive "$PULSE_NATIVE_ROOT/archives/pulse-v0.0.1-linux-amd64.tar.gz" v0.0.1; '
|
||||
'install_systemd_service; setup_update_command')
|
||||
self.operation = "baseline-configure"
|
||||
data = self.root / "data"
|
||||
(data / ".env").write_text("PULSE_TELEMETRY=false\nBIND_ADDRESS=127.0.0.1\n")
|
||||
(data / "system.json").write_text('{"autoUpdateEnabled":false,"updateChannel":"stable"}\n')
|
||||
(data / "native-persistence-sentinel").write_text("original persistent data\n")
|
||||
self.operation = "baseline-start"
|
||||
self.run("systemctl", "start", self.unit.name)
|
||||
self.operation = "baseline-readback"
|
||||
self.assert_runtime(TAGS[0], "active")
|
||||
time.sleep(2) # Let asynchronous initial persistence settle once.
|
||||
self.operation = "baseline-persistence"
|
||||
files = (".env", "system.json", ".encryption.key", "native-persistence-sentinel")
|
||||
self.baseline_data = {p: digest(data / p) for p in files}
|
||||
self.baseline_mode = (self.root / "install/bin/pulse").stat().st_mode & 0o777
|
||||
@@ -274,6 +289,7 @@ class NativeLab:
|
||||
why a hosted OpenSSH invocation failed before the first native phase.
|
||||
Diagnostic labels below mean observed text, not a causal diagnosis.
|
||||
"""
|
||||
self.operation = operation
|
||||
try:
|
||||
return self.run("ssh-keygen", *arguments)
|
||||
except RuntimeError:
|
||||
@@ -373,6 +389,7 @@ class NativeLab:
|
||||
return state
|
||||
|
||||
def phase(self, name, tag, expected_exit=0, active="active", fault="", retain=False, manual=False):
|
||||
self.operation = name
|
||||
before_state = self.state()
|
||||
self.env.update(PULSE_NATIVE_TARGET=tag, PULSE_NATIVE_FAULT=fault)
|
||||
result = self.run("/bin/bash", "-c", '"$PULSE_UPDATE_HELPER_PATH" --version "$PULSE_NATIVE_TARGET"'
|
||||
@@ -404,18 +421,22 @@ class NativeLab:
|
||||
self.phase("forward", TAGS[1])
|
||||
self.phase("manual-reverse", TAGS[0], manual=True)
|
||||
self.phase("failed-active", TAGS[1], 1, fault="installer")
|
||||
self.operation = "prepare-inactive"
|
||||
self.run("systemctl", "stop", self.unit.name)
|
||||
self.phase("failed-inactive", TAGS[1], 1, active="inactive", fault="installer")
|
||||
# A real /dev/null mask, not a manager double returning 'masked'.
|
||||
self.operation = "prepare-masked"
|
||||
saved_unit = self.unit.read_bytes()
|
||||
self.unit.unlink()
|
||||
self.run("systemctl", "mask", self.unit.name)
|
||||
self.run("systemctl", "daemon-reload")
|
||||
self.phase("failed-masked", TAGS[1], 1, active="inactive", fault="installer")
|
||||
self.operation = "restore-unmasked"
|
||||
self.run("systemctl", "unmask", self.unit.name)
|
||||
self.unit.write_bytes(saved_unit)
|
||||
self.run("systemctl", "daemon-reload")
|
||||
self.run("systemctl", "start", self.unit.name)
|
||||
self.operation = "prepare-bad-signature"
|
||||
installer = self.root / "installer"
|
||||
original = installer.read_bytes()
|
||||
calls = (self.root / "installer-calls").read_bytes()
|
||||
@@ -430,6 +451,7 @@ class NativeLab:
|
||||
self.phase("incomplete-restore", TAGS[1], 1, active="inactive", fault="restore", retain=True)
|
||||
|
||||
def cleanup(self):
|
||||
self.operation = "cleanup"
|
||||
if self.owned_unit:
|
||||
state = self.state()
|
||||
if state["LoadState"] != "not-found":
|
||||
@@ -479,13 +501,23 @@ def main():
|
||||
lab.exercise()
|
||||
except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error:
|
||||
failed = True
|
||||
lab.receipt["failure"] = {"type": type(error).__name__, "message": str(error)}
|
||||
lab.receipt["failure"] = {"type": type(error).__name__, "message": str(error),
|
||||
"operation": lab.operation}
|
||||
finally:
|
||||
# Cleanup runs its own commands. Preserve the original transaction
|
||||
# observation before any later failure can replace it or be mistaken
|
||||
# for that transaction. A logical assertion has no command observation.
|
||||
transaction_failure = lab.receipt.pop("command_failure", None)
|
||||
try:
|
||||
lab.cleanup()
|
||||
except (OSError, RuntimeError, subprocess.SubprocessError) as error:
|
||||
failed = True
|
||||
lab.receipt["cleanup"] = {"complete": False, "error_type": type(error).__name__}
|
||||
cleanup_failure = lab.receipt.pop("command_failure", None)
|
||||
if cleanup_failure is not None:
|
||||
lab.receipt["cleanup"]["command_failure"] = cleanup_failure
|
||||
if transaction_failure is not None:
|
||||
lab.receipt["command_failure"] = transaction_failure
|
||||
receipt.write_text(json.dumps(lab.receipt, indent=2) + "\n")
|
||||
if failed or len(lab.receipt["phases"]) != 7:
|
||||
raise SystemExit("Native updater proof incomplete; inspect the bounded receipt.")
|
||||
|
||||
@@ -37,6 +37,10 @@ func TestAutoUpdateNativeSystemdHarnessContracts(t *testing.T) {
|
||||
"test_phase_cannot_accept_wrong_bytes_state_mask_persistence_or_backup",
|
||||
"test_managed_process_identity_follows_served_readiness_and_rejects_residuals",
|
||||
"test_runtime_readback_rejects_post_comparison_drift",
|
||||
"test_primary_command_survives_successful_or_failed_cleanup",
|
||||
"test_primary_timeout_is_not_replaced_by_cleanup_exit",
|
||||
"test_assertion_and_cleanup_only_errors_do_not_invent_primary_command",
|
||||
"test_successful_receipt_retains_no_failure_record",
|
||||
"test_connected_linux_control_is_required_without_macho_or_validator_bypass",
|
||||
} {
|
||||
if results[name] != "ok" {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Discriminating controls for the native CI recipe; no host manager/root use."""
|
||||
import contextlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
@@ -299,6 +300,7 @@ class NativeContracts(unittest.TestCase):
|
||||
else:
|
||||
lab.phase("forward", "v0.0.2", active="inactive")
|
||||
self.assertTrue(lab.receipt["phases"][0]["passed"])
|
||||
self.assertEqual(lab.operation, "forward")
|
||||
|
||||
def test_managed_process_identity_follows_served_readiness_and_rejects_residuals(self):
|
||||
for mutation in ("", "process-bytes", "stopped-after-readiness", "inactive-pid"):
|
||||
@@ -665,6 +667,93 @@ else: sys.exit(97)
|
||||
source.replace("runs-on: ubuntu-24.04", "runs-on: self-hosted")):
|
||||
with self.assertRaises(AssertionError): recipe(changed)
|
||||
|
||||
def run_receipt(self, primary, cleanup_fails):
|
||||
# Execute the actual receipt producer, with real unprivileged failing
|
||||
# children. Substitute only native admission and service/account work.
|
||||
# These controls cannot provide a native transaction or cleanup proof.
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
receipt = root / "receipt.json"
|
||||
lab = native.NativeLab(root, ROOT, root / "a", root / "b", "a" * 40)
|
||||
owned_cleanup = lab.cleanup
|
||||
def prepare():
|
||||
lab.operation = "baseline-install"
|
||||
if primary == "command":
|
||||
lab.run("/bin/bash", "-c", "printf 'private-transaction-fixture'; exit 23")
|
||||
if primary == "timeout":
|
||||
lab.run(sys.executable, "-c", "import time; time.sleep(60)", timeout=0.05)
|
||||
if primary == "assertion":
|
||||
raise RuntimeError("confined persistence assertion")
|
||||
def exercise():
|
||||
lab.receipt["phases"] = [{"name": "confined-control"}] * 7
|
||||
def cleanup():
|
||||
lab.operation = "cleanup"
|
||||
if cleanup_fails:
|
||||
lab.run(sys.executable, "-c", "print('private-cleanup-fixture'); raise SystemExit(31)")
|
||||
owned_cleanup() # Owns nothing; touches no manager.
|
||||
argv = ["native-receipt-control", "--before", "unused-before", "--after", "unused-after",
|
||||
"--receipt", str(receipt)]
|
||||
with patch.dict(os.environ, {"EXPECTED_SOURCE_SHA": "a" * 40, "RUNNER_TEMP": str(root)}), \
|
||||
patch.object(sys, "argv", argv), patch.object(native, "hosted_admission"), \
|
||||
patch.object(native, "NativeLab", return_value=lab), \
|
||||
patch.object(lab, "prepare", side_effect=prepare), \
|
||||
patch.object(lab, "exercise", side_effect=exercise), \
|
||||
patch.object(lab, "cleanup", side_effect=cleanup), \
|
||||
contextlib.redirect_stdout(io.StringIO()):
|
||||
if primary or cleanup_fails:
|
||||
with self.assertRaises(SystemExit): native.main()
|
||||
else:
|
||||
native.main()
|
||||
result = json.loads(receipt.read_text())
|
||||
for private in ("private-transaction-fixture", "private-cleanup-fixture", str(root)):
|
||||
self.assertNotIn(private, json.dumps(result))
|
||||
return result
|
||||
|
||||
def test_primary_command_survives_successful_or_failed_cleanup(self):
|
||||
for cleanup_fails in (False, True):
|
||||
with self.subTest(cleanup_fails=cleanup_fails):
|
||||
result = self.run_receipt("command", cleanup_fails)
|
||||
self.assertEqual(result["command_failure"]["observed_exit"], 23)
|
||||
self.assertEqual(result["command_failure"]["command"], "bash")
|
||||
self.assertEqual(result["command_failure"]["operation"], "baseline-install")
|
||||
self.assertEqual(result["failure"]["operation"], "baseline-install")
|
||||
self.assertEqual(result["phases"], [])
|
||||
self.assertEqual(result["cleanup"]["complete"], not cleanup_fails)
|
||||
if cleanup_fails:
|
||||
self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31)
|
||||
self.assertEqual(result["cleanup"]["command_failure"]["operation"], "cleanup")
|
||||
else:
|
||||
self.assertNotIn("command_failure", result["cleanup"])
|
||||
|
||||
def test_primary_timeout_is_not_replaced_by_cleanup_exit(self):
|
||||
result = self.run_receipt("timeout", True)
|
||||
self.assertTrue(result["command_failure"]["deadline_exceeded"])
|
||||
self.assertLess(result["command_failure"]["observed_exit"], 0)
|
||||
self.assertEqual(result["command_failure"]["operation"], "baseline-install")
|
||||
self.assertNotIn("output_sha256", result["command_failure"])
|
||||
self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31)
|
||||
self.assertEqual(result["phases"], [])
|
||||
|
||||
def test_assertion_and_cleanup_only_errors_do_not_invent_primary_command(self):
|
||||
for primary in ("assertion", ""):
|
||||
with self.subTest(primary=primary):
|
||||
result = self.run_receipt(primary, True)
|
||||
self.assertNotIn("command_failure", result)
|
||||
self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31)
|
||||
self.assertFalse(result["cleanup"]["complete"])
|
||||
if primary:
|
||||
self.assertEqual(result["failure"]["operation"], "baseline-install")
|
||||
self.assertEqual(result["phases"], [])
|
||||
else:
|
||||
self.assertNotIn("failure", result)
|
||||
|
||||
def test_successful_receipt_retains_no_failure_record(self):
|
||||
result = self.run_receipt("", False)
|
||||
self.assertNotIn("failure", result)
|
||||
self.assertNotIn("command_failure", result)
|
||||
self.assertNotIn("command_failure", result["cleanup"])
|
||||
self.assertTrue(result["cleanup"]["complete"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user