From e1cd6843e9d9aee5b864c64cf285fc048b504bc7 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 09:16:19 +0100 Subject: [PATCH] Preserve native updater failures across cleanup Keep the original failed transaction command distinct from any later cleanup command. Retain fixed operation labels and require portable controls for command, timeout, assertion, cleanup-only and successful receipts, without changing update transactions or disclosing command output. Change-source: pulse-maintainer --- .../subsystems/deployment-installability.md | 17 ++++ scripts/installtests/auto_update_systemd.py | 36 +++++++- .../installtests/pulse_auto_update_test.go | 4 + scripts/tests/test_auto_update_systemd.py | 89 +++++++++++++++++++ 4 files changed, 144 insertions(+), 2 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 2826ba8d24..c59dedcfed 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -369,6 +369,23 @@ cleanup removes only objects created by this attempt. Compiler and root-command failures stay gating under the existing job's deadline; a seven-day closed receipt contains source/binary/output hashes, all phases and cleanup, never signer keys, persistent contents or raw application/installer logs. + +Failed commands and assertions name their fixed preparation or transaction +operation. The original transaction's `command_failure` is preserved before +cleanup runs, and a cleanup command failure is retained separately under +`cleanup.command_failure`. A successful cleanup must not label the earlier +transaction failure as its own; an assertion or cleanup-only failure must not +invent an executed transaction command. Timeout facts remain distinct from a +later numeric cleanup exit, with no stale output hash attached. Neither these +labels nor completed owned cleanup diagnose why an update failed or accept a +native phase. `test_primary_command_survives_successful_or_failed_cleanup`, +`test_primary_timeout_is_not_replaced_by_cleanup_exit`, +`test_assertion_and_cleanup_only_errors_do_not_invent_primary_command` and +`test_successful_receipt_retains_no_failure_record` drive the actual receipt +producer with real confined failing/timeout children and no native manager; +the registered Go harness requires every portable verdict. Raw command output, +private fixture contents, argv and paths remain outside these observations. + The private fixture disables telemetry and binds the existing service to loopback; it exposes no test server on the hosted VM's external interface. Before compiling the native pair, CI requires the exact checkout head and no diff --git a/scripts/installtests/auto_update_systemd.py b/scripts/installtests/auto_update_systemd.py index 51e853eb1c..424d440d10 100644 --- a/scripts/installtests/auto_update_systemd.py +++ b/scripts/installtests/auto_update_systemd.py @@ -146,6 +146,7 @@ class NativeLab: self.owned_unit = False self.owned_root = False self.owned_user = False + self.operation = "prepare-admission" def run(self, *args, timeout=120, ok=(0,)): process = subprocess.Popen(args, env=self.env, stdin=subprocess.DEVNULL, @@ -162,7 +163,8 @@ class NativeLab: process.communicate(timeout=2) self.receipt["command_failure"] = {"command": Path(args[0]).name, "deadline_exceeded": True, - "observed_exit": process.returncode} + "observed_exit": process.returncode, + "operation": self.operation} raise RuntimeError("native command deadline exceeded") from None # Private in-memory diagnostic for confined controls, never a receipt field. self._last_output = output @@ -170,6 +172,7 @@ class NativeLab: if result.returncode not in ok: self.receipt["command_failure"] = {"command": Path(args[0]).name, "observed_exit": result.returncode, + "operation": self.operation, "output_bytes": len(output), "output_sha256": hashlib.sha256(output).hexdigest()} raise RuntimeError(f"native command failed ({result.returncode}): {Path(args[0]).name}") @@ -193,6 +196,7 @@ class NativeLab: return pairs def prepare(self): + self.operation = "prepare-admission" for path in (self.unit, Path("/run/systemd/system") / self.unit.name): if path.exists() or path.is_symlink(): raise ValueError("native fixture unit already exists") @@ -205,6 +209,7 @@ class NativeLab: raise ValueError("native fixture unit is already known to systemd") for name in ("tools", "tmp", "archives", "signer"): (self.root / name).mkdir(mode=0o700 if name == "signer" else 0o755) + self.operation = "prepare-source" source = (self.repo / "install.sh").read_text() (self.root / "installer-definitions").write_text(installer_definitions(source)) self.receipt["sources"] = {p: digest(self.repo / p) for p in @@ -214,10 +219,12 @@ class NativeLab: if len(set(self.receipt["binaries"].values())) != 2: raise ValueError("same executable bytes prove no replacement") for tag, binary in self.binaries.items(): + self.operation = "verify-binary-" + tag if self.run(str(binary), "--version").stdout.decode().splitlines()[0] != "Pulse " + tag: raise ValueError("synthetic binary identity mismatch") self.create_signer() for tag, binary in self.binaries.items(): + self.operation = "prepare-archive-" + tag version = self.root / "VERSION" version.write_text(tag + "\n") archive = self.root / "archives" / f"pulse-{tag}-linux-amd64.tar.gz" @@ -225,27 +232,35 @@ class NativeLab: tar.add(binary, arcname="bin/pulse") tar.add(version, arcname="VERSION") self.sign(archive) + self.operation = "prepare-installer" (self.root / "installer").write_text(INSTALLER) self.sign(self.root / "installer") + self.operation = "prepare-transport" for name in ("curl", "mktemp"): path = self.root / "tools" / name path.write_text(TRANSPORT) path.chmod(0o755) # Establish the baseline using real source staging and unit/helper writers. + self.operation = "baseline-account-admission" self.owned_user = self.run("id", "-u", "pulse", ok=(0, 1)).returncode == 1 self.owned_unit = True # Any partial writer output is ours to clean up. + self.operation = "baseline-install" self.shell('source "$PULSE_NATIVE_ROOT/installer-definitions"; ' 'PINNED_RELEASE_SSH_PUBLIC_KEY=$(cat "$PULSE_NATIVE_ROOT/signer/key.pub"); ' 'create_user; setup_directories; ' 'install_pulse_archive "$PULSE_NATIVE_ROOT/archives/pulse-v0.0.1-linux-amd64.tar.gz" v0.0.1; ' 'install_systemd_service; setup_update_command') + self.operation = "baseline-configure" data = self.root / "data" (data / ".env").write_text("PULSE_TELEMETRY=false\nBIND_ADDRESS=127.0.0.1\n") (data / "system.json").write_text('{"autoUpdateEnabled":false,"updateChannel":"stable"}\n') (data / "native-persistence-sentinel").write_text("original persistent data\n") + self.operation = "baseline-start" self.run("systemctl", "start", self.unit.name) + self.operation = "baseline-readback" self.assert_runtime(TAGS[0], "active") time.sleep(2) # Let asynchronous initial persistence settle once. + self.operation = "baseline-persistence" files = (".env", "system.json", ".encryption.key", "native-persistence-sentinel") self.baseline_data = {p: digest(data / p) for p in files} self.baseline_mode = (self.root / "install/bin/pulse").stat().st_mode & 0o777 @@ -274,6 +289,7 @@ class NativeLab: why a hosted OpenSSH invocation failed before the first native phase. Diagnostic labels below mean observed text, not a causal diagnosis. """ + self.operation = operation try: return self.run("ssh-keygen", *arguments) except RuntimeError: @@ -373,6 +389,7 @@ class NativeLab: return state def phase(self, name, tag, expected_exit=0, active="active", fault="", retain=False, manual=False): + self.operation = name before_state = self.state() self.env.update(PULSE_NATIVE_TARGET=tag, PULSE_NATIVE_FAULT=fault) result = self.run("/bin/bash", "-c", '"$PULSE_UPDATE_HELPER_PATH" --version "$PULSE_NATIVE_TARGET"' @@ -404,18 +421,22 @@ class NativeLab: self.phase("forward", TAGS[1]) self.phase("manual-reverse", TAGS[0], manual=True) self.phase("failed-active", TAGS[1], 1, fault="installer") + self.operation = "prepare-inactive" self.run("systemctl", "stop", self.unit.name) self.phase("failed-inactive", TAGS[1], 1, active="inactive", fault="installer") # A real /dev/null mask, not a manager double returning 'masked'. + self.operation = "prepare-masked" saved_unit = self.unit.read_bytes() self.unit.unlink() self.run("systemctl", "mask", self.unit.name) self.run("systemctl", "daemon-reload") self.phase("failed-masked", TAGS[1], 1, active="inactive", fault="installer") + self.operation = "restore-unmasked" self.run("systemctl", "unmask", self.unit.name) self.unit.write_bytes(saved_unit) self.run("systemctl", "daemon-reload") self.run("systemctl", "start", self.unit.name) + self.operation = "prepare-bad-signature" installer = self.root / "installer" original = installer.read_bytes() calls = (self.root / "installer-calls").read_bytes() @@ -430,6 +451,7 @@ class NativeLab: self.phase("incomplete-restore", TAGS[1], 1, active="inactive", fault="restore", retain=True) def cleanup(self): + self.operation = "cleanup" if self.owned_unit: state = self.state() if state["LoadState"] != "not-found": @@ -479,13 +501,23 @@ def main(): lab.exercise() except (OSError, ValueError, RuntimeError, subprocess.SubprocessError) as error: failed = True - lab.receipt["failure"] = {"type": type(error).__name__, "message": str(error)} + lab.receipt["failure"] = {"type": type(error).__name__, "message": str(error), + "operation": lab.operation} finally: + # Cleanup runs its own commands. Preserve the original transaction + # observation before any later failure can replace it or be mistaken + # for that transaction. A logical assertion has no command observation. + transaction_failure = lab.receipt.pop("command_failure", None) try: lab.cleanup() except (OSError, RuntimeError, subprocess.SubprocessError) as error: failed = True lab.receipt["cleanup"] = {"complete": False, "error_type": type(error).__name__} + cleanup_failure = lab.receipt.pop("command_failure", None) + if cleanup_failure is not None: + lab.receipt["cleanup"]["command_failure"] = cleanup_failure + if transaction_failure is not None: + lab.receipt["command_failure"] = transaction_failure receipt.write_text(json.dumps(lab.receipt, indent=2) + "\n") if failed or len(lab.receipt["phases"]) != 7: raise SystemExit("Native updater proof incomplete; inspect the bounded receipt.") diff --git a/scripts/installtests/pulse_auto_update_test.go b/scripts/installtests/pulse_auto_update_test.go index c9bc2ff6e3..edf05eb6b9 100644 --- a/scripts/installtests/pulse_auto_update_test.go +++ b/scripts/installtests/pulse_auto_update_test.go @@ -37,6 +37,10 @@ func TestAutoUpdateNativeSystemdHarnessContracts(t *testing.T) { "test_phase_cannot_accept_wrong_bytes_state_mask_persistence_or_backup", "test_managed_process_identity_follows_served_readiness_and_rejects_residuals", "test_runtime_readback_rejects_post_comparison_drift", + "test_primary_command_survives_successful_or_failed_cleanup", + "test_primary_timeout_is_not_replaced_by_cleanup_exit", + "test_assertion_and_cleanup_only_errors_do_not_invent_primary_command", + "test_successful_receipt_retains_no_failure_record", "test_connected_linux_control_is_required_without_macho_or_validator_bypass", } { if results[name] != "ok" { diff --git a/scripts/tests/test_auto_update_systemd.py b/scripts/tests/test_auto_update_systemd.py index 7dc8754030..b934388e40 100644 --- a/scripts/tests/test_auto_update_systemd.py +++ b/scripts/tests/test_auto_update_systemd.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 """Discriminating controls for the native CI recipe; no host manager/root use.""" +import contextlib import importlib.util import io import json @@ -299,6 +300,7 @@ class NativeContracts(unittest.TestCase): else: lab.phase("forward", "v0.0.2", active="inactive") self.assertTrue(lab.receipt["phases"][0]["passed"]) + self.assertEqual(lab.operation, "forward") def test_managed_process_identity_follows_served_readiness_and_rejects_residuals(self): for mutation in ("", "process-bytes", "stopped-after-readiness", "inactive-pid"): @@ -665,6 +667,93 @@ else: sys.exit(97) source.replace("runs-on: ubuntu-24.04", "runs-on: self-hosted")): with self.assertRaises(AssertionError): recipe(changed) + def run_receipt(self, primary, cleanup_fails): + # Execute the actual receipt producer, with real unprivileged failing + # children. Substitute only native admission and service/account work. + # These controls cannot provide a native transaction or cleanup proof. + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + receipt = root / "receipt.json" + lab = native.NativeLab(root, ROOT, root / "a", root / "b", "a" * 40) + owned_cleanup = lab.cleanup + def prepare(): + lab.operation = "baseline-install" + if primary == "command": + lab.run("/bin/bash", "-c", "printf 'private-transaction-fixture'; exit 23") + if primary == "timeout": + lab.run(sys.executable, "-c", "import time; time.sleep(60)", timeout=0.05) + if primary == "assertion": + raise RuntimeError("confined persistence assertion") + def exercise(): + lab.receipt["phases"] = [{"name": "confined-control"}] * 7 + def cleanup(): + lab.operation = "cleanup" + if cleanup_fails: + lab.run(sys.executable, "-c", "print('private-cleanup-fixture'); raise SystemExit(31)") + owned_cleanup() # Owns nothing; touches no manager. + argv = ["native-receipt-control", "--before", "unused-before", "--after", "unused-after", + "--receipt", str(receipt)] + with patch.dict(os.environ, {"EXPECTED_SOURCE_SHA": "a" * 40, "RUNNER_TEMP": str(root)}), \ + patch.object(sys, "argv", argv), patch.object(native, "hosted_admission"), \ + patch.object(native, "NativeLab", return_value=lab), \ + patch.object(lab, "prepare", side_effect=prepare), \ + patch.object(lab, "exercise", side_effect=exercise), \ + patch.object(lab, "cleanup", side_effect=cleanup), \ + contextlib.redirect_stdout(io.StringIO()): + if primary or cleanup_fails: + with self.assertRaises(SystemExit): native.main() + else: + native.main() + result = json.loads(receipt.read_text()) + for private in ("private-transaction-fixture", "private-cleanup-fixture", str(root)): + self.assertNotIn(private, json.dumps(result)) + return result + + def test_primary_command_survives_successful_or_failed_cleanup(self): + for cleanup_fails in (False, True): + with self.subTest(cleanup_fails=cleanup_fails): + result = self.run_receipt("command", cleanup_fails) + self.assertEqual(result["command_failure"]["observed_exit"], 23) + self.assertEqual(result["command_failure"]["command"], "bash") + self.assertEqual(result["command_failure"]["operation"], "baseline-install") + self.assertEqual(result["failure"]["operation"], "baseline-install") + self.assertEqual(result["phases"], []) + self.assertEqual(result["cleanup"]["complete"], not cleanup_fails) + if cleanup_fails: + self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31) + self.assertEqual(result["cleanup"]["command_failure"]["operation"], "cleanup") + else: + self.assertNotIn("command_failure", result["cleanup"]) + + def test_primary_timeout_is_not_replaced_by_cleanup_exit(self): + result = self.run_receipt("timeout", True) + self.assertTrue(result["command_failure"]["deadline_exceeded"]) + self.assertLess(result["command_failure"]["observed_exit"], 0) + self.assertEqual(result["command_failure"]["operation"], "baseline-install") + self.assertNotIn("output_sha256", result["command_failure"]) + self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31) + self.assertEqual(result["phases"], []) + + def test_assertion_and_cleanup_only_errors_do_not_invent_primary_command(self): + for primary in ("assertion", ""): + with self.subTest(primary=primary): + result = self.run_receipt(primary, True) + self.assertNotIn("command_failure", result) + self.assertEqual(result["cleanup"]["command_failure"]["observed_exit"], 31) + self.assertFalse(result["cleanup"]["complete"]) + if primary: + self.assertEqual(result["failure"]["operation"], "baseline-install") + self.assertEqual(result["phases"], []) + else: + self.assertNotIn("failure", result) + + def test_successful_receipt_retains_no_failure_record(self): + result = self.run_receipt("", False) + self.assertNotIn("failure", result) + self.assertNotIn("command_failure", result) + self.assertNotIn("command_failure", result["cleanup"]) + self.assertTrue(result["cleanup"]["complete"]) + if __name__ == "__main__": unittest.main()