From e836dfda13f3f4ccda22d929da754c5a22e1c361 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 2 Sep 2026 02:43:30 +0100 Subject: [PATCH 1/2] Fix rootful qualification readiness The disposable host reached multi-user.target in a degraded state because distro Podman housekeeping used overlay storage inside the outer container. Mask unused units, force VFS for explicit daemons, and fail closed on systemd unit failures so retained rootful evidence remains trustworthy. --- .../v6/internal/subsystems/agent-lifecycle.md | 6 +- .../subsystems/deployment-installability.md | 6 +- ...cure_runtime_rootful_qualification_test.go | 88 +++++++++++++++++-- ...ure_runtime_rootful_attestation_v1_test.py | 1 + ...re_runtime_rootful_source_manifest_v1.json | 1 + ...un-secure-runtime-rootful-qualification.sh | 35 ++++++-- scripts/secure-runtime-rootful-runtime.sh | 21 +++++ 7 files changed, 143 insertions(+), 15 deletions(-) create mode 100644 scripts/secure-runtime-rootful-runtime.sh diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 75cedde86..f51553c32 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -7399,7 +7399,11 @@ the product default. `scripts/run-secure-runtime-rootful-qualification.sh` is the destructive, explicit entrypoint for real rootful Docker and Podman proof on two distinct disposable Ubuntu/systemd hosts. It never mounts a host daemon socket and runs -the outer containers without a default route. The packet binds exact clean Go +the outer containers without a default route. The host image masks Docker and +Podman's distro-managed service, socket, auto-update, transient-cleanup, and +restart units before first boot; the packet starts only its explicit +runtime-specific daemon units and requires systemd's strict running state +before any scenario begins. The packet binds exact clean Go artifacts, every compiled installtests package input, the canonical remote-main commit, the immutable Ubuntu base-image digest, and governed source hashes, then records fresh install, legacy migration with authority reduction, diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index f62068c0e..09590b5a2 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -326,7 +326,11 @@ Live rootful-runtime proof is also separated from the schema-v7 systemd packet. The explicit `scripts/run-secure-runtime-rootful-qualification.sh` wrapper creates distinct network-isolated Ubuntu/systemd hosts for real rootful Docker and Podman and -never mounts a host daemon socket. Its standalone schema-v1 receipt binds the +never mounts a host daemon socket. The disposable image masks the distro's +Docker and Podman service, socket, auto-update, transient-cleanup, and restart +units before first boot, so strict systemd readiness cannot be confused with +expected nested-runtime housekeeping failures; each scenario starts only its +explicit runtime-specific daemon unit. Its standalone schema-v1 receipt binds the exact canonical remote-main commit, immutable Ubuntu base-image digest, qualification, collector, helper, installer, every compiled installtests package input, source manifest, canonical root-owned socket, summary inventory, diff --git a/scripts/installtests/secure_runtime_rootful_qualification_test.go b/scripts/installtests/secure_runtime_rootful_qualification_test.go index 831070965..18eefe31a 100644 --- a/scripts/installtests/secure_runtime_rootful_qualification_test.go +++ b/scripts/installtests/secure_runtime_rootful_qualification_test.go @@ -97,6 +97,7 @@ type rootfulQualDaemon struct { unit string socket string dataRoot string + runRoot string fixture string } @@ -454,9 +455,9 @@ func rootfulQualRequireDisposableHost(t *testing.T, runtimeKind, receiptPath str func rootfulQualDaemonFor(runtimeKind string) rootfulQualDaemon { if runtimeKind == "docker" { - return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-docker", socket: "/var/run/docker.sock", dataRoot: "/var/lib/pulse-rootful-docker", fixture: "/opt/pulse/rootful-fixture"} + return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-docker", socket: "/var/run/docker.sock", dataRoot: "/var/lib/pulse-rootful-docker", runRoot: "/run/pulse-rootful-docker", fixture: "/opt/pulse/rootful-fixture"} } - return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-podman", socket: "/run/podman/podman.sock", dataRoot: "/var/lib/containers", fixture: "/opt/pulse/rootful-fixture"} + return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-podman", socket: "/run/podman/podman.sock", dataRoot: "/var/lib/pulse-rootful-podman", runRoot: "/run/pulse-rootful-podman", fixture: "/opt/pulse/rootful-fixture"} } func rootfulQualPrepareFixture(t *testing.T, daemon rootfulQualDaemon) { @@ -486,11 +487,23 @@ func rootfulQualStartDaemon(t *testing.T, daemon rootfulQualDaemon) { "--exec-root=/run/pulse-rootful-docker", "--pidfile=/run/pulse-rootful-docker.pid", "--storage-driver=vfs", "--iptables=false", "--bridge=none") } else { rootlessQualCommand(t, 20*time.Second, "systemd-run", "--quiet", "--collect", "--unit", daemon.unit, "--property=Type=exec", "--", - "/usr/bin/podman", "system", "service", "--time=0", "unix://"+daemon.socket) + "/usr/bin/podman", "--storage-driver=vfs", "--root="+daemon.dataRoot, "--runroot="+daemon.runRoot, + "system", "service", "--time=0", "unix://"+daemon.socket) } rootlessQualWaitSocket(t, daemon.socket) rootlessQualCommand(t, 10*time.Second, "chmod", "0660", daemon.socket) rootfulQualRuntimeCommand(t, daemon, 30*time.Second, "info") + if driver := rootfulQualRuntimeStorageDriver(t, daemon); driver != "vfs" { + t.Fatalf("rootful %s storage driver = %q, want vfs", daemon.runtime, driver) + } +} + +func rootfulQualRuntimeStorageDriver(t *testing.T, daemon rootfulQualDaemon) string { + t.Helper() + if daemon.runtime == "docker" { + return rootfulQualRuntimeCommand(t, daemon, 30*time.Second, "info", "--format", "{{.Driver}}") + } + return rootfulQualRuntimeCommand(t, daemon, 30*time.Second, "info", "--format", "{{.Store.GraphDriverName}}") } func rootfulQualStopDaemon(t *testing.T, daemon rootfulQualDaemon) { @@ -697,7 +710,7 @@ func rootfulQualRunBoundProbe(t *testing.T, deadline time.Duration) time.Duratio func rootfulQualRemoveRuntimeState(t *testing.T, daemon rootfulQualDaemon) { t.Helper() - roots := []string{daemon.dataRoot, "/run/pulse-rootful-docker"} + roots := []string{daemon.dataRoot, daemon.runRoot} deadline := time.Now().Add(30 * time.Second) for { mountInfo := string(rootlessQualReadFile(t, "/proc/self/mountinfo")) @@ -713,7 +726,7 @@ func rootfulQualRemoveRuntimeState(t *testing.T, daemon rootfulQualDaemon) { } time.Sleep(100 * time.Millisecond) } - for _, path := range []string{daemon.dataRoot, "/run/pulse-rootful-docker", daemon.fixture} { + for _, path := range []string{daemon.dataRoot, daemon.runRoot, daemon.fixture} { if err := os.RemoveAll(path); err != nil { t.Fatalf("remove disposable runtime path %s: %v", path, err) } @@ -721,7 +734,7 @@ func rootfulQualRemoveRuntimeState(t *testing.T, daemon rootfulQualDaemon) { } func rootfulQualRuntimeStateClean(daemon rootfulQualDaemon) bool { - for _, path := range []string{daemon.socket, daemon.dataRoot, "/run/pulse-rootful-docker", daemon.fixture} { + for _, path := range []string{daemon.socket, daemon.dataRoot, daemon.runRoot, daemon.fixture} { if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { return false } @@ -1001,6 +1014,9 @@ func TestRootfulQualificationWrapperInvariants(t *testing.T) { "github.com/rcourtman/pulse-go-rewrite/scripts/installtests.test", "https://github.com/rcourtman/Pulse.git", "refs/remotes/origin/main", "refs/heads/main", "PULSE_ROOTFUL_UBUNTU_IMAGE", "PULSE_ROOTFUL_BOUND_PROBE_BINARY", + "rootful_qualification_systemd_readiness", + "podman-auto-update.service", "podman-auto-update.timer", + "podman-clean-transient.service", "podman-restart.service", rootfulQualBoundProbe, } { if !strings.Contains(script, required) { @@ -1016,3 +1032,63 @@ func TestRootfulQualificationWrapperInvariants(t *testing.T) { } } } + +func TestRootfulQualificationSystemdReadiness(t *testing.T) { + runtimeScript := repoFile("scripts", "secure-runtime-rootful-runtime.sh") + binDir := t.TempDir() + fakeDocker := filepath.Join(binDir, "docker") + fake := `#!/bin/sh +case "$*" in + *"systemctl show --property=ActiveState --value multi-user.target"*) + printf '%s\n' "${FAKE_SYSTEMD_TARGET_STATE:-inactive}" + ;; + *"systemctl list-units --state=failed --no-legend --no-pager --plain"*) + printf '%s' "${FAKE_SYSTEMD_FAILED_UNITS:-}" + ;; + *) + printf 'unexpected docker arguments: %s\n' "$*" >&2 + exit 99 + ;; +esac +` + if err := os.WriteFile(fakeDocker, []byte(fake), 0o700); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + targetState string + failedUnits string + wantExit int + wantOutput string + }{ + {name: "target inactive", targetState: "inactive", wantExit: 1}, + {name: "clean multi-user target", targetState: "active", wantExit: 0}, + {name: "failed unit", targetState: "active", failedUnits: "podman-restart.service loaded failed failed", wantExit: 2, wantOutput: "podman-restart.service"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := exec.Command("bash", "-c", `source "$1"; rootful_qualification_systemd_readiness fixture`, "bash", runtimeScript) + cmd.Env = append(os.Environ(), + "PATH="+binDir+":"+os.Getenv("PATH"), + "FAKE_SYSTEMD_TARGET_STATE="+test.targetState, + "FAKE_SYSTEMD_FAILED_UNITS="+test.failedUnits, + ) + output, err := cmd.CombinedOutput() + gotExit := 0 + if err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("readiness helper failed without an exit status: %v", err) + } + gotExit = exitErr.ExitCode() + } + if gotExit != test.wantExit { + t.Fatalf("readiness exit = %d, want %d\n%s", gotExit, test.wantExit, output) + } + if test.wantOutput != "" && !strings.Contains(string(output), test.wantOutput) { + t.Fatalf("readiness output missing %q: %s", test.wantOutput, output) + } + }) + } +} diff --git a/scripts/release_control/secure_runtime_rootful_attestation_v1_test.py b/scripts/release_control/secure_runtime_rootful_attestation_v1_test.py index efe64e636..beb40af50 100644 --- a/scripts/release_control/secure_runtime_rootful_attestation_v1_test.py +++ b/scripts/release_control/secure_runtime_rootful_attestation_v1_test.py @@ -493,6 +493,7 @@ if module.MAX_RECEIPT_BYTES <= 0: "scripts/installtests/secure_runtime_rootless_qualification_test.go", "scripts/installtests/secure_runtime_systemd_lab_test.go", "scripts/release_control/secure_runtime_rootful_attestation_v1.py", + "scripts/secure-runtime-rootful-runtime.sh", "scripts/release_control/secure_runtime_rootful_source_manifest_v1.json", "scripts/release_control/secure_runtime_rootless_attestation_v1.py", "scripts/run-secure-runtime-rootful-qualification.sh", diff --git a/scripts/release_control/secure_runtime_rootful_source_manifest_v1.json b/scripts/release_control/secure_runtime_rootful_source_manifest_v1.json index b5a76c0e7..0ccc60714 100644 --- a/scripts/release_control/secure_runtime_rootful_source_manifest_v1.json +++ b/scripts/release_control/secure_runtime_rootful_source_manifest_v1.json @@ -37,6 +37,7 @@ "scripts/installtests/umask_unix_test.go", "scripts/installtests/uninstall_sensor_proxy_test.go", "scripts/release_control/secure_runtime_rootful_attestation_v1.py", + "scripts/secure-runtime-rootful-runtime.sh", "scripts/release_control/secure_runtime_rootful_source_manifest_v1.json", "scripts/release_control/secure_runtime_rootless_attestation_v1.py", "scripts/run-secure-runtime-rootful-qualification.sh" diff --git a/scripts/run-secure-runtime-rootful-qualification.sh b/scripts/run-secure-runtime-rootful-qualification.sh index f1bca2dc8..2a90e39b5 100755 --- a/scripts/run-secure-runtime-rootful-qualification.sh +++ b/scripts/run-secure-runtime-rootful-qualification.sh @@ -2,6 +2,8 @@ set -euo pipefail readonly REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/secure-runtime-rootful-runtime.sh +source "${REPO_ROOT}/scripts/secure-runtime-rootful-runtime.sh" readonly UBUNTU_IMAGE="${PULSE_ROOTFUL_UBUNTU_IMAGE:?set PULSE_ROOTFUL_UBUNTU_IMAGE to an immutable ubuntu@sha256:... Ubuntu 24.04 image}" readonly OUTPUT_PARENT="${PULSE_ROOTFUL_QUALIFICATION_OUTPUT_DIR:?set PULSE_ROOTFUL_QUALIFICATION_OUTPUT_DIR to an existing absolute private directory}" readonly CONFIRM="${PULSE_ROOTFUL_QUALIFICATION_CONFIRM:-}" @@ -261,10 +263,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable\n' "\$(dpkg --print-architecture)" >/etc/apt/sources.list.d/docker.list && \ apt-get update && apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ - ln -sf /dev/null /etc/systemd/system/docker.service && \ - ln -sf /dev/null /etc/systemd/system/docker.socket && \ - ln -sf /dev/null /etc/systemd/system/podman.service && \ - ln -sf /dev/null /etc/systemd/system/podman.socket && \ + for unit in \ + docker.service docker.socket \ + podman.service podman.socket \ + podman-auto-update.service podman-auto-update.timer \ + podman-clean-transient.service podman-restart.service; do \ + ln -sf /dev/null "/etc/systemd/system/\${unit}"; \ + done && \ install -d -m 0700 /opt/pulse/packet /opt/pulse/result && \ printf '%s\n' disposable-v1 >/etc/pulse-secure-runtime-rootful-qualification && \ rm -f /etc/machine-id && touch /etc/machine-id && \ @@ -284,13 +289,19 @@ capture_qualification_container_diagnostics() { local runtime_name="$1" container_id="$2" docker logs "${container_id}" >"${OUTPUT_DIR}/${runtime_name}-container.log" 2>&1 || true docker exec "${container_id}" journalctl --no-pager -n 2000 >"${OUTPUT_DIR}/${runtime_name}-journal.log" 2>&1 || true - chmod 0600 "${OUTPUT_DIR}/${runtime_name}-container.log" "${OUTPUT_DIR}/${runtime_name}-journal.log" + docker exec "${container_id}" systemctl is-system-running >"${OUTPUT_DIR}/${runtime_name}-systemd-state.log" 2>&1 || true + docker exec "${container_id}" systemctl list-units --state=failed --no-legend --no-pager --plain >"${OUTPUT_DIR}/${runtime_name}-failed-units.log" 2>&1 || true + chmod 0600 \ + "${OUTPUT_DIR}/${runtime_name}-container.log" \ + "${OUTPUT_DIR}/${runtime_name}-journal.log" \ + "${OUTPUT_DIR}/${runtime_name}-systemd-state.log" \ + "${OUTPUT_DIR}/${runtime_name}-failed-units.log" } run_runtime() { local runtime_name="$1" local container_name="pulse-rootful-qual-${runtime_name}-${SOURCE_COMMIT:0:8}-$$" - local container_id local_receipt machine_id_file machine_id deadline mounts packet_probe_hash installed_probe_hash + local container_id local_receipt machine_id_file machine_id deadline mounts packet_probe_hash installed_probe_hash readiness_status local_receipt="${OUTPUT_DIR}/${runtime_name}-receipt.json" machine_id_file="${PACKET_DIR}/.machine-id-${runtime_name}" machine_id="$(openssl rand -hex 16)" @@ -313,7 +324,17 @@ run_runtime() { docker start "${container_id}" >/dev/null deadline=$((SECONDS + 60)) - until docker exec "${container_id}" systemctl is-system-running --wait >/dev/null 2>&1; do + while true; do + readiness_status=0 + rootful_qualification_systemd_readiness "${container_id}" || readiness_status=$? + if (( readiness_status == 0 )); then + break + fi + if (( readiness_status == 2 )); then + capture_qualification_container_diagnostics "${runtime_name}" "${container_id}" + echo "ERROR: ${runtime_name} disposable systemd container reached multi-user.target with failed units" >&2 + return 1 + fi if (( SECONDS >= deadline )); then capture_qualification_container_diagnostics "${runtime_name}" "${container_id}" echo "ERROR: ${runtime_name} disposable systemd container did not become ready" >&2 diff --git a/scripts/secure-runtime-rootful-runtime.sh b/scripts/secure-runtime-rootful-runtime.sh new file mode 100644 index 000000000..f7065841e --- /dev/null +++ b/scripts/secure-runtime-rootful-runtime.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +# This file is sourced by run-secure-runtime-rootful-qualification.sh and kept +# separately so the fail-closed systemd readiness predicate can be exercised +# without entering the destructive qualification wrapper. + +rootful_qualification_systemd_readiness() { + local container_id="$1" + local target_state failed_units + + target_state="$(docker exec "${container_id}" systemctl show \ + --property=ActiveState --value multi-user.target 2>/dev/null)" || return 1 + [[ "${target_state}" == "active" ]] || return 1 + + failed_units="$(docker exec "${container_id}" systemctl list-units \ + --state=failed --no-legend --no-pager --plain 2>/dev/null)" || return 1 + if [[ -n "${failed_units//[[:space:]]/}" ]]; then + printf 'ERROR: disposable systemd container has failed units:\n%s\n' "${failed_units}" >&2 + return 2 + fi +} From e93ee380847f8877e73b9d6add15b17589f14ce4 Mon Sep 17 00:00:00 2001 From: rcourtman <8825017+rcourtman@users.noreply.github.com> Date: Wed, 2 Sep 2026 02:52:13 +0100 Subject: [PATCH 2/2] Tighten rootful runtime cleanup proof Rootful evidence must reject non-running systemd states and prove no unrelated container runtime survives. Require exact manager state, mask distro containerd, recheck readiness after scenarios, and remove the complete Podman socket boundary. --- .../v6/internal/subsystems/agent-lifecycle.md | 8 ++- .../subsystems/deployment-installability.md | 9 ++- ...cure_runtime_rootful_qualification_test.go | 70 ++++++++++++++++--- ...un-secure-runtime-rootful-qualification.sh | 13 +++- scripts/secure-runtime-rootful-runtime.sh | 23 +++++- 5 files changed, 105 insertions(+), 18 deletions(-) diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index f51553c32..f5e4cf8c3 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -7401,9 +7401,11 @@ explicit entrypoint for real rootful Docker and Podman proof on two distinct disposable Ubuntu/systemd hosts. It never mounts a host daemon socket and runs the outer containers without a default route. The host image masks Docker and Podman's distro-managed service, socket, auto-update, transient-cleanup, and -restart units before first boot; the packet starts only its explicit -runtime-specific daemon units and requires systemd's strict running state -before any scenario begins. The packet binds exact clean Go +restart units plus the distro containerd service before first boot; the packet +starts only its explicit runtime-specific daemon units, requires systemd's +exact running state with no failed units before and after each runtime packet, +and proves the explicit Docker daemon's child containerd exits with that +daemon. The packet binds exact clean Go artifacts, every compiled installtests package input, the canonical remote-main commit, the immutable Ubuntu base-image digest, and governed source hashes, then records fresh install, legacy migration with authority reduction, diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index 09590b5a2..91e7124ad 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -328,9 +328,12 @@ packet. The explicit network-isolated Ubuntu/systemd hosts for real rootful Docker and Podman and never mounts a host daemon socket. The disposable image masks the distro's Docker and Podman service, socket, auto-update, transient-cleanup, and restart -units before first boot, so strict systemd readiness cannot be confused with -expected nested-runtime housekeeping failures; each scenario starts only its -explicit runtime-specific daemon unit. Its standalone schema-v1 receipt binds the +units plus the distro containerd service before first boot, so exact systemd +running state cannot be confused with expected nested-runtime housekeeping +failures. Each scenario starts only its explicit runtime-specific daemon unit, +rechecks zero failed units before receipt acceptance, proves the explicit +Docker daemon's child containerd exits, and removes the complete Podman socket +boundary during cleanup. Its standalone schema-v1 receipt binds the exact canonical remote-main commit, immutable Ubuntu base-image digest, qualification, collector, helper, installer, every compiled installtests package input, source manifest, canonical root-owned socket, summary inventory, diff --git a/scripts/installtests/secure_runtime_rootful_qualification_test.go b/scripts/installtests/secure_runtime_rootful_qualification_test.go index 18eefe31a..98a9b4513 100644 --- a/scripts/installtests/secure_runtime_rootful_qualification_test.go +++ b/scripts/installtests/secure_runtime_rootful_qualification_test.go @@ -108,6 +108,7 @@ func TestSecureRuntimeRootfulQualification(t *testing.T) { runtimeKind := strings.TrimSpace(os.Getenv("PULSE_ROOTFUL_RUNTIME")) receiptPath := strings.TrimSpace(os.Getenv("PULSE_ROOTFUL_RECEIPT")) rootfulQualRequireDisposableHost(t, runtimeKind, receiptPath) + rootfulQualAssertSystemContainerdDisabled(t) collector := secureRuntimeReadArtifact(t, "PULSE_SECURE_RUNTIME_COLLECTOR") collectorSignature := secureRuntimeReadSignature(t, "PULSE_SECURE_RUNTIME_COLLECTOR_SIGNATURE") @@ -510,6 +511,38 @@ func rootfulQualStopDaemon(t *testing.T, daemon rootfulQualDaemon) { t.Helper() rootlessQualStopUnit(t, daemon.unit) _ = os.Remove(daemon.socket) + if daemon.runtime == "docker" { + rootfulQualWaitNoContainerd(t, 20*time.Second) + } +} + +func rootfulQualAssertSystemContainerdDisabled(t *testing.T) { + t.Helper() + unitState := rootlessQualCommand(t, 10*time.Second, "systemctl", "show", "containerd.service", "--property=UnitFileState", "--value") + activeState := rootlessQualCommand(t, 10*time.Second, "systemctl", "show", "containerd.service", "--property=ActiveState", "--value") + if unitState != "masked" || activeState != "inactive" { + t.Fatalf("distro containerd service must be masked and inactive: UnitFileState=%q ActiveState=%q", unitState, activeState) + } + rootfulQualWaitNoContainerd(t, 5*time.Second) +} + +func rootfulQualWaitNoContainerd(t *testing.T, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for { + output, err := rootlessQualCommandError(3*time.Second, "pgrep", "-x", "containerd") + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { + return + } + t.Fatalf("inspect containerd processes: %v\n%s", err, output) + } + if time.Now().After(deadline) { + t.Fatalf("containerd process remained after explicit Docker daemon stop: %s", strings.TrimSpace(output)) + } + time.Sleep(100 * time.Millisecond) + } } func rootfulQualRuntimeCommand(t *testing.T, daemon rootfulQualDaemon, timeout time.Duration, args ...string) string { @@ -731,10 +764,19 @@ func rootfulQualRemoveRuntimeState(t *testing.T, daemon rootfulQualDaemon) { t.Fatalf("remove disposable runtime path %s: %v", path, err) } } + if daemon.runtime == "podman" { + if err := os.Remove(filepath.Dir(daemon.socket)); err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatalf("remove disposable Podman socket directory: %v", err) + } + } } func rootfulQualRuntimeStateClean(daemon rootfulQualDaemon) bool { - for _, path := range []string{daemon.socket, daemon.dataRoot, daemon.runRoot, daemon.fixture} { + paths := []string{daemon.socket, daemon.dataRoot, daemon.runRoot, daemon.fixture} + if daemon.runtime == "podman" { + paths = append(paths, filepath.Dir(daemon.socket)) + } + for _, path := range paths { if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { return false } @@ -1015,6 +1057,7 @@ func TestRootfulQualificationWrapperInvariants(t *testing.T) { "https://github.com/rcourtman/Pulse.git", "refs/remotes/origin/main", "refs/heads/main", "PULSE_ROOTFUL_UBUNTU_IMAGE", "PULSE_ROOTFUL_BOUND_PROBE_BINARY", "rootful_qualification_systemd_readiness", + "containerd.service", "podman-auto-update.service", "podman-auto-update.timer", "podman-clean-transient.service", "podman-restart.service", rootfulQualBoundProbe, @@ -1039,6 +1082,10 @@ func TestRootfulQualificationSystemdReadiness(t *testing.T) { fakeDocker := filepath.Join(binDir, "docker") fake := `#!/bin/sh case "$*" in + *"systemctl is-system-running"*) + printf '%s\n' "${FAKE_SYSTEMD_MANAGER_STATE:-starting}" + [ "${FAKE_SYSTEMD_MANAGER_STATE:-starting}" = running ] + ;; *"systemctl show --property=ActiveState --value multi-user.target"*) printf '%s\n' "${FAKE_SYSTEMD_TARGET_STATE:-inactive}" ;; @@ -1056,21 +1103,26 @@ esac } tests := []struct { - name string - targetState string - failedUnits string - wantExit int - wantOutput string + name string + managerState string + targetState string + failedUnits string + wantExit int + wantOutput string }{ - {name: "target inactive", targetState: "inactive", wantExit: 1}, - {name: "clean multi-user target", targetState: "active", wantExit: 0}, - {name: "failed unit", targetState: "active", failedUnits: "podman-restart.service loaded failed failed", wantExit: 2, wantOutput: "podman-restart.service"}, + {name: "manager starting", managerState: "starting", targetState: "active", wantExit: 1}, + {name: "target inactive", managerState: "running", targetState: "inactive", wantExit: 1}, + {name: "clean running manager", managerState: "running", targetState: "active", wantExit: 0}, + {name: "degraded manager", managerState: "degraded", targetState: "active", wantExit: 2, wantOutput: "degraded"}, + {name: "maintenance manager", managerState: "maintenance", targetState: "active", wantExit: 2, wantOutput: "maintenance"}, + {name: "failed unit", managerState: "running", targetState: "active", failedUnits: "podman-restart.service loaded failed failed", wantExit: 2, wantOutput: "podman-restart.service"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { cmd := exec.Command("bash", "-c", `source "$1"; rootful_qualification_systemd_readiness fixture`, "bash", runtimeScript) cmd.Env = append(os.Environ(), "PATH="+binDir+":"+os.Getenv("PATH"), + "FAKE_SYSTEMD_MANAGER_STATE="+test.managerState, "FAKE_SYSTEMD_TARGET_STATE="+test.targetState, "FAKE_SYSTEMD_FAILED_UNITS="+test.failedUnits, ) diff --git a/scripts/run-secure-runtime-rootful-qualification.sh b/scripts/run-secure-runtime-rootful-qualification.sh index 2a90e39b5..70041c190 100755 --- a/scripts/run-secure-runtime-rootful-qualification.sh +++ b/scripts/run-secure-runtime-rootful-qualification.sh @@ -265,6 +265,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ for unit in \ docker.service docker.socket \ + containerd.service \ podman.service podman.socket \ podman-auto-update.service podman-auto-update.timer \ podman-clean-transient.service podman-restart.service; do \ @@ -332,7 +333,7 @@ run_runtime() { fi if (( readiness_status == 2 )); then capture_qualification_container_diagnostics "${runtime_name}" "${container_id}" - echo "ERROR: ${runtime_name} disposable systemd container reached multi-user.target with failed units" >&2 + echo "ERROR: ${runtime_name} disposable systemd container entered a terminal non-running state or has failed units" >&2 return 1 fi if (( SECONDS >= deadline )); then @@ -374,11 +375,19 @@ run_runtime() { -e PULSE_SECURE_RUNTIME_INSTALLER=/opt/pulse/packet/install.sh \ "${container_id}" /opt/pulse/packet/dockeragent.test \ -test.run '^TestSecureRuntimeRootfulQualification$' -test.count=1 -test.v -test.timeout=45m \ - | tee "${OUTPUT_DIR}/${runtime_name}-test.log"; then + | tee "${OUTPUT_DIR}/${runtime_name}-test.log"; then capture_qualification_container_diagnostics "${runtime_name}" "${container_id}" chmod 0600 "${OUTPUT_DIR}/${runtime_name}-test.log" return 1 fi + readiness_status=0 + rootful_qualification_systemd_readiness "${container_id}" || readiness_status=$? + if (( readiness_status != 0 )); then + capture_qualification_container_diagnostics "${runtime_name}" "${container_id}" + chmod 0600 "${OUTPUT_DIR}/${runtime_name}-test.log" + echo "ERROR: ${runtime_name} systemd readiness changed before receipt acceptance" >&2 + return 1 + fi docker exec "${container_id}" test -f /opt/pulse/result/rootful-receipt.json || { capture_qualification_container_diagnostics "${runtime_name}" "${container_id}" echo "ERROR: ${runtime_name} qualification did not retain its receipt" >&2 diff --git a/scripts/secure-runtime-rootful-runtime.sh b/scripts/secure-runtime-rootful-runtime.sh index f7065841e..0627b96e9 100644 --- a/scripts/secure-runtime-rootful-runtime.sh +++ b/scripts/secure-runtime-rootful-runtime.sh @@ -6,7 +6,28 @@ rootful_qualification_systemd_readiness() { local container_id="$1" - local target_state failed_units + local manager_state manager_status=0 target_state failed_units + + if manager_state="$(docker exec "${container_id}" systemctl is-system-running 2>/dev/null)"; then + manager_status=0 + else + manager_status=$? + fi + case "${manager_state}" in + running) + if (( manager_status != 0 )); then + printf 'ERROR: systemd reported running with exit status %d\n' "${manager_status}" >&2 + return 2 + fi + ;; + ""|initializing|starting) + return 1 + ;; + *) + printf 'ERROR: disposable systemd container entered non-running state %q\n' "${manager_state}" >&2 + return 2 + ;; + esac target_state="$(docker exec "${container_id}" systemctl show \ --property=ActiveState --value multi-user.target 2>/dev/null)" || return 1