Merge pull request #1839 from rcourtman/pulse/fix-rootful-systemd-readiness

Fix rootful qualification readiness
This commit is contained in:
rcourtman
2026-09-02 03:39:50 +01:00
committed by GitHub
7 changed files with 231 additions and 16 deletions
@@ -7399,7 +7399,13 @@ the product default.
`scripts/run-secure-runtime-rootful-qualification.sh` is the destructive,
explicit entrypoint for real rootful Docker and Podman proof on two distinct
disposable Ubuntu/systemd hosts. It never mounts a host daemon socket and runs
the outer containers without a default route. The packet binds exact clean Go
the outer containers without a default route. The host image masks Docker and
Podman's distro-managed service, socket, auto-update, transient-cleanup, and
restart units plus the distro containerd service before first boot; the packet
starts only its explicit runtime-specific daemon units, requires systemd's
exact running state with no failed units before and after each runtime packet,
and proves the explicit Docker daemon's child containerd exits with that
daemon. The packet binds exact clean Go
artifacts, every compiled installtests package input, the canonical remote-main
commit, the immutable Ubuntu base-image digest, and governed source hashes,
then records fresh install, legacy migration with authority reduction,
@@ -326,7 +326,14 @@ Live rootful-runtime proof is also separated from the schema-v7 systemd
packet. The explicit
`scripts/run-secure-runtime-rootful-qualification.sh` wrapper creates distinct
network-isolated Ubuntu/systemd hosts for real rootful Docker and Podman and
never mounts a host daemon socket. Its standalone schema-v1 receipt binds the
never mounts a host daemon socket. The disposable image masks the distro's
Docker and Podman service, socket, auto-update, transient-cleanup, and restart
units plus the distro containerd service before first boot, so exact systemd
running state cannot be confused with expected nested-runtime housekeeping
failures. Each scenario starts only its explicit runtime-specific daemon unit,
rechecks zero failed units before receipt acceptance, proves the explicit
Docker daemon's child containerd exits, and removes the complete Podman socket
boundary during cleanup. Its standalone schema-v1 receipt binds the
exact canonical remote-main commit, immutable Ubuntu base-image digest,
qualification, collector, helper, installer, every compiled installtests
package input, source manifest, canonical root-owned socket, summary inventory,
@@ -97,6 +97,7 @@ type rootfulQualDaemon struct {
unit string
socket string
dataRoot string
runRoot string
fixture string
}
@@ -107,6 +108,7 @@ func TestSecureRuntimeRootfulQualification(t *testing.T) {
runtimeKind := strings.TrimSpace(os.Getenv("PULSE_ROOTFUL_RUNTIME"))
receiptPath := strings.TrimSpace(os.Getenv("PULSE_ROOTFUL_RECEIPT"))
rootfulQualRequireDisposableHost(t, runtimeKind, receiptPath)
rootfulQualAssertSystemContainerdDisabled(t)
collector := secureRuntimeReadArtifact(t, "PULSE_SECURE_RUNTIME_COLLECTOR")
collectorSignature := secureRuntimeReadSignature(t, "PULSE_SECURE_RUNTIME_COLLECTOR_SIGNATURE")
@@ -454,9 +456,9 @@ func rootfulQualRequireDisposableHost(t *testing.T, runtimeKind, receiptPath str
func rootfulQualDaemonFor(runtimeKind string) rootfulQualDaemon {
if runtimeKind == "docker" {
return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-docker", socket: "/var/run/docker.sock", dataRoot: "/var/lib/pulse-rootful-docker", fixture: "/opt/pulse/rootful-fixture"}
return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-docker", socket: "/var/run/docker.sock", dataRoot: "/var/lib/pulse-rootful-docker", runRoot: "/run/pulse-rootful-docker", fixture: "/opt/pulse/rootful-fixture"}
}
return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-podman", socket: "/run/podman/podman.sock", dataRoot: "/var/lib/containers", fixture: "/opt/pulse/rootful-fixture"}
return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-podman", socket: "/run/podman/podman.sock", dataRoot: "/var/lib/pulse-rootful-podman", runRoot: "/run/pulse-rootful-podman", fixture: "/opt/pulse/rootful-fixture"}
}
func rootfulQualPrepareFixture(t *testing.T, daemon rootfulQualDaemon) {
@@ -486,17 +488,61 @@ func rootfulQualStartDaemon(t *testing.T, daemon rootfulQualDaemon) {
"--exec-root=/run/pulse-rootful-docker", "--pidfile=/run/pulse-rootful-docker.pid", "--storage-driver=vfs", "--iptables=false", "--bridge=none")
} else {
rootlessQualCommand(t, 20*time.Second, "systemd-run", "--quiet", "--collect", "--unit", daemon.unit, "--property=Type=exec", "--",
"/usr/bin/podman", "system", "service", "--time=0", "unix://"+daemon.socket)
"/usr/bin/podman", "--storage-driver=vfs", "--root="+daemon.dataRoot, "--runroot="+daemon.runRoot,
"system", "service", "--time=0", "unix://"+daemon.socket)
}
rootlessQualWaitSocket(t, daemon.socket)
rootlessQualCommand(t, 10*time.Second, "chmod", "0660", daemon.socket)
rootfulQualRuntimeCommand(t, daemon, 30*time.Second, "info")
if driver := rootfulQualRuntimeStorageDriver(t, daemon); driver != "vfs" {
t.Fatalf("rootful %s storage driver = %q, want vfs", daemon.runtime, driver)
}
}
func rootfulQualRuntimeStorageDriver(t *testing.T, daemon rootfulQualDaemon) string {
t.Helper()
if daemon.runtime == "docker" {
return rootfulQualRuntimeCommand(t, daemon, 30*time.Second, "info", "--format", "{{.Driver}}")
}
return rootfulQualRuntimeCommand(t, daemon, 30*time.Second, "info", "--format", "{{.Store.GraphDriverName}}")
}
func rootfulQualStopDaemon(t *testing.T, daemon rootfulQualDaemon) {
t.Helper()
rootlessQualStopUnit(t, daemon.unit)
_ = os.Remove(daemon.socket)
if daemon.runtime == "docker" {
rootfulQualWaitNoContainerd(t, 20*time.Second)
}
}
func rootfulQualAssertSystemContainerdDisabled(t *testing.T) {
t.Helper()
unitState := rootlessQualCommand(t, 10*time.Second, "systemctl", "show", "containerd.service", "--property=UnitFileState", "--value")
activeState := rootlessQualCommand(t, 10*time.Second, "systemctl", "show", "containerd.service", "--property=ActiveState", "--value")
if unitState != "masked" || activeState != "inactive" {
t.Fatalf("distro containerd service must be masked and inactive: UnitFileState=%q ActiveState=%q", unitState, activeState)
}
rootfulQualWaitNoContainerd(t, 5*time.Second)
}
func rootfulQualWaitNoContainerd(t *testing.T, timeout time.Duration) {
t.Helper()
deadline := time.Now().Add(timeout)
for {
output, err := rootlessQualCommandError(3*time.Second, "pgrep", "-x", "containerd")
if err != nil {
var exitErr *exec.ExitError
if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 {
return
}
t.Fatalf("inspect containerd processes: %v\n%s", err, output)
}
if time.Now().After(deadline) {
t.Fatalf("containerd process remained after explicit Docker daemon stop: %s", strings.TrimSpace(output))
}
time.Sleep(100 * time.Millisecond)
}
}
func rootfulQualRuntimeCommand(t *testing.T, daemon rootfulQualDaemon, timeout time.Duration, args ...string) string {
@@ -697,7 +743,7 @@ func rootfulQualRunBoundProbe(t *testing.T, deadline time.Duration) time.Duratio
func rootfulQualRemoveRuntimeState(t *testing.T, daemon rootfulQualDaemon) {
t.Helper()
roots := []string{daemon.dataRoot, "/run/pulse-rootful-docker"}
roots := []string{daemon.dataRoot, daemon.runRoot}
deadline := time.Now().Add(30 * time.Second)
for {
mountInfo := string(rootlessQualReadFile(t, "/proc/self/mountinfo"))
@@ -713,15 +759,24 @@ func rootfulQualRemoveRuntimeState(t *testing.T, daemon rootfulQualDaemon) {
}
time.Sleep(100 * time.Millisecond)
}
for _, path := range []string{daemon.dataRoot, "/run/pulse-rootful-docker", daemon.fixture} {
for _, path := range []string{daemon.dataRoot, daemon.runRoot, daemon.fixture} {
if err := os.RemoveAll(path); err != nil {
t.Fatalf("remove disposable runtime path %s: %v", path, err)
}
}
if daemon.runtime == "podman" {
if err := os.Remove(filepath.Dir(daemon.socket)); err != nil && !errors.Is(err, os.ErrNotExist) {
t.Fatalf("remove disposable Podman socket directory: %v", err)
}
}
}
func rootfulQualRuntimeStateClean(daemon rootfulQualDaemon) bool {
for _, path := range []string{daemon.socket, daemon.dataRoot, "/run/pulse-rootful-docker", daemon.fixture} {
paths := []string{daemon.socket, daemon.dataRoot, daemon.runRoot, daemon.fixture}
if daemon.runtime == "podman" {
paths = append(paths, filepath.Dir(daemon.socket))
}
for _, path := range paths {
if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) {
return false
}
@@ -1001,6 +1056,10 @@ func TestRootfulQualificationWrapperInvariants(t *testing.T) {
"github.com/rcourtman/pulse-go-rewrite/scripts/installtests.test",
"https://github.com/rcourtman/Pulse.git", "refs/remotes/origin/main", "refs/heads/main",
"PULSE_ROOTFUL_UBUNTU_IMAGE", "PULSE_ROOTFUL_BOUND_PROBE_BINARY",
"rootful_qualification_systemd_readiness",
"containerd.service",
"podman-auto-update.service", "podman-auto-update.timer",
"podman-clean-transient.service", "podman-restart.service",
rootfulQualBoundProbe,
} {
if !strings.Contains(script, required) {
@@ -1016,3 +1075,72 @@ func TestRootfulQualificationWrapperInvariants(t *testing.T) {
}
}
}
func TestRootfulQualificationSystemdReadiness(t *testing.T) {
runtimeScript := repoFile("scripts", "secure-runtime-rootful-runtime.sh")
binDir := t.TempDir()
fakeDocker := filepath.Join(binDir, "docker")
fake := `#!/bin/sh
case "$*" in
*"systemctl is-system-running"*)
printf '%s\n' "${FAKE_SYSTEMD_MANAGER_STATE:-starting}"
[ "${FAKE_SYSTEMD_MANAGER_STATE:-starting}" = running ]
;;
*"systemctl show --property=ActiveState --value multi-user.target"*)
printf '%s\n' "${FAKE_SYSTEMD_TARGET_STATE:-inactive}"
;;
*"systemctl list-units --state=failed --no-legend --no-pager --plain"*)
printf '%s' "${FAKE_SYSTEMD_FAILED_UNITS:-}"
;;
*)
printf 'unexpected docker arguments: %s\n' "$*" >&2
exit 99
;;
esac
`
if err := os.WriteFile(fakeDocker, []byte(fake), 0o700); err != nil {
t.Fatal(err)
}
tests := []struct {
name string
managerState string
targetState string
failedUnits string
wantExit int
wantOutput string
}{
{name: "manager starting", managerState: "starting", targetState: "active", wantExit: 1},
{name: "target inactive", managerState: "running", targetState: "inactive", wantExit: 1},
{name: "clean running manager", managerState: "running", targetState: "active", wantExit: 0},
{name: "degraded manager", managerState: "degraded", targetState: "active", wantExit: 2, wantOutput: "degraded"},
{name: "maintenance manager", managerState: "maintenance", targetState: "active", wantExit: 2, wantOutput: "maintenance"},
{name: "failed unit", managerState: "running", targetState: "active", failedUnits: "podman-restart.service loaded failed failed", wantExit: 2, wantOutput: "podman-restart.service"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cmd := exec.Command("bash", "-c", `source "$1"; rootful_qualification_systemd_readiness fixture`, "bash", runtimeScript)
cmd.Env = append(os.Environ(),
"PATH="+binDir+":"+os.Getenv("PATH"),
"FAKE_SYSTEMD_MANAGER_STATE="+test.managerState,
"FAKE_SYSTEMD_TARGET_STATE="+test.targetState,
"FAKE_SYSTEMD_FAILED_UNITS="+test.failedUnits,
)
output, err := cmd.CombinedOutput()
gotExit := 0
if err != nil {
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) {
t.Fatalf("readiness helper failed without an exit status: %v", err)
}
gotExit = exitErr.ExitCode()
}
if gotExit != test.wantExit {
t.Fatalf("readiness exit = %d, want %d\n%s", gotExit, test.wantExit, output)
}
if test.wantOutput != "" && !strings.Contains(string(output), test.wantOutput) {
t.Fatalf("readiness output missing %q: %s", test.wantOutput, output)
}
})
}
}
@@ -493,6 +493,7 @@ if module.MAX_RECEIPT_BYTES <= 0:
"scripts/installtests/secure_runtime_rootless_qualification_test.go",
"scripts/installtests/secure_runtime_systemd_lab_test.go",
"scripts/release_control/secure_runtime_rootful_attestation_v1.py",
"scripts/secure-runtime-rootful-runtime.sh",
"scripts/release_control/secure_runtime_rootful_source_manifest_v1.json",
"scripts/release_control/secure_runtime_rootless_attestation_v1.py",
"scripts/run-secure-runtime-rootful-qualification.sh",
@@ -37,6 +37,7 @@
"scripts/installtests/umask_unix_test.go",
"scripts/installtests/uninstall_sensor_proxy_test.go",
"scripts/release_control/secure_runtime_rootful_attestation_v1.py",
"scripts/secure-runtime-rootful-runtime.sh",
"scripts/release_control/secure_runtime_rootful_source_manifest_v1.json",
"scripts/release_control/secure_runtime_rootless_attestation_v1.py",
"scripts/run-secure-runtime-rootful-qualification.sh"
@@ -2,6 +2,8 @@
set -euo pipefail
readonly REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
# shellcheck source=scripts/secure-runtime-rootful-runtime.sh
source "${REPO_ROOT}/scripts/secure-runtime-rootful-runtime.sh"
readonly UBUNTU_IMAGE="${PULSE_ROOTFUL_UBUNTU_IMAGE:?set PULSE_ROOTFUL_UBUNTU_IMAGE to an immutable ubuntu@sha256:... Ubuntu 24.04 image}"
readonly OUTPUT_PARENT="${PULSE_ROOTFUL_QUALIFICATION_OUTPUT_DIR:?set PULSE_ROOTFUL_QUALIFICATION_OUTPUT_DIR to an existing absolute private directory}"
readonly CONFIRM="${PULSE_ROOTFUL_QUALIFICATION_CONFIRM:-}"
@@ -261,10 +263,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable\n' "\$(dpkg --print-architecture)" >/etc/apt/sources.list.d/docker.list && \
apt-get update && apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io && \
apt-get clean && rm -rf /var/lib/apt/lists/* && \
ln -sf /dev/null /etc/systemd/system/docker.service && \
ln -sf /dev/null /etc/systemd/system/docker.socket && \
ln -sf /dev/null /etc/systemd/system/podman.service && \
ln -sf /dev/null /etc/systemd/system/podman.socket && \
for unit in \
docker.service docker.socket \
containerd.service \
podman.service podman.socket \
podman-auto-update.service podman-auto-update.timer \
podman-clean-transient.service podman-restart.service; do \
ln -sf /dev/null "/etc/systemd/system/\${unit}"; \
done && \
install -d -m 0700 /opt/pulse/packet /opt/pulse/result && \
printf '%s\n' disposable-v1 >/etc/pulse-secure-runtime-rootful-qualification && \
rm -f /etc/machine-id && touch /etc/machine-id && \
@@ -284,13 +290,19 @@ capture_qualification_container_diagnostics() {
local runtime_name="$1" container_id="$2"
docker logs "${container_id}" >"${OUTPUT_DIR}/${runtime_name}-container.log" 2>&1 || true
docker exec "${container_id}" journalctl --no-pager -n 2000 >"${OUTPUT_DIR}/${runtime_name}-journal.log" 2>&1 || true
chmod 0600 "${OUTPUT_DIR}/${runtime_name}-container.log" "${OUTPUT_DIR}/${runtime_name}-journal.log"
docker exec "${container_id}" systemctl is-system-running >"${OUTPUT_DIR}/${runtime_name}-systemd-state.log" 2>&1 || true
docker exec "${container_id}" systemctl list-units --state=failed --no-legend --no-pager --plain >"${OUTPUT_DIR}/${runtime_name}-failed-units.log" 2>&1 || true
chmod 0600 \
"${OUTPUT_DIR}/${runtime_name}-container.log" \
"${OUTPUT_DIR}/${runtime_name}-journal.log" \
"${OUTPUT_DIR}/${runtime_name}-systemd-state.log" \
"${OUTPUT_DIR}/${runtime_name}-failed-units.log"
}
run_runtime() {
local runtime_name="$1"
local container_name="pulse-rootful-qual-${runtime_name}-${SOURCE_COMMIT:0:8}-$$"
local container_id local_receipt machine_id_file machine_id deadline mounts packet_probe_hash installed_probe_hash
local container_id local_receipt machine_id_file machine_id deadline mounts packet_probe_hash installed_probe_hash readiness_status
local_receipt="${OUTPUT_DIR}/${runtime_name}-receipt.json"
machine_id_file="${PACKET_DIR}/.machine-id-${runtime_name}"
machine_id="$(openssl rand -hex 16)"
@@ -313,7 +325,17 @@ run_runtime() {
docker start "${container_id}" >/dev/null
deadline=$((SECONDS + 60))
until docker exec "${container_id}" systemctl is-system-running --wait >/dev/null 2>&1; do
while true; do
readiness_status=0
rootful_qualification_systemd_readiness "${container_id}" || readiness_status=$?
if (( readiness_status == 0 )); then
break
fi
if (( readiness_status == 2 )); then
capture_qualification_container_diagnostics "${runtime_name}" "${container_id}"
echo "ERROR: ${runtime_name} disposable systemd container entered a terminal non-running state or has failed units" >&2
return 1
fi
if (( SECONDS >= deadline )); then
capture_qualification_container_diagnostics "${runtime_name}" "${container_id}"
echo "ERROR: ${runtime_name} disposable systemd container did not become ready" >&2
@@ -353,11 +375,19 @@ run_runtime() {
-e PULSE_SECURE_RUNTIME_INSTALLER=/opt/pulse/packet/install.sh \
"${container_id}" /opt/pulse/packet/dockeragent.test \
-test.run '^TestSecureRuntimeRootfulQualification$' -test.count=1 -test.v -test.timeout=45m \
| tee "${OUTPUT_DIR}/${runtime_name}-test.log"; then
| tee "${OUTPUT_DIR}/${runtime_name}-test.log"; then
capture_qualification_container_diagnostics "${runtime_name}" "${container_id}"
chmod 0600 "${OUTPUT_DIR}/${runtime_name}-test.log"
return 1
fi
readiness_status=0
rootful_qualification_systemd_readiness "${container_id}" || readiness_status=$?
if (( readiness_status != 0 )); then
capture_qualification_container_diagnostics "${runtime_name}" "${container_id}"
chmod 0600 "${OUTPUT_DIR}/${runtime_name}-test.log"
echo "ERROR: ${runtime_name} systemd readiness changed before receipt acceptance" >&2
return 1
fi
docker exec "${container_id}" test -f /opt/pulse/result/rootful-receipt.json || {
capture_qualification_container_diagnostics "${runtime_name}" "${container_id}"
echo "ERROR: ${runtime_name} qualification did not retain its receipt" >&2
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env bash
# This file is sourced by run-secure-runtime-rootful-qualification.sh and kept
# separately so the fail-closed systemd readiness predicate can be exercised
# without entering the destructive qualification wrapper.
rootful_qualification_systemd_readiness() {
local container_id="$1"
local manager_state manager_status=0 target_state failed_units
if manager_state="$(docker exec "${container_id}" systemctl is-system-running 2>/dev/null)"; then
manager_status=0
else
manager_status=$?
fi
case "${manager_state}" in
running)
if (( manager_status != 0 )); then
printf 'ERROR: systemd reported running with exit status %d\n' "${manager_status}" >&2
return 2
fi
;;
""|initializing|starting)
return 1
;;
*)
printf 'ERROR: disposable systemd container entered non-running state %q\n' "${manager_state}" >&2
return 2
;;
esac
target_state="$(docker exec "${container_id}" systemctl show \
--property=ActiveState --value multi-user.target 2>/dev/null)" || return 1
[[ "${target_state}" == "active" ]] || return 1
failed_units="$(docker exec "${container_id}" systemctl list-units \
--state=failed --no-legend --no-pager --plain 2>/dev/null)" || return 1
if [[ -n "${failed_units//[[:space:]]/}" ]]; then
printf 'ERROR: disposable systemd container has failed units:\n%s\n' "${failed_units}" >&2
return 2
fi
}