diff --git a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md index 75cedde86..f5e4cf8c3 100644 --- a/docs/release-control/v6/internal/subsystems/agent-lifecycle.md +++ b/docs/release-control/v6/internal/subsystems/agent-lifecycle.md @@ -7399,7 +7399,13 @@ the product default. `scripts/run-secure-runtime-rootful-qualification.sh` is the destructive, explicit entrypoint for real rootful Docker and Podman proof on two distinct disposable Ubuntu/systemd hosts. It never mounts a host daemon socket and runs -the outer containers without a default route. The packet binds exact clean Go +the outer containers without a default route. The host image masks Docker and +Podman's distro-managed service, socket, auto-update, transient-cleanup, and +restart units plus the distro containerd service before first boot; the packet +starts only its explicit runtime-specific daemon units, requires systemd's +exact running state with no failed units before and after each runtime packet, +and proves the explicit Docker daemon's child containerd exits with that +daemon. The packet binds exact clean Go artifacts, every compiled installtests package input, the canonical remote-main commit, the immutable Ubuntu base-image digest, and governed source hashes, then records fresh install, legacy migration with authority reduction, diff --git a/docs/release-control/v6/internal/subsystems/deployment-installability.md b/docs/release-control/v6/internal/subsystems/deployment-installability.md index f62068c0e..91e7124ad 100644 --- a/docs/release-control/v6/internal/subsystems/deployment-installability.md +++ b/docs/release-control/v6/internal/subsystems/deployment-installability.md @@ -326,7 +326,14 @@ Live rootful-runtime proof is also separated from the schema-v7 systemd packet. The explicit `scripts/run-secure-runtime-rootful-qualification.sh` wrapper creates distinct network-isolated Ubuntu/systemd hosts for real rootful Docker and Podman and -never mounts a host daemon socket. Its standalone schema-v1 receipt binds the +never mounts a host daemon socket. The disposable image masks the distro's +Docker and Podman service, socket, auto-update, transient-cleanup, and restart +units plus the distro containerd service before first boot, so exact systemd +running state cannot be confused with expected nested-runtime housekeeping +failures. Each scenario starts only its explicit runtime-specific daemon unit, +rechecks zero failed units before receipt acceptance, proves the explicit +Docker daemon's child containerd exits, and removes the complete Podman socket +boundary during cleanup. Its standalone schema-v1 receipt binds the exact canonical remote-main commit, immutable Ubuntu base-image digest, qualification, collector, helper, installer, every compiled installtests package input, source manifest, canonical root-owned socket, summary inventory, diff --git a/scripts/installtests/secure_runtime_rootful_qualification_test.go b/scripts/installtests/secure_runtime_rootful_qualification_test.go index 831070965..98a9b4513 100644 --- a/scripts/installtests/secure_runtime_rootful_qualification_test.go +++ b/scripts/installtests/secure_runtime_rootful_qualification_test.go @@ -97,6 +97,7 @@ type rootfulQualDaemon struct { unit string socket string dataRoot string + runRoot string fixture string } @@ -107,6 +108,7 @@ func TestSecureRuntimeRootfulQualification(t *testing.T) { runtimeKind := strings.TrimSpace(os.Getenv("PULSE_ROOTFUL_RUNTIME")) receiptPath := strings.TrimSpace(os.Getenv("PULSE_ROOTFUL_RECEIPT")) rootfulQualRequireDisposableHost(t, runtimeKind, receiptPath) + rootfulQualAssertSystemContainerdDisabled(t) collector := secureRuntimeReadArtifact(t, "PULSE_SECURE_RUNTIME_COLLECTOR") collectorSignature := secureRuntimeReadSignature(t, "PULSE_SECURE_RUNTIME_COLLECTOR_SIGNATURE") @@ -454,9 +456,9 @@ func rootfulQualRequireDisposableHost(t *testing.T, runtimeKind, receiptPath str func rootfulQualDaemonFor(runtimeKind string) rootfulQualDaemon { if runtimeKind == "docker" { - return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-docker", socket: "/var/run/docker.sock", dataRoot: "/var/lib/pulse-rootful-docker", fixture: "/opt/pulse/rootful-fixture"} + return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-docker", socket: "/var/run/docker.sock", dataRoot: "/var/lib/pulse-rootful-docker", runRoot: "/run/pulse-rootful-docker", fixture: "/opt/pulse/rootful-fixture"} } - return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-podman", socket: "/run/podman/podman.sock", dataRoot: "/var/lib/containers", fixture: "/opt/pulse/rootful-fixture"} + return rootfulQualDaemon{runtime: runtimeKind, unit: "pulse-rootful-podman", socket: "/run/podman/podman.sock", dataRoot: "/var/lib/pulse-rootful-podman", runRoot: "/run/pulse-rootful-podman", fixture: "/opt/pulse/rootful-fixture"} } func rootfulQualPrepareFixture(t *testing.T, daemon rootfulQualDaemon) { @@ -486,17 +488,61 @@ func rootfulQualStartDaemon(t *testing.T, daemon rootfulQualDaemon) { "--exec-root=/run/pulse-rootful-docker", "--pidfile=/run/pulse-rootful-docker.pid", "--storage-driver=vfs", "--iptables=false", "--bridge=none") } else { rootlessQualCommand(t, 20*time.Second, "systemd-run", "--quiet", "--collect", "--unit", daemon.unit, "--property=Type=exec", "--", - "/usr/bin/podman", "system", "service", "--time=0", "unix://"+daemon.socket) + "/usr/bin/podman", "--storage-driver=vfs", "--root="+daemon.dataRoot, "--runroot="+daemon.runRoot, + "system", "service", "--time=0", "unix://"+daemon.socket) } rootlessQualWaitSocket(t, daemon.socket) rootlessQualCommand(t, 10*time.Second, "chmod", "0660", daemon.socket) rootfulQualRuntimeCommand(t, daemon, 30*time.Second, "info") + if driver := rootfulQualRuntimeStorageDriver(t, daemon); driver != "vfs" { + t.Fatalf("rootful %s storage driver = %q, want vfs", daemon.runtime, driver) + } +} + +func rootfulQualRuntimeStorageDriver(t *testing.T, daemon rootfulQualDaemon) string { + t.Helper() + if daemon.runtime == "docker" { + return rootfulQualRuntimeCommand(t, daemon, 30*time.Second, "info", "--format", "{{.Driver}}") + } + return rootfulQualRuntimeCommand(t, daemon, 30*time.Second, "info", "--format", "{{.Store.GraphDriverName}}") } func rootfulQualStopDaemon(t *testing.T, daemon rootfulQualDaemon) { t.Helper() rootlessQualStopUnit(t, daemon.unit) _ = os.Remove(daemon.socket) + if daemon.runtime == "docker" { + rootfulQualWaitNoContainerd(t, 20*time.Second) + } +} + +func rootfulQualAssertSystemContainerdDisabled(t *testing.T) { + t.Helper() + unitState := rootlessQualCommand(t, 10*time.Second, "systemctl", "show", "containerd.service", "--property=UnitFileState", "--value") + activeState := rootlessQualCommand(t, 10*time.Second, "systemctl", "show", "containerd.service", "--property=ActiveState", "--value") + if unitState != "masked" || activeState != "inactive" { + t.Fatalf("distro containerd service must be masked and inactive: UnitFileState=%q ActiveState=%q", unitState, activeState) + } + rootfulQualWaitNoContainerd(t, 5*time.Second) +} + +func rootfulQualWaitNoContainerd(t *testing.T, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for { + output, err := rootlessQualCommandError(3*time.Second, "pgrep", "-x", "containerd") + if err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { + return + } + t.Fatalf("inspect containerd processes: %v\n%s", err, output) + } + if time.Now().After(deadline) { + t.Fatalf("containerd process remained after explicit Docker daemon stop: %s", strings.TrimSpace(output)) + } + time.Sleep(100 * time.Millisecond) + } } func rootfulQualRuntimeCommand(t *testing.T, daemon rootfulQualDaemon, timeout time.Duration, args ...string) string { @@ -697,7 +743,7 @@ func rootfulQualRunBoundProbe(t *testing.T, deadline time.Duration) time.Duratio func rootfulQualRemoveRuntimeState(t *testing.T, daemon rootfulQualDaemon) { t.Helper() - roots := []string{daemon.dataRoot, "/run/pulse-rootful-docker"} + roots := []string{daemon.dataRoot, daemon.runRoot} deadline := time.Now().Add(30 * time.Second) for { mountInfo := string(rootlessQualReadFile(t, "/proc/self/mountinfo")) @@ -713,15 +759,24 @@ func rootfulQualRemoveRuntimeState(t *testing.T, daemon rootfulQualDaemon) { } time.Sleep(100 * time.Millisecond) } - for _, path := range []string{daemon.dataRoot, "/run/pulse-rootful-docker", daemon.fixture} { + for _, path := range []string{daemon.dataRoot, daemon.runRoot, daemon.fixture} { if err := os.RemoveAll(path); err != nil { t.Fatalf("remove disposable runtime path %s: %v", path, err) } } + if daemon.runtime == "podman" { + if err := os.Remove(filepath.Dir(daemon.socket)); err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatalf("remove disposable Podman socket directory: %v", err) + } + } } func rootfulQualRuntimeStateClean(daemon rootfulQualDaemon) bool { - for _, path := range []string{daemon.socket, daemon.dataRoot, "/run/pulse-rootful-docker", daemon.fixture} { + paths := []string{daemon.socket, daemon.dataRoot, daemon.runRoot, daemon.fixture} + if daemon.runtime == "podman" { + paths = append(paths, filepath.Dir(daemon.socket)) + } + for _, path := range paths { if _, err := os.Lstat(path); !errors.Is(err, os.ErrNotExist) { return false } @@ -1001,6 +1056,10 @@ func TestRootfulQualificationWrapperInvariants(t *testing.T) { "github.com/rcourtman/pulse-go-rewrite/scripts/installtests.test", "https://github.com/rcourtman/Pulse.git", "refs/remotes/origin/main", "refs/heads/main", "PULSE_ROOTFUL_UBUNTU_IMAGE", "PULSE_ROOTFUL_BOUND_PROBE_BINARY", + "rootful_qualification_systemd_readiness", + "containerd.service", + "podman-auto-update.service", "podman-auto-update.timer", + "podman-clean-transient.service", "podman-restart.service", rootfulQualBoundProbe, } { if !strings.Contains(script, required) { @@ -1016,3 +1075,72 @@ func TestRootfulQualificationWrapperInvariants(t *testing.T) { } } } + +func TestRootfulQualificationSystemdReadiness(t *testing.T) { + runtimeScript := repoFile("scripts", "secure-runtime-rootful-runtime.sh") + binDir := t.TempDir() + fakeDocker := filepath.Join(binDir, "docker") + fake := `#!/bin/sh +case "$*" in + *"systemctl is-system-running"*) + printf '%s\n' "${FAKE_SYSTEMD_MANAGER_STATE:-starting}" + [ "${FAKE_SYSTEMD_MANAGER_STATE:-starting}" = running ] + ;; + *"systemctl show --property=ActiveState --value multi-user.target"*) + printf '%s\n' "${FAKE_SYSTEMD_TARGET_STATE:-inactive}" + ;; + *"systemctl list-units --state=failed --no-legend --no-pager --plain"*) + printf '%s' "${FAKE_SYSTEMD_FAILED_UNITS:-}" + ;; + *) + printf 'unexpected docker arguments: %s\n' "$*" >&2 + exit 99 + ;; +esac +` + if err := os.WriteFile(fakeDocker, []byte(fake), 0o700); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + managerState string + targetState string + failedUnits string + wantExit int + wantOutput string + }{ + {name: "manager starting", managerState: "starting", targetState: "active", wantExit: 1}, + {name: "target inactive", managerState: "running", targetState: "inactive", wantExit: 1}, + {name: "clean running manager", managerState: "running", targetState: "active", wantExit: 0}, + {name: "degraded manager", managerState: "degraded", targetState: "active", wantExit: 2, wantOutput: "degraded"}, + {name: "maintenance manager", managerState: "maintenance", targetState: "active", wantExit: 2, wantOutput: "maintenance"}, + {name: "failed unit", managerState: "running", targetState: "active", failedUnits: "podman-restart.service loaded failed failed", wantExit: 2, wantOutput: "podman-restart.service"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := exec.Command("bash", "-c", `source "$1"; rootful_qualification_systemd_readiness fixture`, "bash", runtimeScript) + cmd.Env = append(os.Environ(), + "PATH="+binDir+":"+os.Getenv("PATH"), + "FAKE_SYSTEMD_MANAGER_STATE="+test.managerState, + "FAKE_SYSTEMD_TARGET_STATE="+test.targetState, + "FAKE_SYSTEMD_FAILED_UNITS="+test.failedUnits, + ) + output, err := cmd.CombinedOutput() + gotExit := 0 + if err != nil { + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("readiness helper failed without an exit status: %v", err) + } + gotExit = exitErr.ExitCode() + } + if gotExit != test.wantExit { + t.Fatalf("readiness exit = %d, want %d\n%s", gotExit, test.wantExit, output) + } + if test.wantOutput != "" && !strings.Contains(string(output), test.wantOutput) { + t.Fatalf("readiness output missing %q: %s", test.wantOutput, output) + } + }) + } +} diff --git a/scripts/release_control/secure_runtime_rootful_attestation_v1_test.py b/scripts/release_control/secure_runtime_rootful_attestation_v1_test.py index efe64e636..beb40af50 100644 --- a/scripts/release_control/secure_runtime_rootful_attestation_v1_test.py +++ b/scripts/release_control/secure_runtime_rootful_attestation_v1_test.py @@ -493,6 +493,7 @@ if module.MAX_RECEIPT_BYTES <= 0: "scripts/installtests/secure_runtime_rootless_qualification_test.go", "scripts/installtests/secure_runtime_systemd_lab_test.go", "scripts/release_control/secure_runtime_rootful_attestation_v1.py", + "scripts/secure-runtime-rootful-runtime.sh", "scripts/release_control/secure_runtime_rootful_source_manifest_v1.json", "scripts/release_control/secure_runtime_rootless_attestation_v1.py", "scripts/run-secure-runtime-rootful-qualification.sh", diff --git a/scripts/release_control/secure_runtime_rootful_source_manifest_v1.json b/scripts/release_control/secure_runtime_rootful_source_manifest_v1.json index b5a76c0e7..0ccc60714 100644 --- a/scripts/release_control/secure_runtime_rootful_source_manifest_v1.json +++ b/scripts/release_control/secure_runtime_rootful_source_manifest_v1.json @@ -37,6 +37,7 @@ "scripts/installtests/umask_unix_test.go", "scripts/installtests/uninstall_sensor_proxy_test.go", "scripts/release_control/secure_runtime_rootful_attestation_v1.py", + "scripts/secure-runtime-rootful-runtime.sh", "scripts/release_control/secure_runtime_rootful_source_manifest_v1.json", "scripts/release_control/secure_runtime_rootless_attestation_v1.py", "scripts/run-secure-runtime-rootful-qualification.sh" diff --git a/scripts/run-secure-runtime-rootful-qualification.sh b/scripts/run-secure-runtime-rootful-qualification.sh index f1bca2dc8..70041c190 100755 --- a/scripts/run-secure-runtime-rootful-qualification.sh +++ b/scripts/run-secure-runtime-rootful-qualification.sh @@ -2,6 +2,8 @@ set -euo pipefail readonly REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +# shellcheck source=scripts/secure-runtime-rootful-runtime.sh +source "${REPO_ROOT}/scripts/secure-runtime-rootful-runtime.sh" readonly UBUNTU_IMAGE="${PULSE_ROOTFUL_UBUNTU_IMAGE:?set PULSE_ROOTFUL_UBUNTU_IMAGE to an immutable ubuntu@sha256:... Ubuntu 24.04 image}" readonly OUTPUT_PARENT="${PULSE_ROOTFUL_QUALIFICATION_OUTPUT_DIR:?set PULSE_ROOTFUL_QUALIFICATION_OUTPUT_DIR to an existing absolute private directory}" readonly CONFIRM="${PULSE_ROOTFUL_QUALIFICATION_CONFIRM:-}" @@ -261,10 +263,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu noble stable\n' "\$(dpkg --print-architecture)" >/etc/apt/sources.list.d/docker.list && \ apt-get update && apt-get install -y --no-install-recommends docker-ce docker-ce-cli containerd.io && \ apt-get clean && rm -rf /var/lib/apt/lists/* && \ - ln -sf /dev/null /etc/systemd/system/docker.service && \ - ln -sf /dev/null /etc/systemd/system/docker.socket && \ - ln -sf /dev/null /etc/systemd/system/podman.service && \ - ln -sf /dev/null /etc/systemd/system/podman.socket && \ + for unit in \ + docker.service docker.socket \ + containerd.service \ + podman.service podman.socket \ + podman-auto-update.service podman-auto-update.timer \ + podman-clean-transient.service podman-restart.service; do \ + ln -sf /dev/null "/etc/systemd/system/\${unit}"; \ + done && \ install -d -m 0700 /opt/pulse/packet /opt/pulse/result && \ printf '%s\n' disposable-v1 >/etc/pulse-secure-runtime-rootful-qualification && \ rm -f /etc/machine-id && touch /etc/machine-id && \ @@ -284,13 +290,19 @@ capture_qualification_container_diagnostics() { local runtime_name="$1" container_id="$2" docker logs "${container_id}" >"${OUTPUT_DIR}/${runtime_name}-container.log" 2>&1 || true docker exec "${container_id}" journalctl --no-pager -n 2000 >"${OUTPUT_DIR}/${runtime_name}-journal.log" 2>&1 || true - chmod 0600 "${OUTPUT_DIR}/${runtime_name}-container.log" "${OUTPUT_DIR}/${runtime_name}-journal.log" + docker exec "${container_id}" systemctl is-system-running >"${OUTPUT_DIR}/${runtime_name}-systemd-state.log" 2>&1 || true + docker exec "${container_id}" systemctl list-units --state=failed --no-legend --no-pager --plain >"${OUTPUT_DIR}/${runtime_name}-failed-units.log" 2>&1 || true + chmod 0600 \ + "${OUTPUT_DIR}/${runtime_name}-container.log" \ + "${OUTPUT_DIR}/${runtime_name}-journal.log" \ + "${OUTPUT_DIR}/${runtime_name}-systemd-state.log" \ + "${OUTPUT_DIR}/${runtime_name}-failed-units.log" } run_runtime() { local runtime_name="$1" local container_name="pulse-rootful-qual-${runtime_name}-${SOURCE_COMMIT:0:8}-$$" - local container_id local_receipt machine_id_file machine_id deadline mounts packet_probe_hash installed_probe_hash + local container_id local_receipt machine_id_file machine_id deadline mounts packet_probe_hash installed_probe_hash readiness_status local_receipt="${OUTPUT_DIR}/${runtime_name}-receipt.json" machine_id_file="${PACKET_DIR}/.machine-id-${runtime_name}" machine_id="$(openssl rand -hex 16)" @@ -313,7 +325,17 @@ run_runtime() { docker start "${container_id}" >/dev/null deadline=$((SECONDS + 60)) - until docker exec "${container_id}" systemctl is-system-running --wait >/dev/null 2>&1; do + while true; do + readiness_status=0 + rootful_qualification_systemd_readiness "${container_id}" || readiness_status=$? + if (( readiness_status == 0 )); then + break + fi + if (( readiness_status == 2 )); then + capture_qualification_container_diagnostics "${runtime_name}" "${container_id}" + echo "ERROR: ${runtime_name} disposable systemd container entered a terminal non-running state or has failed units" >&2 + return 1 + fi if (( SECONDS >= deadline )); then capture_qualification_container_diagnostics "${runtime_name}" "${container_id}" echo "ERROR: ${runtime_name} disposable systemd container did not become ready" >&2 @@ -353,11 +375,19 @@ run_runtime() { -e PULSE_SECURE_RUNTIME_INSTALLER=/opt/pulse/packet/install.sh \ "${container_id}" /opt/pulse/packet/dockeragent.test \ -test.run '^TestSecureRuntimeRootfulQualification$' -test.count=1 -test.v -test.timeout=45m \ - | tee "${OUTPUT_DIR}/${runtime_name}-test.log"; then + | tee "${OUTPUT_DIR}/${runtime_name}-test.log"; then capture_qualification_container_diagnostics "${runtime_name}" "${container_id}" chmod 0600 "${OUTPUT_DIR}/${runtime_name}-test.log" return 1 fi + readiness_status=0 + rootful_qualification_systemd_readiness "${container_id}" || readiness_status=$? + if (( readiness_status != 0 )); then + capture_qualification_container_diagnostics "${runtime_name}" "${container_id}" + chmod 0600 "${OUTPUT_DIR}/${runtime_name}-test.log" + echo "ERROR: ${runtime_name} systemd readiness changed before receipt acceptance" >&2 + return 1 + fi docker exec "${container_id}" test -f /opt/pulse/result/rootful-receipt.json || { capture_qualification_container_diagnostics "${runtime_name}" "${container_id}" echo "ERROR: ${runtime_name} qualification did not retain its receipt" >&2 diff --git a/scripts/secure-runtime-rootful-runtime.sh b/scripts/secure-runtime-rootful-runtime.sh new file mode 100644 index 000000000..0627b96e9 --- /dev/null +++ b/scripts/secure-runtime-rootful-runtime.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# This file is sourced by run-secure-runtime-rootful-qualification.sh and kept +# separately so the fail-closed systemd readiness predicate can be exercised +# without entering the destructive qualification wrapper. + +rootful_qualification_systemd_readiness() { + local container_id="$1" + local manager_state manager_status=0 target_state failed_units + + if manager_state="$(docker exec "${container_id}" systemctl is-system-running 2>/dev/null)"; then + manager_status=0 + else + manager_status=$? + fi + case "${manager_state}" in + running) + if (( manager_status != 0 )); then + printf 'ERROR: systemd reported running with exit status %d\n' "${manager_status}" >&2 + return 2 + fi + ;; + ""|initializing|starting) + return 1 + ;; + *) + printf 'ERROR: disposable systemd container entered non-running state %q\n' "${manager_state}" >&2 + return 2 + ;; + esac + + target_state="$(docker exec "${container_id}" systemctl show \ + --property=ActiveState --value multi-user.target 2>/dev/null)" || return 1 + [[ "${target_state}" == "active" ]] || return 1 + + failed_units="$(docker exec "${container_id}" systemctl list-units \ + --state=failed --no-legend --no-pager --plain 2>/dev/null)" || return 1 + if [[ -n "${failed_units//[[:space:]]/}" ]]; then + printf 'ERROR: disposable systemd container has failed units:\n%s\n' "${failed_units}" >&2 + return 2 + fi +}