Harden rootful qualification evidence boundary

This commit is contained in:
rcourtman
2026-09-01 23:53:36 +01:00
parent 584cef81a1
commit 500cc1bf17
7 changed files with 184 additions and 25 deletions
@@ -7400,11 +7400,14 @@ the product default.
explicit entrypoint for real rootful Docker and Podman proof on two distinct
disposable Ubuntu/systemd hosts. It never mounts a host daemon socket and runs
the outer containers without a default route. The packet binds exact clean Go
artifacts and governed source hashes, then records fresh install, legacy
migration with authority reduction, collector and helper restart continuity,
helper loss without an authoritative empty replacement, exact recovery,
bounded helper-operation failure, ordinary collector-update preservation,
authority isolation, and cleanup for both runtimes. Rootful telemetry is
artifacts, every compiled installtests package input, the canonical remote-main
commit, the immutable Ubuntu base-image digest, and governed source hashes,
then records fresh install, legacy migration with authority reduction,
collector and helper restart continuity, helper loss without an authoritative
empty replacement, exact recovery, bounded helper-operation failure through a
root-owned collector-executable copy of the qualification binary, ordinary
collector-update preservation, authority isolation, and cleanup for both
runtimes. Rootful telemetry is
intentionally summary-only: the collector remains unable to open the
root-owned daemon socket and gains neither container actions nor updates.
@@ -327,8 +327,10 @@ packet. The explicit
`scripts/run-secure-runtime-rootful-qualification.sh` wrapper creates distinct
network-isolated Ubuntu/systemd hosts for real rootful Docker and Podman and
never mounts a host daemon socket. Its standalone schema-v1 receipt binds the
exact qualification, collector, helper, installer, source manifest, canonical
root-owned socket, summary inventory, migration, restart, loss, recovery,
exact canonical remote-main commit, immutable Ubuntu base-image digest,
qualification, collector, helper, installer, every compiled installtests
package input, source manifest, canonical root-owned socket, summary inventory,
migration, restart, loss, recovery,
bounded operation, update-preservation, authority-isolation, and cleanup
evidence before the independent validator can emit a local artifact-bound
self-attestation. All Go artifacts require exact clean VCS metadata and all
@@ -17,6 +17,7 @@ import (
"os"
"os/exec"
"path/filepath"
"regexp"
"runtime"
"slices"
"strconv"
@@ -40,26 +41,31 @@ const (
rootfulQualRunningName = "pulse-rootful-running"
rootfulQualExitedName = "pulse-rootful-exited"
rootfulQualHelperSock = "/run/pulse-agent/helper.sock"
rootfulQualBoundProbe = "/usr/local/libexec/pulse-rootful-qualification/dockeragent.test"
)
var rootfulQualScenarioOrder = []string{
"fresh_install",
"legacy_migration",
"collector_restart",
"helper_restart",
"helper_loss",
"helper_recovery",
"operation_bounds",
"update_preservation",
"authority_isolation",
"cleanup",
}
var (
rootfulQualBaseImagePattern = regexp.MustCompile(`^ubuntu@sha256:[0-9a-f]{64}$`)
rootfulQualScenarioOrder = []string{
"fresh_install",
"legacy_migration",
"collector_restart",
"helper_restart",
"helper_loss",
"helper_recovery",
"operation_bounds",
"update_preservation",
"authority_isolation",
"cleanup",
}
)
type rootfulQualReceipt struct {
SchemaVersion int `json:"schema_version"`
Kind string `json:"kind"`
Result string `json:"result"`
SourceCommit string `json:"source_commit"`
BaseImage string `json:"base_image"`
StartedAt string `json:"started_at"`
CompletedAt string `json:"completed_at"`
SourceHashes map[string]string `json:"source_hashes"`
@@ -357,6 +363,7 @@ func TestSecureRuntimeRootfulQualification(t *testing.T) {
receipt := rootfulQualReceipt{
SchemaVersion: 1, Kind: "pulse-secure-runtime-rootful-qualification", Result: "passed",
SourceCommit: strings.TrimSpace(os.Getenv("PULSE_ROOTFUL_SOURCE_COMMIT")),
BaseImage: strings.TrimSpace(os.Getenv("PULSE_ROOTFUL_UBUNTU_IMAGE")),
StartedAt: started.Format(time.RFC3339Nano), CompletedAt: time.Now().UTC().Format(time.RFC3339Nano),
SourceHashes: rootfulQualSourceHashes(t), Artifacts: rootfulQualArtifactIdentities(t, installerPath),
Runs: []rootfulQualRun{{
@@ -653,12 +660,30 @@ func rootfulQualRunBoundProbe(t *testing.T, deadline time.Duration) time.Duratio
if err != nil {
t.Fatal(err)
}
boundProbe := strings.TrimSpace(os.Getenv("PULSE_ROOTFUL_BOUND_PROBE_BINARY"))
if boundProbe != rootfulQualBoundProbe {
t.Fatalf("PULSE_ROOTFUL_BOUND_PROBE_BINARY must use %q: %q", rootfulQualBoundProbe, boundProbe)
}
if secureRuntimeHash(rootlessQualReadFile(t, boundProbe)) != secureRuntimeHash(rootlessQualReadFile(t, executable)) {
t.Fatal("collector-executable bound probe differs from the qualification binary")
}
for _, path := range []string{filepath.Dir(boundProbe), boundProbe} {
info, statErr := os.Lstat(path)
if statErr != nil {
t.Fatalf("stat bound-probe path %s: %v", path, statErr)
}
statInfo, ok := info.Sys().(*syscall.Stat_t)
isProbe := path == boundProbe
if !ok || statInfo.Uid != 0 || info.Mode().Perm() != 0o755 || isProbe && !info.Mode().IsRegular() || !isProbe && !info.IsDir() {
t.Fatalf("bound-probe path is not root-owned mode 0755 with a regular executable: %s %+v", path, info)
}
}
ctx, cancel := context.WithTimeout(context.Background(), deadline+3*time.Second)
defer cancel()
started := time.Now()
cmd := exec.CommandContext(ctx, "runuser", "-u", "pulse-agent", "--", "env",
"PULSE_ROOTFUL_BOUND_PROBE=1", fmt.Sprintf("PULSE_ROOTFUL_BOUND_DEADLINE_MS=%d", deadline.Milliseconds()),
executable, "-test.run", "^TestSecureRuntimeRootfulBoundProbe$", "-test.count=1", "-test.v", "-test.timeout=10s")
boundProbe, "-test.run", "^TestSecureRuntimeRootfulBoundProbe$", "-test.count=1", "-test.v", "-test.timeout=10s")
output, err := cmd.CombinedOutput()
elapsed := time.Since(started)
if err != nil || !strings.Contains(string(output), "ROOTFUL_BOUND_RESULT=deadline_exceeded") {
@@ -760,7 +785,7 @@ func rootfulQualValidateReceipt(receipt rootfulQualReceipt, expectedRuns int) er
if receipt.SchemaVersion != 1 || receipt.Kind != "pulse-secure-runtime-rootful-qualification" || receipt.Result != "passed" {
return errors.New("invalid rootful qualification identity")
}
if len(receipt.SourceCommit) != 40 || receipt.StartedAt == "" || receipt.CompletedAt == "" || len(receipt.SourceHashes) == 0 || len(receipt.Runs) != expectedRuns {
if len(receipt.SourceCommit) != 40 || !rootfulQualBaseImagePattern.MatchString(receipt.BaseImage) || receipt.StartedAt == "" || receipt.CompletedAt == "" || len(receipt.SourceHashes) == 0 || len(receipt.Runs) != expectedRuns {
return errors.New("incomplete rootful qualification envelope")
}
for _, run := range receipt.Runs {
@@ -801,6 +826,7 @@ func TestRootfulQualificationReceiptContract(t *testing.T) {
receipt := rootfulQualReceipt{
SchemaVersion: 1, Kind: "pulse-secure-runtime-rootful-qualification", Result: "passed",
SourceCommit: strings.Repeat("a", 40), StartedAt: time.Now().UTC().Format(time.RFC3339Nano), CompletedAt: time.Now().UTC().Format(time.RFC3339Nano),
BaseImage: "ubuntu@sha256:" + strings.Repeat("c", 64),
SourceHashes: map[string]string{"go.mod": strings.Repeat("b", 64)},
Runs: []rootfulQualRun{{Host: rootlessQualHost{MachineID: strings.Repeat("1", 32)}, Runtime: rootfulQualRuntime{Runtime: "docker", RuntimeVersion: "1", DaemonID: "daemon", SocketPath: "/var/run/docker.sock", SocketUID: 0, SocketGID: 999, SocketMode: "0660", SocketType: "unix"}, Scenarios: scenarios}},
}
@@ -820,6 +846,7 @@ func TestRootfulQualificationGoSchemaPassesPythonValidator(t *testing.T) {
receipt := rootfulQualReceipt{
SchemaVersion: 1, Kind: "pulse-secure-runtime-rootful-qualification", Result: "passed",
SourceCommit: commit, StartedAt: started.Format(time.RFC3339Nano),
BaseImage: "ubuntu@sha256:" + strings.Repeat("c", 64),
CompletedAt: started.Add(2 * time.Minute).Format(time.RFC3339Nano),
SourceHashes: map[string]string{"internal/agenthelper/container_inventory.go": digest, "scripts/install.sh": digest},
Artifacts: rootlessQualArtifacts{
@@ -972,6 +999,9 @@ func TestRootfulQualificationWrapperInvariants(t *testing.T) {
"capture_qualification_container_diagnostics", "journalctl --no-pager -n 2000",
"org.pulse.rootful-qualification.run", "-buildvcs=true",
"github.com/rcourtman/pulse-go-rewrite/scripts/installtests.test",
"https://github.com/rcourtman/Pulse.git", "refs/remotes/origin/main", "refs/heads/main",
"PULSE_ROOTFUL_UBUNTU_IMAGE", "PULSE_ROOTFUL_BOUND_PROBE_BINARY",
rootfulQualBoundProbe,
} {
if !strings.Contains(script, required) {
t.Fatalf("rootful qualification wrapper missing %q", required)
@@ -64,6 +64,7 @@ IDENTITY_RE = hardened.IDENTITY_RE
VERSION_RE = hardened.VERSION_RE
GO_VERSION_RE = hardened.GO_VERSION_RE
COMMIT_RE = hardened.COMMIT_RE
BASE_IMAGE_RE = re.compile(r"^ubuntu@sha256:[0-9a-f]{64}$")
SUMMARY_KEYS = {
@@ -357,11 +358,12 @@ def validate_run(run_value: Any, expected_runtime: str, index: int, receipt_star
def validate_receipt(receipt: Any) -> dict[str, Any]:
reject_sensitive_evidence(receipt)
root = require_object(receipt, "receipt", {"schema_version", "kind", "result", "source_commit", "started_at", "completed_at", "artifacts", "source_hashes", "runs"})
root = require_object(receipt, "receipt", {"schema_version", "kind", "result", "source_commit", "base_image", "started_at", "completed_at", "artifacts", "source_hashes", "runs"})
require(type(root["schema_version"]) is int and root["schema_version"] == RECEIPT_SCHEMA_VERSION, "receipt.schema_version must be 1")
require(root["kind"] == RECEIPT_KIND, f"receipt.kind must be {RECEIPT_KIND}")
require(root["result"] == "passed", "receipt.result must be passed")
source_commit = require_text(root["source_commit"], "receipt.source_commit", pattern=COMMIT_RE, maximum=40)
require_text(root["base_image"], "receipt.base_image", pattern=BASE_IMAGE_RE, maximum=78)
started = parse_timestamp(root["started_at"], "receipt.started_at")
completed = parse_timestamp(root["completed_at"], "receipt.completed_at")
require(started < completed, "receipt chronology is invalid")
@@ -531,6 +533,7 @@ def create_attestation(
"source_manifest_sha256": hashlib.sha256(manifest_bytes).hexdigest(),
"source_hash_count": len(source_hashes),
"source_commit": receipt["source_commit"],
"qualified_base_image": receipt["base_image"],
"source_commit_verified": verify_git_commit,
"artifact_bindings": bindings,
"validated_runtimes": list(REQUIRED_RUNTIMES),
@@ -270,6 +270,7 @@ class RootfulAttestationV1Test(unittest.TestCase):
"kind": attester.RECEIPT_KIND,
"result": "passed",
"source_commit": self.commit,
"base_image": "ubuntu@sha256:" + "c" * 64,
"started_at": self.ts(0),
"completed_at": self.ts(50),
"source_hashes": {name: sha(data) for name, data in self.sources.items()},
@@ -314,6 +315,7 @@ class RootfulAttestationV1Test(unittest.TestCase):
result = self.attest()
self.assertEqual(result["validated_runtimes"], ["docker", "podman"])
self.assertEqual(result["classification"], attester.CLASSIFICATION)
self.assertEqual(result["qualified_base_image"], self.receipt["base_image"])
self.assertEqual(set(result["artifact_bindings"]), set(self.artifacts))
self.assertEqual(
result["limitations"],
@@ -328,6 +330,7 @@ class RootfulAttestationV1Test(unittest.TestCase):
def test_exact_runtime_topology_and_order_fail_closed(self) -> None:
mutations = [
lambda r: r.update(schema_version=True),
lambda r: r.update(base_image="ubuntu:latest"),
lambda r: r["runs"].pop(),
lambda r: r["runs"].reverse(),
lambda r: r["runs"][1]["host"].update(machine_id=r["runs"][0]["host"]["machine_id"]),
@@ -467,6 +470,37 @@ class RootfulAttestationV1Test(unittest.TestCase):
"scripts/run-secure-runtime-rootful-qualification.sh",
}
self.assertTrue(required.issubset(manifest["exact_paths"]))
repo_root = Path(__file__).resolve().parents[2]
compiled_test_inputs = {
path.relative_to(repo_root).as_posix()
for path in (repo_root / "scripts" / "installtests").glob("*_test.go")
}
self.assertTrue(compiled_test_inputs.issubset(manifest["exact_paths"]))
go_list = subprocess.run(
[
"go",
"list",
"-deps",
"-test",
"-f",
'{{if and (not .Standard) .Module}}{{if eq .Module.Path "github.com/rcourtman/pulse-go-rewrite"}}{{.Dir}}{{end}}{{end}}',
"./scripts/installtests",
],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
)
recursive_roots = set(manifest["recursive_roots"])
for raw_directory in go_list.stdout.splitlines():
directory = Path(raw_directory)
if not raw_directory or directory == repo_root / "scripts" / "installtests":
continue
relative = directory.relative_to(repo_root).as_posix()
self.assertTrue(
any(relative == root or relative.startswith(root + "/") for root in recursive_roots),
f"compiled qualification dependency is outside the source manifest: {relative}",
)
self.assertIn("internal/agenthelper", manifest["recursive_roots"])
self.assertIn("pkg/auth", manifest["recursive_roots"])
@@ -2,7 +2,7 @@
"schema_version": 1,
"manifest_id": "secure-runtime-rootful-v1",
"target_os": "linux",
"description": "Production source boundary for rootful Docker and Podman typed-helper summary inventory, loss and recovery, operation bounds, update preservation, installer migration, and the standalone rootful qualification packet.",
"description": "Production and complete repo-local compiled source boundary for rootful Docker and Podman typed-helper summary inventory, loss and recovery, operation bounds, update preservation, installer migration, and the standalone rootful qualification packet.",
"exact_paths": [
"VERSION",
"go.mod",
@@ -11,10 +11,31 @@
"pkg/agents/docker/report_limits.go",
"scripts/install.sh",
"scripts/release_ldflags.sh",
"scripts/installtests/agent_state_dir_lifecycle_test.go",
"scripts/installtests/backfill_release_assets_test.go",
"scripts/installtests/build_release_assets_test.go",
"scripts/installtests/container_dependency_contract_test.go",
"scripts/installtests/dependency_update_test.go",
"scripts/installtests/docker_entrypoint_test.go",
"scripts/installtests/install_docker_sh_test.go",
"scripts/installtests/install_mcp_test.go",
"scripts/installtests/install_ps1_test.go",
"scripts/installtests/install_sh_qnap_data_volume_test.go",
"scripts/installtests/install_sh_test.go",
"scripts/installtests/integration_container_test.go",
"scripts/installtests/native_pve_action_qualification_test.go",
"scripts/installtests/provider_msp_deploy_test.go",
"scripts/installtests/pulse_auto_update_test.go",
"scripts/installtests/release_ldflags_test.go",
"scripts/installtests/root_install_sh_test.go",
"scripts/installtests/safe_profile_migration_test.go",
"scripts/installtests/secure_runtime_platform_matrix_test.go",
"scripts/installtests/secure_runtime_rootful_qualification_test.go",
"scripts/installtests/secure_runtime_rootless_qualification_test.go",
"scripts/installtests/secure_runtime_systemd_lab_test.go",
"scripts/installtests/testdata/secure_runtime_docker_fixture.go",
"scripts/installtests/umask_unix_test.go",
"scripts/installtests/uninstall_sensor_proxy_test.go",
"scripts/release_control/secure_runtime_rootful_attestation_v1.py",
"scripts/release_control/secure_runtime_rootful_source_manifest_v1.json",
"scripts/release_control/secure_runtime_rootless_attestation_v1.py",
@@ -24,20 +45,57 @@
"cmd/pulse-agent",
"cmd/pulse-agent-helper",
"internal/actionrunner",
"internal/agentcapabilities",
"internal/agentexec",
"internal/agenthelper",
"internal/agenttarget",
"internal/agenttls",
"internal/agentupdate",
"internal/ai/memory",
"internal/alerts",
"internal/api",
"internal/availabilityprobe",
"internal/collectorlifecycle",
"internal/config",
"internal/crypto",
"internal/dockeragent",
"internal/hostagent",
"internal/hostmetrics",
"internal/logging",
"internal/mock",
"internal/mockmode",
"internal/mockmodel",
"internal/mockruntime",
"internal/models",
"internal/notifications",
"internal/operationaltrust",
"internal/operationreceipt",
"internal/platformsupport",
"internal/proxmoxidentity",
"internal/recovery",
"internal/relay",
"internal/remoteconfig",
"internal/securityutil",
"internal/sensors",
"internal/ssh/knownhosts",
"internal/storagehealth",
"internal/truenas",
"internal/unifiedresources",
"internal/unraid",
"internal/updatesignature",
"internal/utils",
"internal/vmware",
"pkg/auth",
"pkg/agents/docker",
"pkg/agents/host",
"pkg/agents/kubernetes",
"pkg/aicontracts",
"pkg/diskinventory",
"pkg/fsfilters",
"pkg/licensing",
"pkg/pbs",
"pkg/pmg",
"pkg/proxmox",
"pkg/securityutil",
"pkg/tlsutil"
],
@@ -5,6 +5,8 @@ readonly REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
readonly UBUNTU_IMAGE="${PULSE_ROOTFUL_UBUNTU_IMAGE:?set PULSE_ROOTFUL_UBUNTU_IMAGE to an immutable ubuntu@sha256:... Ubuntu 24.04 image}"
readonly OUTPUT_PARENT="${PULSE_ROOTFUL_QUALIFICATION_OUTPUT_DIR:?set PULSE_ROOTFUL_QUALIFICATION_OUTPUT_DIR to an existing absolute private directory}"
readonly CONFIRM="${PULSE_ROOTFUL_QUALIFICATION_CONFIRM:-}"
readonly CANONICAL_ORIGIN_URL="https://github.com/rcourtman/Pulse.git"
readonly BOUND_PROBE_PATH="/usr/local/libexec/pulse-rootful-qualification/dockeragent.test"
portable_mode() {
stat -c '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1"
@@ -44,6 +46,22 @@ if [[ -n "$(git -C "${REPO_ROOT}" status --porcelain)" ]]; then
fi
readonly SOURCE_COMMIT="$(git -C "${REPO_ROOT}" rev-parse HEAD)"
origin_url="$(git -C "${REPO_ROOT}" remote get-url origin)"
if [[ "${origin_url}" != "${CANONICAL_ORIGIN_URL}" ]]; then
echo "ERROR: qualification requires the canonical Pulse origin URL" >&2
exit 2
fi
origin_main="$(git -C "${REPO_ROOT}" rev-parse refs/remotes/origin/main)"
remote_main_record="$(git -C "${REPO_ROOT}" ls-remote --exit-code origin refs/heads/main)"
if [[ ! "${remote_main_record}" =~ ^([0-9a-f]{40})$'\t'refs/heads/main$ ]]; then
echo "ERROR: canonical remote main lookup returned an unexpected result" >&2
exit 2
fi
remote_main="${BASH_REMATCH[1]}"
if [[ "${SOURCE_COMMIT}" != "${origin_main}" || "${SOURCE_COMMIT}" != "${remote_main}" ]]; then
echo "ERROR: qualification requires HEAD, origin/main, and canonical remote main to be identical" >&2
exit 2
fi
readonly EXPECTED_CONFIRM="I_HAVE_VERIFIED_THESE_ARE_DISPOSABLE_ROOTFUL_SYSTEMD_CONTAINERS_COMMIT_${SOURCE_COMMIT}"
if [[ "${CONFIRM}" != "${EXPECTED_CONFIRM}" ]]; then
echo "ERROR: exact destructive opt-in required:" >&2
@@ -272,7 +290,7 @@ capture_qualification_container_diagnostics() {
run_runtime() {
local runtime_name="$1"
local container_name="pulse-rootful-qual-${runtime_name}-${SOURCE_COMMIT:0:8}-$$"
local container_id local_receipt machine_id_file machine_id deadline mounts
local container_id local_receipt machine_id_file machine_id deadline mounts packet_probe_hash installed_probe_hash
local_receipt="${OUTPUT_DIR}/${runtime_name}-receipt.json"
machine_id_file="${PACKET_DIR}/.machine-id-${runtime_name}"
machine_id="$(openssl rand -hex 16)"
@@ -303,6 +321,14 @@ run_runtime() {
fi
sleep 1
done
docker exec "${container_id}" install -d -o root -g root -m 0755 "$(dirname "${BOUND_PROBE_PATH}")"
docker exec "${container_id}" install -o root -g root -m 0755 /opt/pulse/packet/dockeragent.test "${BOUND_PROBE_PATH}"
packet_probe_hash="$(sha256_files "${PACKET_DIR}/dockeragent.test" | awk '{print $1}')"
installed_probe_hash="$(docker exec "${container_id}" sha256sum "${BOUND_PROBE_PATH}" | awk '{print $1}')"
if [[ ! "${packet_probe_hash}" =~ ^[0-9a-f]{64}$ || "${installed_probe_hash}" != "${packet_probe_hash}" ]]; then
echo "ERROR: ${runtime_name} collector-executable bound probe differs from the qualification binary" >&2
return 1
fi
if docker exec "${container_id}" sh -c 'ip route | grep -q "^default "'; then
echo "ERROR: ${runtime_name} qualification container unexpectedly has a default route" >&2
return 1
@@ -319,6 +345,8 @@ run_runtime() {
-e PULSE_ROOTFUL_RECEIPT=/opt/pulse/result/rootful-receipt.json \
-e PULSE_ROOTFUL_SOURCE_HASHES=/opt/pulse/packet/source-hashes.json \
-e "PULSE_ROOTFUL_SOURCE_COMMIT=${SOURCE_COMMIT}" \
-e "PULSE_ROOTFUL_UBUNTU_IMAGE=${UBUNTU_IMAGE}" \
-e "PULSE_ROOTFUL_BOUND_PROBE_BINARY=${BOUND_PROBE_PATH}" \
-e PULSE_SECURE_RUNTIME_COLLECTOR=/opt/pulse/packet/pulse-agent \
-e PULSE_SECURE_RUNTIME_COLLECTOR_SIGNATURE=/opt/pulse/packet/pulse-agent.sig \
-e PULSE_SECURE_RUNTIME_HELPER=/opt/pulse/packet/pulse-agent-helper \
@@ -355,7 +383,7 @@ docker = json.loads(docker_path.read_text())
podman = json.loads(podman_path.read_text())
if docker.get("result") != "passed" or podman.get("result") != "passed":
raise SystemExit('per-runtime qualification result != "passed"')
for field in ("schema_version", "kind", "source_commit", "source_hashes", "artifacts"):
for field in ("schema_version", "kind", "source_commit", "base_image", "source_hashes", "artifacts"):
if docker.get(field) != podman.get(field):
raise SystemExit(f"per-runtime qualification field differs: {field}")
runs = docker.get("runs", []) + podman.get("runs", [])
@@ -368,6 +396,7 @@ if len(set(machine_ids)) != 2 or len(set(daemon_ids)) != 2:
combined = {
"schema_version": docker["schema_version"], "kind": docker["kind"], "result": "passed",
"source_commit": docker["source_commit"],
"base_image": docker["base_image"],
"started_at": min(docker["started_at"], podman["started_at"]),
"completed_at": max(docker["completed_at"], podman["completed_at"]),
"source_hashes": docker["source_hashes"], "artifacts": docker["artifacts"], "runs": runs,