From 80ab3fb37a0c053d16ca8d5027922cdbf6fd1b4b Mon Sep 17 00:00:00 2001
From: joshsiv-mulga <mrk@mulgadc.com>
Date: Fri, 22 May 2026 16:52:38 +1000
Subject: [PATCH] ci: prune rootless docker + trap container leaks in microvm
 build

Self-hosted runner accumulated 18GB in overlay2 from leaked container
layers. Two causes: build-microvm-image.sh only removed the alpine
container on the success path, and no workflow step pruned docker
between runs.

- scripts/build-microvm-image.sh: container id now removed via the
  existing EXIT trap, so failures, signals, and runner kills no longer
  leak the writable layer.
- .github/workflows/e2e-go.yml: prune containers / dangling images /
  buildx cache before each Build Install Artifacts step, in all three
  e2e jobs (single, multinode, reboot).
---
 .github/workflows/e2e-go.yml   | 30 ++++++++++++++++++++++++++++++
 scripts/build-microvm-image.sh | 15 +++++++++------
 2 files changed, 39 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/e2e-go.yml b/.github/workflows/e2e-go.yml
index ced66a21f..49141b187 100644
--- a/.github/workflows/e2e-go.yml
+++ b/.github/workflows/e2e-go.yml
@@ -112,6 +112,16 @@ jobs:
         # Go self-heal by fetching the named minor on demand.
         run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
 
+      - name: Prune rootless docker cache
+        # Self-hosted runner uses rootless docker; build-microvm-image.sh
+        # leaks ~100 MiB overlay2 layer per failed build. Prune before each
+        # run keeps /home/<ciuser>/.local/share/docker bounded.
+        run: |
+          docker container prune -f || true
+          docker image prune -af || true
+          docker buildx prune -af --filter "until=24h" || true
+          docker system df || true
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
@@ -443,6 +453,16 @@ jobs:
       - name: Re-enable Go toolchain auto-download
         run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
 
+      - name: Prune rootless docker cache
+        # Self-hosted runner uses rootless docker; build-microvm-image.sh
+        # leaks ~100 MiB overlay2 layer per failed build. Prune before each
+        # run keeps /home/<ciuser>/.local/share/docker bounded.
+        run: |
+          docker container prune -f || true
+          docker image prune -af || true
+          docker buildx prune -af --filter "until=24h" || true
+          docker system df || true
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
@@ -799,6 +819,16 @@ jobs:
       - name: Re-enable Go toolchain auto-download
         run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
 
+      - name: Prune rootless docker cache
+        # Self-hosted runner uses rootless docker; build-microvm-image.sh
+        # leaks ~100 MiB overlay2 layer per failed build. Prune before each
+        # run keeps /home/<ciuser>/.local/share/docker bounded.
+        run: |
+          docker container prune -f || true
+          docker image prune -af || true
+          docker buildx prune -af --filter "until=24h" || true
+          docker system df || true
+
       - name: Build Install Artifacts
         working-directory: mulga/scripts/tofu-cluster
         run: |
diff --git a/scripts/build-microvm-image.sh b/scripts/build-microvm-image.sh
index 5b44b0244..5804cdf1e 100755
--- a/scripts/build-microvm-image.sh
+++ b/scripts/build-microvm-image.sh
@@ -35,9 +35,15 @@ done
 
 # --- Create chroot ---
 CHROOT_DIR=$(mktemp -d)
+CONTAINER_TOOL=""
+CONTAINER_CID=""
 cleanup() {
     echo "[build-microvm-image] cleaning up chroot: $CHROOT_DIR"
     rm -rf "$CHROOT_DIR"
+    if [ -n "$CONTAINER_CID" ] && [ -n "$CONTAINER_TOOL" ]; then
+        echo "[build-microvm-image] cleaning up container: $CONTAINER_CID"
+        $CONTAINER_TOOL rm -f "$CONTAINER_CID" >/dev/null 2>&1 || true
+    fi
 }
 trap cleanup EXIT
 
@@ -63,7 +69,6 @@ EOF
 
 # Strategy 2/3: container runtime (dev machines)
 else
-    CONTAINER_TOOL=""
     if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
         CONTAINER_TOOL=docker
     elif command -v podman >/dev/null 2>&1; then
@@ -75,19 +80,17 @@ else
     fi
 
     echo "[build-microvm-image] rootfs: $CONTAINER_TOOL export (alpine:${ALPINE_VERSION})"
-    cid=$($CONTAINER_TOOL run -d \
+    CONTAINER_CID=$($CONTAINER_TOOL run -d \
         "alpine:${ALPINE_VERSION}" \
         sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
 
-    exit_code=$($CONTAINER_TOOL wait "$cid")
+    exit_code=$($CONTAINER_TOOL wait "$CONTAINER_CID")
     if [ "$exit_code" != "0" ]; then
         echo "ERROR: package installation failed in container (exit $exit_code)" >&2
-        $CONTAINER_TOOL rm "$cid" >/dev/null 2>&1 || true
         exit 1
     fi
 
-    $CONTAINER_TOOL export "$cid" | tar -x -C "$CHROOT_DIR"
-    $CONTAINER_TOOL rm "$cid" >/dev/null 2>&1
+    $CONTAINER_TOOL export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
 fi
 
 # --- Fix /dev device nodes ---