fix: build microVM rootfs via buildx to stop overlay2 export leak (#358)

docker export of a container leaks its overlay mount on Docker's
overlay2 driver — the mount survives docker rm, so the layer dirs are
never reaped except by a daemon restart. The lb-agent microVM build ran
docker export on every e2e job across ~9 runners, accumulating ~69G of
orphaned overlay2 layers on the CI host (disk hit 94%).

Build the rootfs with `docker buildx --output type=local` from a small
generated Dockerfile instead, which writes the image filesystem to a
directory and releases all mounts — nothing leaks. Keep the podman path
(its store is unaffected). Tighten the e2e cleanup image prune to
dangling-only so the alpine base stays cached instead of re-pulling.

.github/workflows/e2e-suites.yml

@@ -119,10 +119,10 @@ jobs:
           JSON=$(printf '%s\n' $LIST | jq -R . | jq -cs .)
           echo "matrix=${JSON}" >> "$GITHUB_OUTPUT"
 
-      - name: Prune rootless docker cache
+      - name: Reclaim docker build cache
         run: |
           docker container prune -f || true
-          docker image prune -af || true
+          docker image prune -f || true
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 
@@ -524,10 +524,10 @@ jobs:
       - name: Re-enable Go toolchain auto-download
         run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
 
-      - name: Prune rootless docker cache
+      - name: Reclaim docker build cache
         run: |
           docker container prune -f || true
-          docker image prune -af || true
+          docker image prune -f || true
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 

.github/workflows/e2e.yml

@@ -137,10 +137,10 @@ jobs:
       - name: Re-enable Go toolchain auto-download
         run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
 
-      - name: Prune rootless docker cache
+      - name: Reclaim docker build cache
         run: |
           docker container prune -f || true
-          docker image prune -af || true
+          docker image prune -f || true
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 
@@ -533,10 +533,10 @@ jobs:
       - name: Re-enable Go toolchain auto-download
         run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
 
-      - name: Prune rootless docker cache
+      - name: Reclaim docker build cache
         run: |
           docker container prune -f || true
-          docker image prune -af || true
+          docker image prune -f || true
           docker buildx prune -af --filter "until=24h" || true
           docker system df || true
 

scripts/build-microvm-image.sh

@@ -81,30 +81,46 @@ EOF
         --initdb \
         $ALPINE_PACKAGES
 
-# Strategy 2/3: container runtime (dev machines)
-else
-    if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
-        CONTAINER_TOOL=docker
-    elif command -v podman >/dev/null 2>&1; then
-        CONTAINER_TOOL=podman
-    else
-        echo "ERROR: no rootfs build tool available." >&2
-        echo "       Install one of: apk (Alpine), docker, podman" >&2
-        exit 1
-    fi
+# Strategy 2: docker -> BuildKit local export.
+# Do NOT use `docker export` here. On Docker's overlay2 driver it leaks the
+# container's overlay mount (the mount survives `docker rm`), orphaning layers
+# under /var/lib/docker that only a daemon restart reclaims; on a busy CI host
+# this grows without bound. `buildx --output type=local` writes the image
+# filesystem straight to a directory and releases every mount, so nothing leaks.
+elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
+    echo "[build-microvm-image] rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})"
+    BUILD_CTX=$(mktemp -d)
+    cat > "$BUILD_CTX/Dockerfile" <<DOCKERFILE
+FROM alpine:${ALPINE_VERSION}
+RUN apk add --no-cache ${ALPINE_PACKAGES}
+DOCKERFILE
+    docker buildx build \
+        --platform linux/amd64 \
+        --output "type=local,dest=${CHROOT_DIR}" \
+        "$BUILD_CTX"
+    rm -rf "$BUILD_CTX"
 
-    echo "[build-microvm-image] rootfs: $CONTAINER_TOOL export (alpine:${ALPINE_VERSION})"
-    CONTAINER_CID=$($CONTAINER_TOOL run -d \
+# Strategy 3: podman -> run + export. Podman's store does not exhibit the docker
+# overlay2 export-mount leak, and podman has no buildx.
+elif command -v podman >/dev/null 2>&1; then
+    echo "[build-microvm-image] rootfs: podman export (alpine:${ALPINE_VERSION})"
+    CONTAINER_TOOL=podman
+    CONTAINER_CID=$(podman run -d \
         "alpine:${ALPINE_VERSION}" \
         sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
 
-    exit_code=$($CONTAINER_TOOL wait "$CONTAINER_CID")
+    exit_code=$(podman wait "$CONTAINER_CID")
     if [ "$exit_code" != "0" ]; then
         echo "ERROR: package installation failed in container (exit $exit_code)" >&2
         exit 1
     fi
 
-    $CONTAINER_TOOL export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
+    podman export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
+
+else
+    echo "ERROR: no rootfs build tool available." >&2
+    echo "       Install one of: apk (Alpine), docker, podman" >&2
+    exit 1
 fi
 
 # --- /dev: empty mountpoint only ---