From 17c83cb3addcbedf492670630624d6718ad38065 Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Mon, 15 Jun 2026 10:50:10 +1000 Subject: [PATCH] fix: build microVM rootfs via buildx to stop overlay2 export leak (#358) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit docker export of a container leaks its overlay mount on Docker's overlay2 driver — the mount survives docker rm, so the layer dirs are never reaped except by a daemon restart. The lb-agent microVM build ran docker export on every e2e job across ~9 runners, accumulating ~69G of orphaned overlay2 layers on the CI host (disk hit 94%). Build the rootfs with `docker buildx --output type=local` from a small generated Dockerfile instead, which writes the image filesystem to a directory and releases all mounts — nothing leaks. Keep the podman path (its store is unaffected). Tighten the e2e cleanup image prune to dangling-only so the alpine base stays cached instead of re-pulling. --- .github/workflows/e2e-suites.yml | 8 +++--- .github/workflows/e2e.yml | 8 +++--- scripts/build-microvm-image.sh | 46 +++++++++++++++++++++----------- 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/.github/workflows/e2e-suites.yml b/.github/workflows/e2e-suites.yml index abc71fea1..a73ce6d41 100644 --- a/.github/workflows/e2e-suites.yml +++ b/.github/workflows/e2e-suites.yml @@ -119,10 +119,10 @@ jobs: JSON=$(printf '%s\n' $LIST | jq -R . | jq -cs .) echo "matrix=${JSON}" >> "$GITHUB_OUTPUT" - - name: Prune rootless docker cache + - name: Reclaim docker build cache run: | docker container prune -f || true - docker image prune -af || true + docker image prune -f || true docker buildx prune -af --filter "until=24h" || true docker system df || true @@ -524,10 +524,10 @@ jobs: - name: Re-enable Go toolchain auto-download run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV" - - name: Prune rootless docker cache + - name: Reclaim docker build cache run: | docker container prune -f || true - docker image prune -af || true + docker image prune -f || true docker buildx prune -af --filter "until=24h" || true docker system df || true diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 11cc173d6..1e7a71733 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -137,10 +137,10 @@ jobs: - name: Re-enable Go toolchain auto-download run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV" - - name: Prune rootless docker cache + - name: Reclaim docker build cache run: | docker container prune -f || true - docker image prune -af || true + docker image prune -f || true docker buildx prune -af --filter "until=24h" || true docker system df || true @@ -533,10 +533,10 @@ jobs: - name: Re-enable Go toolchain auto-download run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV" - - name: Prune rootless docker cache + - name: Reclaim docker build cache run: | docker container prune -f || true - docker image prune -af || true + docker image prune -f || true docker buildx prune -af --filter "until=24h" || true docker system df || true diff --git a/scripts/build-microvm-image.sh b/scripts/build-microvm-image.sh index dba6ffaed..8ad01f319 100755 --- a/scripts/build-microvm-image.sh +++ b/scripts/build-microvm-image.sh @@ -81,30 +81,46 @@ EOF --initdb \ $ALPINE_PACKAGES -# Strategy 2/3: container runtime (dev machines) -else - if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then - CONTAINER_TOOL=docker - elif command -v podman >/dev/null 2>&1; then - CONTAINER_TOOL=podman - else - echo "ERROR: no rootfs build tool available." >&2 - echo " Install one of: apk (Alpine), docker, podman" >&2 - exit 1 - fi +# Strategy 2: docker -> BuildKit local export. +# Do NOT use `docker export` here. On Docker's overlay2 driver it leaks the +# container's overlay mount (the mount survives `docker rm`), orphaning layers +# under /var/lib/docker that only a daemon restart reclaims; on a busy CI host +# this grows without bound. `buildx --output type=local` writes the image +# filesystem straight to a directory and releases every mount, so nothing leaks. +elif command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then + echo "[build-microvm-image] rootfs: docker buildx --output type=local (alpine:${ALPINE_VERSION})" + BUILD_CTX=$(mktemp -d) + cat > "$BUILD_CTX/Dockerfile" < run + export. Podman's store does not exhibit the docker +# overlay2 export-mount leak, and podman has no buildx. +elif command -v podman >/dev/null 2>&1; then + echo "[build-microvm-image] rootfs: podman export (alpine:${ALPINE_VERSION})" + CONTAINER_TOOL=podman + CONTAINER_CID=$(podman run -d \ "alpine:${ALPINE_VERSION}" \ sh -c "apk add --no-cache ${ALPINE_PACKAGES}") - exit_code=$($CONTAINER_TOOL wait "$CONTAINER_CID") + exit_code=$(podman wait "$CONTAINER_CID") if [ "$exit_code" != "0" ]; then echo "ERROR: package installation failed in container (exit $exit_code)" >&2 exit 1 fi - $CONTAINER_TOOL export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR" + podman export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR" + +else + echo "ERROR: no rootfs build tool available." >&2 + echo " Install one of: apk (Alpine), docker, podman" >&2 + exit 1 fi # --- /dev: empty mountpoint only ---