ci: prune rootless docker + trap container leaks in microvm build

Self-hosted runner accumulated 18GB in overlay2 from leaked container
layers. Two causes: build-microvm-image.sh only removed the alpine
container on the success path, and no workflow step pruned docker
between runs.

- scripts/build-microvm-image.sh: container id now removed via the
  existing EXIT trap, so failures, signals, and runner kills no longer
  leak the writable layer.
- .github/workflows/e2e-go.yml: prune containers / dangling images /
  buildx cache before each Build Install Artifacts step, in all three
  e2e jobs (single, multinode, reboot).
This commit is contained in:
joshsiv-mulga
2026-05-22 16:52:38 +10:00
parent 756a5bb31f
commit 80ab3fb37a
2 changed files with 39 additions and 6 deletions
+30
View File
@@ -112,6 +112,16 @@ jobs:
# Go self-heal by fetching the named minor on demand.
run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
- name: Prune rootless docker cache
# Self-hosted runner uses rootless docker; build-microvm-image.sh
# leaks ~100 MiB overlay2 layer per failed build. Prune before each
# run keeps /home/<ciuser>/.local/share/docker bounded.
run: |
docker container prune -f || true
docker image prune -af || true
docker buildx prune -af --filter "until=24h" || true
docker system df || true
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
run: |
@@ -443,6 +453,16 @@ jobs:
- name: Re-enable Go toolchain auto-download
run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
- name: Prune rootless docker cache
# Self-hosted runner uses rootless docker; build-microvm-image.sh
# leaks ~100 MiB overlay2 layer per failed build. Prune before each
# run keeps /home/<ciuser>/.local/share/docker bounded.
run: |
docker container prune -f || true
docker image prune -af || true
docker buildx prune -af --filter "until=24h" || true
docker system df || true
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
run: |
@@ -799,6 +819,16 @@ jobs:
- name: Re-enable Go toolchain auto-download
run: echo "GOTOOLCHAIN=auto" >> "$GITHUB_ENV"
- name: Prune rootless docker cache
# Self-hosted runner uses rootless docker; build-microvm-image.sh
# leaks ~100 MiB overlay2 layer per failed build. Prune before each
# run keeps /home/<ciuser>/.local/share/docker bounded.
run: |
docker container prune -f || true
docker image prune -af || true
docker buildx prune -af --filter "until=24h" || true
docker system df || true
- name: Build Install Artifacts
working-directory: mulga/scripts/tofu-cluster
run: |
+9 -6
View File
@@ -35,9 +35,15 @@ done
# --- Create chroot ---
CHROOT_DIR=$(mktemp -d)
CONTAINER_TOOL=""
CONTAINER_CID=""
cleanup() {
echo "[build-microvm-image] cleaning up chroot: $CHROOT_DIR"
rm -rf "$CHROOT_DIR"
if [ -n "$CONTAINER_CID" ] && [ -n "$CONTAINER_TOOL" ]; then
echo "[build-microvm-image] cleaning up container: $CONTAINER_CID"
$CONTAINER_TOOL rm -f "$CONTAINER_CID" >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT
@@ -63,7 +69,6 @@ EOF
# Strategy 2/3: container runtime (dev machines)
else
CONTAINER_TOOL=""
if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then
CONTAINER_TOOL=docker
elif command -v podman >/dev/null 2>&1; then
@@ -75,19 +80,17 @@ else
fi
echo "[build-microvm-image] rootfs: $CONTAINER_TOOL export (alpine:${ALPINE_VERSION})"
cid=$($CONTAINER_TOOL run -d \
CONTAINER_CID=$($CONTAINER_TOOL run -d \
"alpine:${ALPINE_VERSION}" \
sh -c "apk add --no-cache ${ALPINE_PACKAGES}")
exit_code=$($CONTAINER_TOOL wait "$cid")
exit_code=$($CONTAINER_TOOL wait "$CONTAINER_CID")
if [ "$exit_code" != "0" ]; then
echo "ERROR: package installation failed in container (exit $exit_code)" >&2
$CONTAINER_TOOL rm "$cid" >/dev/null 2>&1 || true
exit 1
fi
$CONTAINER_TOOL export "$cid" | tar -x -C "$CHROOT_DIR"
$CONTAINER_TOOL rm "$cid" >/dev/null 2>&1
$CONTAINER_TOOL export "$CONTAINER_CID" | tar -x -C "$CHROOT_DIR"
fi
# --- Fix /dev device nodes ---