perf(microvm): hit Phase 2 boot/memory gates

t1→t2 hypervisor: 3224 ms → 264 ms (12×). 128 MiB guest now boots
cleanly without OOM, reverting the temporary 512 MiB bump.

Root causes:

1. build-microvm-image.sh staged the entire chroot into the cpio
   without dropping /boot, embedding a duplicate vmlinuz, Alpine's
   stock initramfs-virt, System.map, and a "boot → ." symlink loop
   (~25 MiB). Decompression peak exceeded 128 MiB. Strip /boot after
   the kernel is copied out, before the cpio. Initramfs drops from
   31 MiB → 11 MiB (53 MiB → 28 MiB uncompressed); total artifact
   42 MiB → 22 MiB.

2. vm/lifecycle.go burned 3 s of fixed sleep on every direct-boot
   launch: 2 s nbdkit pre-exec wait (no nbdkit on direct-boot) and
   1 s post-fork settle (QEMU binds QMP within ~10 ms when there is
   no firmware, ROM, or block layer). Both extracted to
   directBoot-aware helpers — nbdkitPreExecWait collapses to 0, and
   qemuStartSettleWait to 50 ms.

3. KernelCmdline only carried `console=ttyS0 virtio_mmio.device=…`.
   Added boot-time perf flags: quiet loglevel=3 (silence printk over
   115200-baud isa-serial), mitigations=off (KVM guest on trusted
   host, single-tenant LB workload), tsc=reliable, no_timer_check,
   reboot=t, i8042.{nopnp,noaux,nokbd} (microvm has no PS/2).

4. init.sh dropped modprobe virtio_mmio (built-in:
   CONFIG_VIRTIO_MMIO=y) and tightened polling: fw_cfg/iface waits go
   from 10 × 200 ms (up to 2 s) to 5 × 20 ms (up to 100 ms).

Phase 2 gates:
  - t1→t2 ≤ 300 ms       ✅ 264 ms / 281 ms
  - artifact ≤ 50 MiB    ✅ 22 MiB (gate already in build script)
  - ELBv2 e2e green      ✅ 32/33 (pre-existing client-VM failure,
                                   unrelated to microvm path)

build/microvm/init.sh

@@ -3,14 +3,15 @@ mount -t proc  proc  /proc
 mount -t sysfs sys   /sys
 mount -t devtmpfs dev /dev
 
+# virtio_mmio is built-in (CONFIG_VIRTIO_MMIO=y) so virtio-net devices are
+# already enumerated by the time PID 1 runs — no modprobe needed.
+# qemu_fw_cfg and virtio_net are =m in stock Alpine linux-virt, so load them.
 modprobe qemu_fw_cfg 2>/dev/null || modprobe fw_cfg_sysfs 2>/dev/null || true
-modprobe virtio_mmio 2>/dev/null || true
 modprobe virtio_net 2>/dev/null || true
 
-echo "[init] waiting for fw_cfg sysfs..."
-for i in $(seq 1 10); do
+for i in 1 2 3 4 5; do
     [ -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ] && break
-    sleep 0.2
+    sleep 0.02
 done
 
 if [ ! -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ]; then
@@ -20,19 +21,15 @@ fi
 
 . /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
 
-echo "[init] network interfaces:"
-ip -o link show
-
 for n in 0 1 2 3 4 5; do
     eval mac=\$NIC${n}_MAC
     [ -z "$mac" ] && continue
 
-    # Retry finding the interface by MAC — virtio-mmio may take a moment to appear.
     iface=""
-    for attempt in $(seq 1 10); do
+    for attempt in 1 2 3 4 5; do
         iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
         [ -n "$iface" ] && break
-        sleep 0.2
+        sleep 0.02
     done
 
     if [ -z "$iface" ]; then
@@ -46,7 +43,6 @@ for n in 0 1 2 3 4 5; do
     eval rdst=\$NIC${n}_ROUTE_DST
     eval rvia=\$NIC${n}_ROUTE_VIA
 
-    echo "[init] configuring NIC$n: iface=$iface cidr=$cidr gw=$gw default=$default"
     ip link set "$iface" up
     [ -n "$cidr" ] && ip addr add "$cidr" dev "$iface"
     [ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface"

scripts/build-microvm-image.sh

@@ -219,6 +219,19 @@ find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \
     strip --strip-all "$bin" 2>/dev/null || true
 done
 
+# --- Copy vmlinuz out before stripping /boot from chroot ---
+VMLINUZ_OUT="$BUILD_DIR/vmlinuz"
+echo "[build-microvm-image] copying kernel: $VMLINUZ_OUT"
+cp "$KERNEL_IMG" "$VMLINUZ_OUT"
+
+# Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map
+# and kernel config all live here (~25 MiB). None are needed at runtime: the
+# guest already has the kernel loaded by QEMU's -kernel, and Alpine's initramfs
+# is unused because we replace it with our own /init. Leaving /boot in the cpio
+# nearly doubles uncompressed initramfs size and forces a 512 MiB guest just to
+# survive decompression.
+rm -rf "$CHROOT_DIR/boot"
+
 # --- Build initramfs ---
 INITRAMFS_OUT="$BUILD_DIR/initramfs.cpio.gz"
 echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT"
@@ -227,11 +240,6 @@ echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT"
     find . | cpio --quiet -o -H newc | gzip -9 > "$INITRAMFS_OUT"
 )
 
-# --- Copy vmlinuz ---
-VMLINUZ_OUT="$BUILD_DIR/vmlinuz"
-echo "[build-microvm-image] copying kernel: $VMLINUZ_OUT"
-cp "$KERNEL_IMG" "$VMLINUZ_OUT"
-
 # --- Log artifact sizes ---
 echo ""
 echo "[build-microvm-image] artifacts:"

spinifex/daemon/daemon_system_instance.go

@@ -488,12 +488,26 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2.
 		return vm.Config{}, fmt.Errorf("write fw_cfg blobs: %w", err)
 	}
 
-	// Build kernel cmdline: serial console + virtio-mmio device descriptors.
+	// Build kernel cmdline.
+	//
 	// QEMU microvm maps virtio-mmio slots at 0xfeb00000 + n*0x200 with IRQ 5+n.
-	// Without these params the kernel never discovers the devices (auto-kernel-cmdline
-	// does not append when -append is specified).
+	// Without virtio_mmio.device params the kernel never discovers the devices
+	// (auto-kernel-cmdline does not append when -append is specified).
+	//
+	// Boot-time perf flags (Phase 2):
+	//   quiet loglevel=3        — suppress printk to ttyS0 (115200 baud serial
+	//                             writes dominate boot time; warnings+errors only)
+	//   mitigations=off         — skip spectre/meltdown microcode patching at
+	//                             boot. Safe here: KVM guest on trusted host CPU,
+	//                             single-tenant LB workload, no untrusted code in
+	//                             the VM. Saves ~200-500ms.
+	//   tsc=reliable            — skip TSC calibration loop on KVM (host already
+	//                             vouched for invariant TSC)
+	//   no_timer_check          — skip APIC/PIT cross-check; microvm has no PIT
+	//   reboot=t                — triple-fault reboot; skip ACPI/keyboard probes
+	//   i8042.no{pnp,aux,kbd}   — microvm has no PS/2; skip i8042 probe timeouts
 	var sb strings.Builder
-	sb.WriteString("console=ttyS0")
+	sb.WriteString("console=ttyS0 quiet loglevel=3 mitigations=off tsc=reliable no_timer_check reboot=t i8042.nopnp i8042.noaux i8042.nokbd")
 	for i := range input.NICs {
 		fmt.Fprintf(&sb, " virtio_mmio.device=0x200@0x%x:%d",
 			0xfeb00000+i*0x200, 5+i)

spinifex/instancetypes/definitions.go

@@ -270,7 +270,7 @@ var p5eSizes = []instanceSize{
 // systemSizes defines internal-only instance types for system VMs (LB, NAT GW, etc.).
 // These are registered in the type map for allocation but excluded from DescribeInstanceTypes.
 var systemSizes = []instanceSize{
-	{"micro", 1, 0.5}, // 1 vCPU, 512 MB — minimum for microvm initramfs (~68 MB uncompressed)
+	{"micro", 1, 0.125}, // 1 vCPU, 128 MB
 }
 
 // instanceFamilyDefs defines all supported instance families with their vendor and sizes.

spinifex/instancetypes/generate_test.go

@@ -46,7 +46,7 @@ func TestGenerateSystemTypes(t *testing.T) {
 	sysMicro, ok := types["sys.micro"]
 	require.True(t, ok, "sys.micro must exist")
 	assert.Equal(t, int64(1), *sysMicro.VCpuInfo.DefaultVCpus, "sys.micro should have 1 vCPU")
-	assert.Equal(t, int64(512), *sysMicro.MemoryInfo.SizeInMiB, "sys.micro should have 512 MiB")
+	assert.Equal(t, int64(128), *sysMicro.MemoryInfo.SizeInMiB, "sys.micro should have 128 MiB")
 	assert.False(t, *sysMicro.BurstablePerformanceSupported, "sys.micro should not be burstable")
 }
 

spinifex/vm/lifecycle.go

@@ -171,6 +171,28 @@ func (m *Manager) launch(instance *VM) error {
 	return nil
 }
 
+// nbdkitPreExecWait returns how long startQEMU sleeps before exec'ing QEMU to
+// give nbdkit time to bind its sockets. Direct-boot instances have no drives
+// and no nbdkit, so the wait collapses to zero and the boot-time budget is
+// preserved.
+func nbdkitPreExecWait(directBoot bool) time.Duration {
+	if directBoot {
+		return 0
+	}
+	return 2 * time.Second
+}
+
+// qemuStartSettleWait returns the post-fork sleep that lets QEMU surface an
+// immediate-start crash (bad cmdline, missing kernel) before we attempt QMP.
+// Direct-boot binds the QMP socket within a few ms because there is no
+// firmware, ROM, or block layer setup, so a full second is unnecessary.
+func qemuStartSettleWait(directBoot bool) time.Duration {
+	if directBoot {
+		return 50 * time.Millisecond
+	}
+	return time.Second
+}
+
 // startQEMU launches the QEMU process for instance and waits for startup to
 // confirm.
 func (m *Manager) startQEMU(instance *VM) error {
@@ -318,7 +340,7 @@ func (m *Manager) startQEMU(instance *VM) error {
 
 	// Wait briefly for nbdkit to start.
 	// TODO: Improve, confirm nbdkit started for each volume.
-	time.Sleep(2 * time.Second)
+	time.Sleep(nbdkitPreExecWait(instance.DirectBoot))
 
 	processChan := make(chan int, 1)
 	exitChan := make(chan int, 1)
@@ -402,7 +424,9 @@ func (m *Manager) startQEMU(instance *VM) error {
 		return fmt.Errorf("failed to start qemu")
 	}
 
-	time.Sleep(1 * time.Second)
+	// Catch immediate-start crashes (bad cmdline, missing kernel, etc.) before
+	// attempting QMP.
+	time.Sleep(qemuStartSettleWait(instance.DirectBoot))
 
 	select {
 	case exitErr := <-exitChan:

spinifex/vm/lifecycle_test.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/types"
 	"github.com/stretchr/testify/assert"
@@ -760,3 +761,20 @@ func (p *scriptedNetworkPlumber) SetupTap(spec TapSpec) error {
 func (p *scriptedNetworkPlumber) CleanupTap(_ string) error { return nil }
 
 var _ NetworkPlumber = (*scriptedNetworkPlumber)(nil)
+
+// TestNbdkitPreExecWait locks in the per-mode pre-exec sleep budget: direct-boot
+// is zero (no drives, no nbdkit) and the PC-machine path keeps the 2 s wait that
+// the rest of the launch flow depends on.
+func TestNbdkitPreExecWait(t *testing.T) {
+	assert.Equal(t, time.Duration(0), nbdkitPreExecWait(true))
+	assert.Equal(t, 2*time.Second, nbdkitPreExecWait(false))
+}
+
+// TestQemuStartSettleWait locks in the per-mode post-fork settle: direct-boot
+// trims to 50 ms (QMP socket binds within a few ms when there is no firmware,
+// ROM, or block layer setup) and the PC-machine path retains the historical
+// 1 s buffer that catches bad-cmdline crashes before the QMP dial.
+func TestQemuStartSettleWait(t *testing.T) {
+	assert.Equal(t, 50*time.Millisecond, qemuStartSettleWait(true))
+	assert.Equal(t, time.Second, qemuStartSettleWait(false))
+}