diff --git a/build/microvm/init.sh b/build/microvm/init.sh index 18bf29e37..869fb59fe 100755 --- a/build/microvm/init.sh +++ b/build/microvm/init.sh @@ -3,14 +3,15 @@ mount -t proc proc /proc mount -t sysfs sys /sys mount -t devtmpfs dev /dev +# virtio_mmio is built-in (CONFIG_VIRTIO_MMIO=y) so virtio-net devices are +# already enumerated by the time PID 1 runs — no modprobe needed. +# qemu_fw_cfg and virtio_net are =m in stock Alpine linux-virt, so load them. modprobe qemu_fw_cfg 2>/dev/null || modprobe fw_cfg_sysfs 2>/dev/null || true -modprobe virtio_mmio 2>/dev/null || true modprobe virtio_net 2>/dev/null || true -echo "[init] waiting for fw_cfg sysfs..." -for i in $(seq 1 10); do +for i in 1 2 3 4 5; do [ -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ] && break - sleep 0.2 + sleep 0.02 done if [ ! -f /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw ]; then @@ -20,19 +21,15 @@ fi . /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw -echo "[init] network interfaces:" -ip -o link show - for n in 0 1 2 3 4 5; do eval mac=\$NIC${n}_MAC [ -z "$mac" ] && continue - # Retry finding the interface by MAC — virtio-mmio may take a moment to appear. iface="" - for attempt in $(seq 1 10); do + for attempt in 1 2 3 4 5; do iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :) [ -n "$iface" ] && break - sleep 0.2 + sleep 0.02 done if [ -z "$iface" ]; then @@ -46,7 +43,6 @@ for n in 0 1 2 3 4 5; do eval rdst=\$NIC${n}_ROUTE_DST eval rvia=\$NIC${n}_ROUTE_VIA - echo "[init] configuring NIC$n: iface=$iface cidr=$cidr gw=$gw default=$default" ip link set "$iface" up [ -n "$cidr" ] && ip addr add "$cidr" dev "$iface" [ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface" diff --git a/scripts/build-microvm-image.sh b/scripts/build-microvm-image.sh index ef48c602e..5b44b0244 100755 --- a/scripts/build-microvm-image.sh +++ b/scripts/build-microvm-image.sh @@ -219,6 +219,19 @@ find "$CHROOT_DIR/usr/sbin" "$CHROOT_DIR/usr/bin" \ strip --strip-all "$bin" 2>/dev/null || true done +# --- Copy vmlinuz out before stripping /boot from chroot --- +VMLINUZ_OUT="$BUILD_DIR/vmlinuz" +echo "[build-microvm-image] copying kernel: $VMLINUZ_OUT" +cp "$KERNEL_IMG" "$VMLINUZ_OUT" + +# Drop /boot from the chroot — vmlinuz, Alpine's stock initramfs-virt, System.map +# and kernel config all live here (~25 MiB). None are needed at runtime: the +# guest already has the kernel loaded by QEMU's -kernel, and Alpine's initramfs +# is unused because we replace it with our own /init. Leaving /boot in the cpio +# nearly doubles uncompressed initramfs size and forces a 512 MiB guest just to +# survive decompression. +rm -rf "$CHROOT_DIR/boot" + # --- Build initramfs --- INITRAMFS_OUT="$BUILD_DIR/initramfs.cpio.gz" echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT" @@ -227,11 +240,6 @@ echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT" find . | cpio --quiet -o -H newc | gzip -9 > "$INITRAMFS_OUT" ) -# --- Copy vmlinuz --- -VMLINUZ_OUT="$BUILD_DIR/vmlinuz" -echo "[build-microvm-image] copying kernel: $VMLINUZ_OUT" -cp "$KERNEL_IMG" "$VMLINUZ_OUT" - # --- Log artifact sizes --- echo "" echo "[build-microvm-image] artifacts:" diff --git a/spinifex/daemon/daemon_system_instance.go b/spinifex/daemon/daemon_system_instance.go index 669d6ff6b..e38764658 100644 --- a/spinifex/daemon/daemon_system_instance.go +++ b/spinifex/daemon/daemon_system_instance.go @@ -488,12 +488,26 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2. return vm.Config{}, fmt.Errorf("write fw_cfg blobs: %w", err) } - // Build kernel cmdline: serial console + virtio-mmio device descriptors. + // Build kernel cmdline. + // // QEMU microvm maps virtio-mmio slots at 0xfeb00000 + n*0x200 with IRQ 5+n. - // Without these params the kernel never discovers the devices (auto-kernel-cmdline - // does not append when -append is specified). + // Without virtio_mmio.device params the kernel never discovers the devices + // (auto-kernel-cmdline does not append when -append is specified). + // + // Boot-time perf flags (Phase 2): + // quiet loglevel=3 — suppress printk to ttyS0 (115200 baud serial + // writes dominate boot time; warnings+errors only) + // mitigations=off — skip spectre/meltdown microcode patching at + // boot. Safe here: KVM guest on trusted host CPU, + // single-tenant LB workload, no untrusted code in + // the VM. Saves ~200-500ms. + // tsc=reliable — skip TSC calibration loop on KVM (host already + // vouched for invariant TSC) + // no_timer_check — skip APIC/PIT cross-check; microvm has no PIT + // reboot=t — triple-fault reboot; skip ACPI/keyboard probes + // i8042.no{pnp,aux,kbd} — microvm has no PS/2; skip i8042 probe timeouts var sb strings.Builder - sb.WriteString("console=ttyS0") + sb.WriteString("console=ttyS0 quiet loglevel=3 mitigations=off tsc=reliable no_timer_check reboot=t i8042.nopnp i8042.noaux i8042.nokbd") for i := range input.NICs { fmt.Fprintf(&sb, " virtio_mmio.device=0x200@0x%x:%d", 0xfeb00000+i*0x200, 5+i) diff --git a/spinifex/instancetypes/definitions.go b/spinifex/instancetypes/definitions.go index 1b6d6a153..153ac3885 100644 --- a/spinifex/instancetypes/definitions.go +++ b/spinifex/instancetypes/definitions.go @@ -270,7 +270,7 @@ var p5eSizes = []instanceSize{ // systemSizes defines internal-only instance types for system VMs (LB, NAT GW, etc.). // These are registered in the type map for allocation but excluded from DescribeInstanceTypes. var systemSizes = []instanceSize{ - {"micro", 1, 0.5}, // 1 vCPU, 512 MB — minimum for microvm initramfs (~68 MB uncompressed) + {"micro", 1, 0.125}, // 1 vCPU, 128 MB } // instanceFamilyDefs defines all supported instance families with their vendor and sizes. diff --git a/spinifex/instancetypes/generate_test.go b/spinifex/instancetypes/generate_test.go index adf83cd3b..0faf5758d 100644 --- a/spinifex/instancetypes/generate_test.go +++ b/spinifex/instancetypes/generate_test.go @@ -46,7 +46,7 @@ func TestGenerateSystemTypes(t *testing.T) { sysMicro, ok := types["sys.micro"] require.True(t, ok, "sys.micro must exist") assert.Equal(t, int64(1), *sysMicro.VCpuInfo.DefaultVCpus, "sys.micro should have 1 vCPU") - assert.Equal(t, int64(512), *sysMicro.MemoryInfo.SizeInMiB, "sys.micro should have 512 MiB") + assert.Equal(t, int64(128), *sysMicro.MemoryInfo.SizeInMiB, "sys.micro should have 128 MiB") assert.False(t, *sysMicro.BurstablePerformanceSupported, "sys.micro should not be burstable") } diff --git a/spinifex/vm/lifecycle.go b/spinifex/vm/lifecycle.go index 48172418a..e5936c447 100644 --- a/spinifex/vm/lifecycle.go +++ b/spinifex/vm/lifecycle.go @@ -171,6 +171,28 @@ func (m *Manager) launch(instance *VM) error { return nil } +// nbdkitPreExecWait returns how long startQEMU sleeps before exec'ing QEMU to +// give nbdkit time to bind its sockets. Direct-boot instances have no drives +// and no nbdkit, so the wait collapses to zero and the boot-time budget is +// preserved. +func nbdkitPreExecWait(directBoot bool) time.Duration { + if directBoot { + return 0 + } + return 2 * time.Second +} + +// qemuStartSettleWait returns the post-fork sleep that lets QEMU surface an +// immediate-start crash (bad cmdline, missing kernel) before we attempt QMP. +// Direct-boot binds the QMP socket within a few ms because there is no +// firmware, ROM, or block layer setup, so a full second is unnecessary. +func qemuStartSettleWait(directBoot bool) time.Duration { + if directBoot { + return 50 * time.Millisecond + } + return time.Second +} + // startQEMU launches the QEMU process for instance and waits for startup to // confirm. func (m *Manager) startQEMU(instance *VM) error { @@ -318,7 +340,7 @@ func (m *Manager) startQEMU(instance *VM) error { // Wait briefly for nbdkit to start. // TODO: Improve, confirm nbdkit started for each volume. - time.Sleep(2 * time.Second) + time.Sleep(nbdkitPreExecWait(instance.DirectBoot)) processChan := make(chan int, 1) exitChan := make(chan int, 1) @@ -402,7 +424,9 @@ func (m *Manager) startQEMU(instance *VM) error { return fmt.Errorf("failed to start qemu") } - time.Sleep(1 * time.Second) + // Catch immediate-start crashes (bad cmdline, missing kernel, etc.) before + // attempting QMP. + time.Sleep(qemuStartSettleWait(instance.DirectBoot)) select { case exitErr := <-exitChan: diff --git a/spinifex/vm/lifecycle_test.go b/spinifex/vm/lifecycle_test.go index ff01c6e17..47c285363 100644 --- a/spinifex/vm/lifecycle_test.go +++ b/spinifex/vm/lifecycle_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/mulgadc/spinifex/spinifex/types" "github.com/stretchr/testify/assert" @@ -760,3 +761,20 @@ func (p *scriptedNetworkPlumber) SetupTap(spec TapSpec) error { func (p *scriptedNetworkPlumber) CleanupTap(_ string) error { return nil } var _ NetworkPlumber = (*scriptedNetworkPlumber)(nil) + +// TestNbdkitPreExecWait locks in the per-mode pre-exec sleep budget: direct-boot +// is zero (no drives, no nbdkit) and the PC-machine path keeps the 2 s wait that +// the rest of the launch flow depends on. +func TestNbdkitPreExecWait(t *testing.T) { + assert.Equal(t, time.Duration(0), nbdkitPreExecWait(true)) + assert.Equal(t, 2*time.Second, nbdkitPreExecWait(false)) +} + +// TestQemuStartSettleWait locks in the per-mode post-fork settle: direct-boot +// trims to 50 ms (QMP socket binds within a few ms when there is no firmware, +// ROM, or block layer setup) and the PC-machine path retains the historical +// 1 s buffer that catches bad-cmdline crashes before the QMP dial. +func TestQemuStartSettleWait(t *testing.T) { + assert.Equal(t, 50*time.Millisecond, qemuStartSettleWait(true)) + assert.Equal(t, time.Second, qemuStartSettleWait(false)) +}