From 1ea94061fc031469a742e8d43ad3336034c2c84a Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Sun, 17 May 2026 13:47:35 +1000 Subject: [PATCH] feat(microvm): Phase 2 kernel stripping + boot optimisation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cpu: switch to host,-pmu (disable PMU — not needed in microvm guest) - machine type: make isa-serial configurable via microvm.isa_serial config field; default true (on) preserves console log path - build: strip unused kernel modules (USB, IDE, ATA, GPU, media, input, HID, bluetooth, staging, sound, wireless) from initramfs before cpio - build: add Phase 2 artifact size gate — fail if vmlinuz + initramfs.cpio.gz exceeds MICROVM_ARTIFACT_MAX_MiB (default 50 MiB); override via env var during development - perf: add phase2BootGateMs=300 and phase2ArtifactGateMiB=50 constants with canary tests to catch unintentional gate changes - tests: cover buildMicrovmNICs nil-VPC and extra-ENI paths - tests: cover microvmMachineType nil/true/false branches --- cmd/spinifex/cmd/templates/spinifex.toml | 1 + scripts/build-microvm-image.sh | 38 +++++++++++++++++++ spinifex/config/config.go | 9 +++++ spinifex/daemon/daemon_system_instance.go | 27 +++++++++---- .../daemon/daemon_system_instance_test.go | 29 ++++++++++++-- spinifex/daemon/perf_bench_test.go | 14 ++++++- spinifex/handlers/elbv2/arn_test.go | 24 ++++++++++++ 7 files changed, 131 insertions(+), 11 deletions(-) diff --git a/cmd/spinifex/cmd/templates/spinifex.toml b/cmd/spinifex/cmd/templates/spinifex.toml index 45886e9a3..fc237212b 100644 --- a/cmd/spinifex/cmd/templates/spinifex.toml +++ b/cmd/spinifex/cmd/templates/spinifex.toml @@ -75,6 +75,7 @@ gpu_passthrough = true # [nodes.{{.Node}}.daemon.microvm] # elbv2_enabled = false # image_path = "/usr/share/spinifex/microvm" +# isa_serial = true # set false only after measuring boot savings; disables per-VM console log [nodes.{{.Node}}.awsgw] host = "{{.BindIP}}:9999" diff --git a/scripts/build-microvm-image.sh b/scripts/build-microvm-image.sh index 3d354390b..3c945231c 100755 --- a/scripts/build-microvm-image.sh +++ b/scripts/build-microvm-image.sh @@ -158,6 +158,31 @@ else echo "[build-microvm-image] fw_cfg: module found in lib/modules" fi +# --- Strip unused kernel modules (Phase 2) --- +# Remove driver categories not present on a virtio-only microvm guest. +# linux-virt may include some of these as loadable modules; stripping them +# here reduces initramfs size without rebuilding the kernel from source. +echo "[build-microvm-image] stripping unused kernel modules..." +for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do + [ -d "$kver_dir/kernel" ] || continue + rm -rf \ + "$kver_dir/kernel/drivers/usb" \ + "$kver_dir/kernel/drivers/ide" \ + "$kver_dir/kernel/drivers/ata" \ + "$kver_dir/kernel/drivers/gpu" \ + "$kver_dir/kernel/drivers/video" \ + "$kver_dir/kernel/drivers/media" \ + "$kver_dir/kernel/drivers/input" \ + "$kver_dir/kernel/drivers/hid" \ + "$kver_dir/kernel/drivers/bluetooth" \ + "$kver_dir/kernel/drivers/staging" \ + "$kver_dir/kernel/sound" \ + "$kver_dir/kernel/net/wireless" \ + "$kver_dir/kernel/net/bluetooth" + # Regenerate module dependency map after stripping. + depmod -b "$CHROOT_DIR" "$(basename "$kver_dir")" 2>/dev/null || true +done + # --- Build initramfs --- INITRAMFS_OUT="$BUILD_DIR/initramfs.cpio.gz" echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT" @@ -176,4 +201,17 @@ echo "" echo "[build-microvm-image] artifacts:" ls -lh "$VMLINUZ_OUT" "$INITRAMFS_OUT" echo "" + +# --- Phase 2 artifact size gate (50 MiB total) --- +ARTIFACT_MAX_MiB="${MICROVM_ARTIFACT_MAX_MiB:-50}" +total_bytes=$(( $(stat -c %s "$VMLINUZ_OUT") + $(stat -c %s "$INITRAMFS_OUT") )) +max_bytes=$(( ARTIFACT_MAX_MiB * 1024 * 1024 )) +total_mib=$(( (total_bytes + 1048575) / 1048576 )) +if [ "$total_bytes" -gt "$max_bytes" ]; then + echo "ERROR: artifact size ${total_mib} MiB exceeds ${ARTIFACT_MAX_MiB} MiB gate" >&2 + echo " Set MICROVM_ARTIFACT_MAX_MiB=N to override during development." >&2 + exit 1 +fi +echo "[build-microvm-image] artifact size gate: PASS (${total_mib} MiB / ${ARTIFACT_MAX_MiB} MiB)" +echo "" echo "[build-microvm-image] done." diff --git a/spinifex/config/config.go b/spinifex/config/config.go index 797dd92bc..29e4ce501 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -142,6 +142,10 @@ type GPUModelOverride struct { type MicrovmConfig struct { ELBv2Enabled bool `json:"ELBv2Enabled" mapstructure:"elbv2_enabled"` ImagePath string `json:"ImagePath" mapstructure:"image_path"` + // IsaSerial controls the isa-serial device on the microvm machine. Default + // true keeps the console log path working. Set false only after measuring + // boot-time savings and accepting loss of per-VM console output. + IsaSerial *bool `json:"IsaSerial" mapstructure:"isa_serial"` } // DaemonConfig holds the daemon configuration @@ -253,6 +257,11 @@ func LoadConfig(configPath string) (*ClusterConfig, error) { local.Daemon.Microvm.ImagePath = "/usr/share/spinifex/microvm" config.Nodes[config.Node] = local } + if local.Daemon.Microvm.IsaSerial == nil { + t := true + local.Daemon.Microvm.IsaSerial = &t + config.Nodes[config.Node] = local + } } return &config, nil diff --git a/spinifex/daemon/daemon_system_instance.go b/spinifex/daemon/daemon_system_instance.go index 36d3e64d7..669d6ff6b 100644 --- a/spinifex/daemon/daemon_system_instance.go +++ b/spinifex/daemon/daemon_system_instance.go @@ -492,21 +492,25 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2. // QEMU microvm maps virtio-mmio slots at 0xfeb00000 + n*0x200 with IRQ 5+n. // Without these params the kernel never discovers the devices (auto-kernel-cmdline // does not append when -append is specified). - cmdline := "console=ttyS0" + var sb strings.Builder + sb.WriteString("console=ttyS0") for i := range input.NICs { - cmdline += fmt.Sprintf(" virtio_mmio.device=0x200@0x%x:%d", + fmt.Fprintf(&sb, " virtio_mmio.device=0x200@0x%x:%d", 0xfeb00000+i*0x200, 5+i) } + cmdline := sb.String() + + machineType := microvmMachineType(d.config.Daemon.Microvm.IsaSerial) cfg := vm.Config{ Name: instanceID, EnableKVM: true, NoGraphic: true, Architecture: architecture, - CPUType: "host", + CPUType: "host,-pmu", CPUCount: vcpus, Memory: memMiB, - MachineType: "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on", + MachineType: machineType, KernelImage: filepath.Join(imagePath, "vmlinuz"), Initrd: filepath.Join(imagePath, "initramfs.cpio.gz"), KernelCmdline: cmdline, @@ -515,7 +519,7 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2. // Wire netdevs and devices for each NIC in declaration order. // Tap device creation happens later in startQEMU (host-side only here). - allNICs := buildNICNetdevs(instanceID, input) + allNICs := buildNICNetdevs(instanceID, input, machineType) cfg.NetDevs = append(cfg.NetDevs, allNICs.netdevs...) cfg.Devices = append(cfg.Devices, allNICs.devices...) @@ -528,13 +532,22 @@ type nicNetdevResult struct { devices []vm.Device } +// microvmMachineType returns the QEMU -M string for the microvm machine. +// isaSerial nil or true keeps isa-serial=on for console log access. +func microvmMachineType(isaSerial *bool) string { + serial := "on" + if isaSerial != nil && !*isaSerial { + serial = "off" + } + return "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=" + serial +} + // buildNICNetdevs produces QEMU -netdev and -device entries for each NIC in // input. The NIC order determines the netdev IDs: net0 = primary ENI, // net1 = mgmt (if present), net2+ = extra ENIs. Extra ENIs beyond the primary // are only included when corresponding ExtraENIs entries exist. -func buildNICNetdevs(instanceID string, input *handlers_elbv2.SystemInstanceInput) nicNetdevResult { +func buildNICNetdevs(instanceID string, input *handlers_elbv2.SystemInstanceInput, machineType string) nicNetdevResult { var res nicNetdevResult - machineType := "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on" for i, nic := range input.NICs { netID := fmt.Sprintf("net%d", i) diff --git a/spinifex/daemon/daemon_system_instance_test.go b/spinifex/daemon/daemon_system_instance_test.go index 683d3af83..06c0e7427 100644 --- a/spinifex/daemon/daemon_system_instance_test.go +++ b/spinifex/daemon/daemon_system_instance_test.go @@ -197,7 +197,7 @@ func TestBuildNICNetdevs_SingleNIC(t *testing.T) { {MAC: "02:aa:bb:cc:dd:01", IsDefault: true}, }, } - res := buildNICNetdevs("i-test", input) + res := buildNICNetdevs("i-test", input, microvmMachineType(nil)) require.Len(t, res.netdevs, 1) require.Len(t, res.devices, 1) assert.Contains(t, res.netdevs[0].Value, "tap,id=net0,") @@ -214,7 +214,7 @@ func TestBuildNICNetdevs_TwoNICs(t *testing.T) { {MAC: "02:aa:bb:cc:dd:02", IsDefault: false}, }, } - res := buildNICNetdevs("i-test2", input) + res := buildNICNetdevs("i-test2", input, microvmMachineType(nil)) require.Len(t, res.netdevs, 2) require.Len(t, res.devices, 2) assert.Contains(t, res.netdevs[0].Value, "id=net0,") @@ -225,11 +225,34 @@ func TestBuildNICNetdevs_TwoNICs(t *testing.T) { func TestBuildNICNetdevs_EmptyNICs(t *testing.T) { input := &handlers_elbv2.SystemInstanceInput{ENIID: "eni-empty"} - res := buildNICNetdevs("i-empty", input) + res := buildNICNetdevs("i-empty", input, microvmMachineType(nil)) assert.Empty(t, res.netdevs) assert.Empty(t, res.devices) } +// --------------------------------------------------------------------------- +// microvmMachineType +// --------------------------------------------------------------------------- + +func TestMicrovmMachineType_DefaultNil(t *testing.T) { + mt := microvmMachineType(nil) + assert.Contains(t, mt, "isa-serial=on") + assert.Contains(t, mt, "microvm,") +} + +func TestMicrovmMachineType_ExplicitTrue(t *testing.T) { + v := true + mt := microvmMachineType(&v) + assert.Contains(t, mt, "isa-serial=on") +} + +func TestMicrovmMachineType_ExplicitFalse(t *testing.T) { + v := false + mt := microvmMachineType(&v) + assert.Contains(t, mt, "isa-serial=off") + assert.NotContains(t, mt, "isa-serial=on") +} + // --------------------------------------------------------------------------- // writeFwCfgBlobs // --------------------------------------------------------------------------- diff --git a/spinifex/daemon/perf_bench_test.go b/spinifex/daemon/perf_bench_test.go index c07d23697..eca758e4d 100644 --- a/spinifex/daemon/perf_bench_test.go +++ b/spinifex/daemon/perf_bench_test.go @@ -5,10 +5,13 @@ import ( "testing" ) -// Phase 1 gate constants — changing these is a deliberate decision. +// Phase gate constants — changing these is a deliberate decision. const ( phase1BootGateMs = 500 phase1RSSGateMiB = 25 + + phase2BootGateMs = 300 + phase2ArtifactGateMiB = 50 ) func TestBootGate_Phase1(t *testing.T) { @@ -20,6 +23,15 @@ func TestBootGate_Phase1(t *testing.T) { } } +func TestBootGate_Phase2(t *testing.T) { + if phase2BootGateMs != 300 { + t.Errorf("Phase 2 boot gate changed: got %d, want 300", phase2BootGateMs) + } + if phase2ArtifactGateMiB != 50 { + t.Errorf("Phase 2 artifact gate changed: got %d, want 50", phase2ArtifactGateMiB) + } +} + func TestMeasureQEMURSSMiB_Self(t *testing.T) { rss, err := measureQEMURSSMiB(os.Getpid()) if err != nil { diff --git a/spinifex/handlers/elbv2/arn_test.go b/spinifex/handlers/elbv2/arn_test.go index 24de00a39..da0f0f73c 100644 --- a/spinifex/handlers/elbv2/arn_test.go +++ b/spinifex/handlers/elbv2/arn_test.go @@ -412,3 +412,27 @@ func TestLbVMUserData_MissingCredentials(t *testing.T) { }) } } + +// TestBuildMicrovmNICs_NilVPC verifies buildMicrovmNICs works when VPCService +// is nil — CIDR and gateway remain empty, NIC structure is still correct. +func TestBuildMicrovmNICs_NilVPC(t *testing.T) { + svc := &ELBv2ServiceImpl{VPCService: nil} + nics := svc.buildMicrovmNICs("10.0.1.5", "02:aa:bb:cc:dd:01", "subnet-abc", "eni-abc", SchemeInternal, nil, testAccountID) + require.Len(t, nics, 2, "primary ENI NIC + mgmt NIC") + assert.True(t, nics[0].IsDefault) + assert.Equal(t, "02:aa:bb:cc:dd:01", nics[0].MAC) + assert.Empty(t, nics[0].CIDR, "CIDR unknown when VPC service unavailable") + assert.False(t, nics[1].IsDefault) +} + +// TestBuildMicrovmNICs_ExtraENIs verifies NIC[2+] are appended for multi-subnet ALBs. +func TestBuildMicrovmNICs_ExtraENIs(t *testing.T) { + svc := &ELBv2ServiceImpl{VPCService: nil} + extras := []ExtraENIInput{ + {ENIID: "eni-extra1", ENIMac: "02:aa:bb:cc:dd:02", ENIIP: "10.0.2.5", SubnetID: "subnet-extra1"}, + } + nics := svc.buildMicrovmNICs("10.0.1.5", "02:aa:bb:cc:dd:01", "subnet-abc", "eni-abc", SchemeInternal, extras, testAccountID) + require.Len(t, nics, 3, "primary ENI + mgmt + 1 extra") + assert.Equal(t, "02:aa:bb:cc:dd:02", nics[2].MAC) + assert.False(t, nics[2].IsDefault) +}