gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-19 10:52:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(microvm): Phase 2 kernel stripping + boot optimisation

Browse Source 
- cpu: switch to host,-pmu (disable PMU — not needed in microvm guest)
- machine type: make isa-serial configurable via microvm.isa_serial config
  field; default true (on) preserves console log path
- build: strip unused kernel modules (USB, IDE, ATA, GPU, media, input,
  HID, bluetooth, staging, sound, wireless) from initramfs before cpio
- build: add Phase 2 artifact size gate — fail if vmlinuz + initramfs.cpio.gz
  exceeds MICROVM_ARTIFACT_MAX_MiB (default 50 MiB); override via env var
  during development
- perf: add phase2BootGateMs=300 and phase2ArtifactGateMiB=50 constants
  with canary tests to catch unintentional gate changes
- tests: cover buildMicrovmNICs nil-VPC and extra-ENI paths
- tests: cover microvmMachineType nil/true/false branches
This commit is contained in:
joshsiv-mulga
2026-05-17 13:47:35 +10:00
parent ae721e7a73
commit 1ea94061fc
7 changed files with 131 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/spinifex/cmd/templates/spinifex.toml
+1
View File
@@ -75,6 +75,7 @@ gpu_passthrough = true
# [nodes.{{.Node}}.daemon.microvm]
# elbv2_enabled = false
# image_path = "/usr/share/spinifex/microvm"
# isa_serial = true # set false only after measuring boot savings; disables per-VM console log
[nodes.{{.Node}}.awsgw]
host = "{{.BindIP}}:9999"
scripts/build-microvm-image.sh
+38
View File
@@ -158,6 +158,31 @@ else
echo "[build-microvm-image] fw_cfg: module found in lib/modules"
fi
# --- Strip unused kernel modules (Phase 2) ---
# Remove driver categories not present on a virtio-only microvm guest.
# linux-virt may include some of these as loadable modules; stripping them
# here reduces initramfs size without rebuilding the kernel from source.
echo "[build-microvm-image] stripping unused kernel modules..."
for kver_dir in "$CHROOT_DIR/lib/modules"/*/; do
[ -d "$kver_dir/kernel" ] || continue
rm -rf \
"$kver_dir/kernel/drivers/usb" \
"$kver_dir/kernel/drivers/ide" \
"$kver_dir/kernel/drivers/ata" \
"$kver_dir/kernel/drivers/gpu" \
"$kver_dir/kernel/drivers/video" \
"$kver_dir/kernel/drivers/media" \
"$kver_dir/kernel/drivers/input" \
"$kver_dir/kernel/drivers/hid" \
"$kver_dir/kernel/drivers/bluetooth" \
"$kver_dir/kernel/drivers/staging" \
"$kver_dir/kernel/sound" \
"$kver_dir/kernel/net/wireless" \
"$kver_dir/kernel/net/bluetooth"
# Regenerate module dependency map after stripping.
depmod -b "$CHROOT_DIR" "$(basename "$kver_dir")" 2>/dev/null || true
done
# --- Build initramfs ---
INITRAMFS_OUT="$BUILD_DIR/initramfs.cpio.gz"
echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT"
@@ -176,4 +201,17 @@ echo ""
echo "[build-microvm-image] artifacts:"
ls -lh "$VMLINUZ_OUT" "$INITRAMFS_OUT"
echo ""
# --- Phase 2 artifact size gate (50 MiB total) ---
ARTIFACT_MAX_MiB="${MICROVM_ARTIFACT_MAX_MiB:-50}"
total_bytes=$(( $(stat -c %s "$VMLINUZ_OUT") + $(stat -c %s "$INITRAMFS_OUT") ))
max_bytes=$(( ARTIFACT_MAX_MiB * 1024 * 1024 ))
total_mib=$(( (total_bytes + 1048575) / 1048576 ))
if [ "$total_bytes" -gt "$max_bytes" ]; then
echo "ERROR: artifact size ${total_mib} MiB exceeds ${ARTIFACT_MAX_MiB} MiB gate" >&2
echo " Set MICROVM_ARTIFACT_MAX_MiB=N to override during development." >&2
exit 1
fi
echo "[build-microvm-image] artifact size gate: PASS (${total_mib} MiB / ${ARTIFACT_MAX_MiB} MiB)"
echo ""
echo "[build-microvm-image] done."
spinifex/config/config.go
+9
View File
@@ -142,6 +142,10 @@ type GPUModelOverride struct {
type MicrovmConfig struct {
ELBv2Enabled bool `json:"ELBv2Enabled" mapstructure:"elbv2_enabled"`
ImagePath string `json:"ImagePath" mapstructure:"image_path"`
// IsaSerial controls the isa-serial device on the microvm machine. Default
// true keeps the console log path working. Set false only after measuring
// boot-time savings and accepting loss of per-VM console output.
IsaSerial *bool `json:"IsaSerial" mapstructure:"isa_serial"`
}
// DaemonConfig holds the daemon configuration
@@ -253,6 +257,11 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
local.Daemon.Microvm.ImagePath = "/usr/share/spinifex/microvm"
config.Nodes[config.Node] = local
}
if local.Daemon.Microvm.IsaSerial == nil {
t := true
local.Daemon.Microvm.IsaSerial = &t
config.Nodes[config.Node] = local
}
}
return &config, nil
spinifex/daemon/daemon_system_instance.go
+20 -7
View File
@@ -492,21 +492,25 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2.
// QEMU microvm maps virtio-mmio slots at 0xfeb00000 + n*0x200 with IRQ 5+n.
// Without these params the kernel never discovers the devices (auto-kernel-cmdline
// does not append when -append is specified).
cmdline := "console=ttyS0"
var sb strings.Builder
sb.WriteString("console=ttyS0")
for i := range input.NICs {
cmdline += fmt.Sprintf(" virtio_mmio.device=0x200@0x%x:%d",
fmt.Fprintf(&sb, " virtio_mmio.device=0x200@0x%x:%d",
0xfeb00000+i*0x200, 5+i)
}
cmdline := sb.String()
machineType := microvmMachineType(d.config.Daemon.Microvm.IsaSerial)
cfg := vm.Config{
Name: instanceID,
EnableKVM: true,
NoGraphic: true,
Architecture: architecture,
CPUType: "host",
CPUType: "host,-pmu",
CPUCount: vcpus,
Memory: memMiB,
MachineType: "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on",
MachineType: machineType,
KernelImage: filepath.Join(imagePath, "vmlinuz"),
Initrd: filepath.Join(imagePath, "initramfs.cpio.gz"),
KernelCmdline: cmdline,
@@ -515,7 +519,7 @@ func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2.
// Wire netdevs and devices for each NIC in declaration order.
// Tap device creation happens later in startQEMU (host-side only here).
allNICs := buildNICNetdevs(instanceID, input)
allNICs := buildNICNetdevs(instanceID, input, machineType)
cfg.NetDevs = append(cfg.NetDevs, allNICs.netdevs...)
cfg.Devices = append(cfg.Devices, allNICs.devices...)
@@ -528,13 +532,22 @@ type nicNetdevResult struct {
devices []vm.Device
}
// microvmMachineType returns the QEMU -M string for the microvm machine.
// isaSerial nil or true keeps isa-serial=on for console log access.
func microvmMachineType(isaSerial *bool) string {
serial := "on"
if isaSerial != nil && !*isaSerial {
serial = "off"
}
return "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=" + serial
}
// buildNICNetdevs produces QEMU -netdev and -device entries for each NIC in
// input. The NIC order determines the netdev IDs: net0 = primary ENI,
// net1 = mgmt (if present), net2+ = extra ENIs. Extra ENIs beyond the primary
// are only included when corresponding ExtraENIs entries exist.
func buildNICNetdevs(instanceID string, input *handlers_elbv2.SystemInstanceInput) nicNetdevResult {
func buildNICNetdevs(instanceID string, input *handlers_elbv2.SystemInstanceInput, machineType string) nicNetdevResult {
var res nicNetdevResult
machineType := "microvm,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on"
for i, nic := range input.NICs {
netID := fmt.Sprintf("net%d", i)
spinifex/daemon/daemon_system_instance_test.go
+26 -3
View File
@@ -197,7 +197,7 @@ func TestBuildNICNetdevs_SingleNIC(t *testing.T) {
{MAC: "02:aa:bb:cc:dd:01", IsDefault: true},
},
}
res := buildNICNetdevs("i-test", input)
res := buildNICNetdevs("i-test", input, microvmMachineType(nil))
require.Len(t, res.netdevs, 1)
require.Len(t, res.devices, 1)
assert.Contains(t, res.netdevs[0].Value, "tap,id=net0,")
@@ -214,7 +214,7 @@ func TestBuildNICNetdevs_TwoNICs(t *testing.T) {
{MAC: "02:aa:bb:cc:dd:02", IsDefault: false},
},
}
res := buildNICNetdevs("i-test2", input)
res := buildNICNetdevs("i-test2", input, microvmMachineType(nil))
require.Len(t, res.netdevs, 2)
require.Len(t, res.devices, 2)
assert.Contains(t, res.netdevs[0].Value, "id=net0,")
@@ -225,11 +225,34 @@ func TestBuildNICNetdevs_TwoNICs(t *testing.T) {
func TestBuildNICNetdevs_EmptyNICs(t *testing.T) {
input := &handlers_elbv2.SystemInstanceInput{ENIID: "eni-empty"}
res := buildNICNetdevs("i-empty", input)
res := buildNICNetdevs("i-empty", input, microvmMachineType(nil))
assert.Empty(t, res.netdevs)
assert.Empty(t, res.devices)
}
// ---------------------------------------------------------------------------
// microvmMachineType
// ---------------------------------------------------------------------------
func TestMicrovmMachineType_DefaultNil(t *testing.T) {
mt := microvmMachineType(nil)
assert.Contains(t, mt, "isa-serial=on")
assert.Contains(t, mt, "microvm,")
}
func TestMicrovmMachineType_ExplicitTrue(t *testing.T) {
v := true
mt := microvmMachineType(&v)
assert.Contains(t, mt, "isa-serial=on")
}
func TestMicrovmMachineType_ExplicitFalse(t *testing.T) {
v := false
mt := microvmMachineType(&v)
assert.Contains(t, mt, "isa-serial=off")
assert.NotContains(t, mt, "isa-serial=on")
}
// ---------------------------------------------------------------------------
// writeFwCfgBlobs
// ---------------------------------------------------------------------------
spinifex/daemon/perf_bench_test.go
+13 -1
View File
@@ -5,10 +5,13 @@ import (
"testing"
)
// Phase 1 gate constants — changing these is a deliberate decision.
// Phase gate constants — changing these is a deliberate decision.
const (
phase1BootGateMs = 500
phase1RSSGateMiB = 25
phase2BootGateMs = 300
phase2ArtifactGateMiB = 50
)
func TestBootGate_Phase1(t *testing.T) {
@@ -20,6 +23,15 @@ func TestBootGate_Phase1(t *testing.T) {
}
}
func TestBootGate_Phase2(t *testing.T) {
if phase2BootGateMs != 300 {
t.Errorf("Phase 2 boot gate changed: got %d, want 300", phase2BootGateMs)
}
if phase2ArtifactGateMiB != 50 {
t.Errorf("Phase 2 artifact gate changed: got %d, want 50", phase2ArtifactGateMiB)
}
}
func TestMeasureQEMURSSMiB_Self(t *testing.T) {
rss, err := measureQEMURSSMiB(os.Getpid())
if err != nil {
spinifex/handlers/elbv2/arn_test.go
+24
View File
@@ -412,3 +412,27 @@ func TestLbVMUserData_MissingCredentials(t *testing.T) {
})
}
}
// TestBuildMicrovmNICs_NilVPC verifies buildMicrovmNICs works when VPCService
// is nil — CIDR and gateway remain empty, NIC structure is still correct.
func TestBuildMicrovmNICs_NilVPC(t *testing.T) {
svc := &ELBv2ServiceImpl{VPCService: nil}
nics := svc.buildMicrovmNICs("10.0.1.5", "02:aa:bb:cc:dd:01", "subnet-abc", "eni-abc", SchemeInternal, nil, testAccountID)
require.Len(t, nics, 2, "primary ENI NIC + mgmt NIC")
assert.True(t, nics[0].IsDefault)
assert.Equal(t, "02:aa:bb:cc:dd:01", nics[0].MAC)
assert.Empty(t, nics[0].CIDR, "CIDR unknown when VPC service unavailable")
assert.False(t, nics[1].IsDefault)
}
// TestBuildMicrovmNICs_ExtraENIs verifies NIC[2+] are appended for multi-subnet ALBs.
func TestBuildMicrovmNICs_ExtraENIs(t *testing.T) {
svc := &ELBv2ServiceImpl{VPCService: nil}
extras := []ExtraENIInput{
{ENIID: "eni-extra1", ENIMac: "02:aa:bb:cc:dd:02", ENIIP: "10.0.2.5", SubnetID: "subnet-extra1"},
}
nics := svc.buildMicrovmNICs("10.0.1.5", "02:aa:bb:cc:dd:01", "subnet-abc", "eni-abc", SchemeInternal, extras, testAccountID)
require.Len(t, nics, 3, "primary ENI + mgmt + 1 extra")
assert.Equal(t, "02:aa:bb:cc:dd:02", nics[2].MAC)
assert.False(t, nics[2].IsDefault)
}