feat(daemon): direct-boot launch path for microvm system instances

When SystemInstanceInput.DirectBoot=true:
- Skip GetAMIConfig and GenerateVolumes
- Build vm.Config directly (no buildBaseVMConfig, no PCIe root ports)
- MachineType=microvm,x-option-roms=off,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on
- Resolve kernel/initrd from Microvm.ImagePath config field
- Serialise NICs to three fw_cfg tmpfiles (netcfg, lb-agent-env, ca-cert)
- Validate exactly one NICConfig.IsDefault=true before writing netcfg blob
- Shutdown path cleans up tmpfiles via existing Config.FwCfg iteration

PC-machine path (DirectBoot=false) unchanged.

Closes mulga-siv-65
This commit is contained in:
joshsiv-mulga
2026-05-17 00:40:30 +10:00
parent 7a8a07b653
commit 228e428ec7
4 changed files with 336 additions and 78 deletions
+229 -23
View File
@@ -4,6 +4,9 @@ import (
"encoding/base64"
"fmt"
"log/slog"
"os"
"path/filepath"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
@@ -24,6 +27,11 @@ var _ handlers_elbv2.SystemInstanceLauncher = (*Daemon)(nil)
// This is the internal equivalent of RunInstances — it reuses the same
// instance service, resource manager, volume preparation, and QEMU launch
// path, but skips the NATS request/response envelope and key pair validation.
//
// When input.DirectBoot is true the PC-machine path (AMI lookup, volume
// generation, cloud-init) is bypassed entirely: the VM boots a bundled
// vmlinuz+initramfs via direct kernel boot with network and lb-agent
// configuration delivered through QEMU fw_cfg blobs.
func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) (*handlers_elbv2.SystemInstanceOutput, error) {
accountID := utils.GlobalAccountID
// ENI account may differ from system account — the ENI is created under
@@ -39,12 +47,14 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
return nil, fmt.Errorf("unknown instance type: %s", input.InstanceType)
}
// Validate AMI
if d.imageService == nil {
return nil, fmt.Errorf("image service not initialized")
}
if _, err := d.imageService.GetAMIConfig(input.ImageID); err != nil {
return nil, fmt.Errorf("AMI %s not found: %w", input.ImageID, err)
if !input.DirectBoot {
// Validate AMI — only needed for the PC-machine path.
if d.imageService == nil {
return nil, fmt.Errorf("image service not initialized")
}
if _, err := d.imageService.GetAMIConfig(input.ImageID); err != nil {
return nil, fmt.Errorf("AMI %s not found: %w", input.ImageID, err)
}
}
// Allocate resources
@@ -57,7 +67,6 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
// customer-facing listings (Nodes page).
runInput := &ec2.RunInstancesInput{
InstanceType: aws.String(input.InstanceType),
ImageId: aws.String(input.ImageID),
MinCount: aws.Int64(1),
MaxCount: aws.Int64(1),
TagSpecifications: []*ec2.TagSpecification{
@@ -69,6 +78,9 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
},
},
}
if !input.DirectBoot && input.ImageID != "" {
runInput.ImageId = aws.String(input.ImageID)
}
if input.SubnetID != "" {
runInput.SubnetId = aws.String(input.SubnetID)
}
@@ -302,23 +314,36 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput)
}
}
// Prepare volumes (root volume clone from AMI, cloud-init ISO, etc.)
volumeInfos, err := d.instanceService.GenerateVolumes(runInput, instance)
if err != nil {
d.cleanupFailedSystemInstance(instance, instanceType)
return nil, fmt.Errorf("generate volumes: %w", err)
}
if input.DirectBoot {
// Direct-boot (microvm) path: skip volume generation and cloud-init.
// Build the vm.Config directly with microvm machine settings and
// fw_cfg blobs for network, lb-agent env, and CA cert.
cfg, err := d.buildDirectBootConfig(instance.ID, input)
if err != nil {
d.cleanupFailedSystemInstance(instance, instanceType)
return nil, fmt.Errorf("build direct-boot config: %w", err)
}
instance.Config = cfg
instance.DirectBoot = true
} else {
// PC-machine path: clone root volume, generate cloud-init ISO.
volumeInfos, err := d.instanceService.GenerateVolumes(runInput, instance)
if err != nil {
d.cleanupFailedSystemInstance(instance, instanceType)
return nil, fmt.Errorf("generate volumes: %w", err)
}
instance.Instance.BlockDeviceMappings = make([]*ec2.InstanceBlockDeviceMapping, 0, len(volumeInfos))
for _, vi := range volumeInfos {
mapping := &ec2.InstanceBlockDeviceMapping{}
mapping.SetDeviceName(vi.DeviceName)
mapping.Ebs = &ec2.EbsInstanceBlockDevice{}
mapping.Ebs.SetVolumeId(vi.VolumeId)
mapping.Ebs.SetAttachTime(vi.AttachTime)
mapping.Ebs.SetDeleteOnTermination(vi.DeleteOnTermination)
mapping.Ebs.SetStatus("attached")
instance.Instance.BlockDeviceMappings = append(instance.Instance.BlockDeviceMappings, mapping)
instance.Instance.BlockDeviceMappings = make([]*ec2.InstanceBlockDeviceMapping, 0, len(volumeInfos))
for _, vi := range volumeInfos {
mapping := &ec2.InstanceBlockDeviceMapping{}
mapping.SetDeviceName(vi.DeviceName)
mapping.Ebs = &ec2.EbsInstanceBlockDevice{}
mapping.Ebs.SetVolumeId(vi.VolumeId)
mapping.Ebs.SetAttachTime(vi.AttachTime)
mapping.Ebs.SetDeleteOnTermination(vi.DeleteOnTermination)
mapping.Ebs.SetStatus("attached")
instance.Instance.BlockDeviceMappings = append(instance.Instance.BlockDeviceMappings, mapping)
}
}
// Launch QEMU VM
@@ -418,3 +443,184 @@ func (d *Daemon) WaitForSystemInstance(instanceID string, timeout time.Duration)
}
return fmt.Errorf("instance %s did not reach running state within %s", instanceID, timeout)
}
// buildDirectBootConfig constructs a vm.Config for a direct-boot microVM
// launch. It does not call buildBaseVMConfig — the microvm machine type
// requires no PCIe root ports, no UEFI/BIOS, and no drives. Runtime paths
// (PID file, console log, serial socket) are filled in later by startQEMU.
//
// Network topology from input.NICs is serialised to three fw_cfg tmpfiles
// (netcfg, lb-agent-env, ca-cert) under utils.RuntimeDir().
func (d *Daemon) buildDirectBootConfig(instanceID string, input *handlers_elbv2.SystemInstanceInput) (vm.Config, error) {
it := d.resourceMgr.instanceTypes[input.InstanceType]
architecture := "x86_64"
if it != nil && it.ProcessorInfo != nil && len(it.ProcessorInfo.SupportedArchitectures) > 0 && it.ProcessorInfo.SupportedArchitectures[0] != nil {
architecture = *it.ProcessorInfo.SupportedArchitectures[0]
}
vcpus := 1
memMiB := 128
if it != nil {
vcpus = int(instanceTypeVCPUs(it))
memMiB = int(instanceTypeMemoryMiB(it))
}
imagePath := d.config.Daemon.Microvm.ImagePath
if imagePath == "" {
imagePath = "/usr/share/spinifex/microvm"
}
fwCfg, err := d.writeFwCfgBlobs(instanceID, input)
if err != nil {
return vm.Config{}, fmt.Errorf("write fw_cfg blobs: %w", err)
}
cfg := vm.Config{
Name: instanceID,
EnableKVM: true,
NoGraphic: true,
Architecture: architecture,
CPUType: "host",
CPUCount: vcpus,
Memory: memMiB,
MachineType: "microvm,x-option-roms=off,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on",
KernelImage: filepath.Join(imagePath, "vmlinuz"),
Initrd: filepath.Join(imagePath, "initramfs.cpio.gz"),
KernelCmdline: "console=ttyS0 quiet",
FwCfg: fwCfg,
}
// Wire netdevs and devices for each NIC in declaration order.
// Tap device creation happens later in startQEMU (host-side only here).
allNICs := buildNICNetdevs(instanceID, input)
cfg.NetDevs = append(cfg.NetDevs, allNICs.netdevs...)
cfg.Devices = append(cfg.Devices, allNICs.devices...)
return cfg, nil
}
// nicNetdevResult holds the QEMU netdev and device entries for all NICs.
type nicNetdevResult struct {
netdevs []vm.NetDev
devices []vm.Device
}
// buildNICNetdevs produces QEMU -netdev and -device entries for each NIC in
// input. The NIC order determines the netdev IDs: net0 = primary ENI,
// net1 = mgmt (if present), net2+ = extra ENIs. Extra ENIs beyond the primary
// are only included when corresponding ExtraENIs entries exist.
func buildNICNetdevs(instanceID string, input *handlers_elbv2.SystemInstanceInput) nicNetdevResult {
var res nicNetdevResult
machineType := "microvm,x-option-roms=off,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on"
for i, nic := range input.NICs {
netID := fmt.Sprintf("net%d", i)
// Resolve the tap name from the corresponding ENI.
tapName := tapNameForNIC(i, nic, instanceID, input)
res.netdevs = append(res.netdevs, vm.NetDev{
Value: fmt.Sprintf("tap,id=%s,ifname=%s,script=no,downscript=no", netID, tapName),
})
res.devices = append(res.devices, vm.NetDevice(machineType, netID, nic.MAC))
}
return res
}
// tapNameForNIC returns the Linux tap device name for a NIC at the given index.
// Index 0 → primary ENI tap, index 1 → mgmt tap, index 2+ → extra ENI taps.
func tapNameForNIC(idx int, _ handlers_elbv2.NICConfig, instanceID string, input *handlers_elbv2.SystemInstanceInput) string {
switch idx {
case 0:
return vm.TapDeviceName(input.ENIID)
case 1:
return vm.MgmtTapName(instanceID)
default:
extraIdx := idx - 2
if extraIdx < len(input.ExtraENIs) {
return vm.TapDeviceName(input.ExtraENIs[extraIdx].ENIID)
}
return fmt.Sprintf("tap-unknown-%d", idx)
}
}
// writeFwCfgBlobs serialises NIC configuration, lb-agent env, and CA cert to
// per-VM tmpfiles under utils.RuntimeDir(). Returns the fw_cfg entries for the
// three blobs and an error if any write fails or the NIC default invariant is
// violated.
func (d *Daemon) writeFwCfgBlobs(instanceID string, input *handlers_elbv2.SystemInstanceInput) ([]vm.FwCfgEntry, error) {
runtimeDir := utils.RuntimeDir()
netcfgPath := filepath.Join(runtimeDir, fmt.Sprintf("fwcfg-%s-netcfg.tmp", instanceID))
lbenvPath := filepath.Join(runtimeDir, fmt.Sprintf("fwcfg-%s-lbenv.tmp", instanceID))
cacertPath := filepath.Join(runtimeDir, fmt.Sprintf("fwcfg-%s-cacert.tmp", instanceID))
netcfg, err := buildNetcfgBlob(input.NICs)
if err != nil {
return nil, err
}
if err := os.WriteFile(netcfgPath, []byte(netcfg), 0600); err != nil {
return nil, fmt.Errorf("write netcfg blob: %w", err)
}
if err := os.WriteFile(lbenvPath, []byte(input.LBAgentEnv), 0600); err != nil {
_ = os.Remove(netcfgPath)
return nil, fmt.Errorf("write lb-agent-env blob: %w", err)
}
if err := os.WriteFile(cacertPath, []byte(input.CACert), 0600); err != nil {
_ = os.Remove(netcfgPath)
_ = os.Remove(lbenvPath)
return nil, fmt.Errorf("write ca-cert blob: %w", err)
}
slog.Debug("wrote fw_cfg blobs",
"instanceId", instanceID,
"netcfg", netcfgPath,
"lbenv", lbenvPath,
"cacert", cacertPath,
)
return []vm.FwCfgEntry{
{Name: "opt/spinifex/netcfg", File: netcfgPath},
{Name: "opt/spinifex/lb-agent-env", File: lbenvPath},
{Name: "opt/spinifex/ca-cert", File: cacertPath},
}, nil
}
// buildNetcfgBlob serialises a slice of NICConfig entries to the shell KEY=value
// format consumed by the initramfs init script. Exactly one NIC must have
// IsDefault=true; returns an error otherwise.
func buildNetcfgBlob(nics []handlers_elbv2.NICConfig) (string, error) {
defaultCount := 0
for _, n := range nics {
if n.IsDefault {
defaultCount++
}
}
if defaultCount != 1 {
return "", fmt.Errorf("netcfg: exactly one NIC must have IsDefault=true, got %d", defaultCount)
}
var sb strings.Builder
for i, n := range nics {
prefix := fmt.Sprintf("NIC%d", i)
fmt.Fprintf(&sb, "%s_MAC=%s\n", prefix, n.MAC)
if n.CIDR != "" {
fmt.Fprintf(&sb, "%s_CIDR=%s\n", prefix, n.CIDR)
}
if n.Gateway != "" {
fmt.Fprintf(&sb, "%s_GW=%s\n", prefix, n.Gateway)
}
if n.IsDefault {
fmt.Fprintf(&sb, "%s_DEFAULT=1\n", prefix)
} else {
fmt.Fprintf(&sb, "%s_DEFAULT=0\n", prefix)
}
if n.RouteDst != "" {
fmt.Fprintf(&sb, "%s_ROUTE_DST=%s\n", prefix, n.RouteDst)
}
if n.RouteVia != "" {
fmt.Fprintf(&sb, "%s_ROUTE_VIA=%s\n", prefix, n.RouteVia)
}
}
return sb.String(), nil
}
+8
View File
@@ -59,6 +59,14 @@ type SystemInstanceInput struct {
// Index 0 is the primary VPC ENI, index 1 is the management NIC, index 2+ are extra ENIs.
// Ignored when DirectBoot is false.
NICs []NICConfig
// LBAgentEnv is a KEY=value blob written to /etc/conf.d/lb-agent inside
// the guest via fw_cfg on the direct-boot path. Ignored when DirectBoot is false.
LBAgentEnv string
// CACert holds PEM-encoded CA certificate bytes delivered to the guest via
// fw_cfg on the direct-boot path. Ignored when DirectBoot is false.
CACert string
}
// ExtraENIInput describes an additional pre-created ENI to attach to a
+93 -55
View File
@@ -192,72 +192,110 @@ func (m *Manager) startQEMU(instance *VM) error {
consoleLogPath := filepath.Join(runtimeDir, fmt.Sprintf("console-%s.log", instance.ID))
serialSocket := filepath.Join(runtimeDir, fmt.Sprintf("serial-%s.sock", instance.ID))
instance.Config = buildBaseVMConfig(instance.ID, pidFile, consoleLogPath, serialSocket, spec.Architecture, spec.VCPUs, spec.MemoryMiB)
if instance.DirectBoot {
// Direct-boot (microvm) path: vm.Config was pre-built by the launcher.
// Only fill in the runtime-generated paths that are not known until now.
instance.Config.PIDFile = pidFile
instance.Config.ConsoleLogPath = consoleLogPath
instance.Config.SerialSocket = serialSocket
} else {
instance.Config = buildBaseVMConfig(instance.ID, pidFile, consoleLogPath, serialSocket, spec.Architecture, spec.VCPUs, spec.MemoryMiB)
instance.EBSRequests.Mu.Lock()
drives, iothreads, devices, err := buildDrives(instance.EBSRequests.Requests, spec.VCPUs, instance.Config.MachineType)
instance.EBSRequests.Mu.Unlock()
if err != nil {
return err
}
instance.Config.Drives = append(instance.Config.Drives, drives...)
instance.Config.IOThreads = append(instance.Config.IOThreads, iothreads...)
instance.Config.Devices = append(instance.Config.Devices, devices...)
if instance.ENIId != "" && m.deps.NetworkPlumber != nil {
spec := VPCTapSpec(instance.ENIId, instance.ENIMac)
if err := m.deps.NetworkPlumber.SetupTap(spec); err != nil {
slog.Error("Failed to set up tap device", "eni", instance.ENIId, "err", err)
return fmt.Errorf("setup tap device: %w", err)
}
tapName := spec.Name
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{
Value: fmt.Sprintf("tap,id=net0,ifname=%s,script=no,downscript=no", tapName),
})
instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, "net0", instance.ENIMac))
slog.Info("VPC networking configured", "tap", tapName, "eni", instance.ENIId, "mac", instance.ENIMac)
if err := m.setupExtraENINICs(instance); err != nil {
instance.EBSRequests.Mu.Lock()
drives, iothreads, devices, err := buildDrives(instance.EBSRequests.Requests, spec.VCPUs, instance.Config.MachineType)
instance.EBSRequests.Mu.Unlock()
if err != nil {
return err
}
instance.Config.Drives = append(instance.Config.Drives, drives...)
instance.Config.IOThreads = append(instance.Config.IOThreads, iothreads...)
instance.Config.Devices = append(instance.Config.Devices, devices...)
}
if m.deps.DevNetworking {
m.appendDevHostfwdNIC(instance)
if instance.DirectBoot {
// Direct-boot (microvm) path: NetDevs and Devices are already set in
// the pre-built Config. Only create host-side tap devices for VPC ENIs
// so the kernel tap interfaces exist before QEMU opens them.
if instance.ENIId != "" && m.deps.NetworkPlumber != nil {
tapSpec := VPCTapSpec(instance.ENIId, instance.ENIMac)
if err := m.deps.NetworkPlumber.SetupTap(tapSpec); err != nil {
slog.Error("Failed to set up tap device (direct-boot)", "eni", instance.ENIId, "err", err)
return fmt.Errorf("setup tap device: %w", err)
}
slog.Info("VPC tap configured (direct-boot)", "tap", tapSpec.Name, "eni", instance.ENIId, "mac", instance.ENIMac)
for _, extra := range instance.ExtraENIs {
extraSpec := VPCTapSpec(extra.ENIID, extra.ENIMac)
if err := m.deps.NetworkPlumber.SetupTap(extraSpec); err != nil {
slog.Error("Failed to set up extra ENI tap (direct-boot)", "eni", extra.ENIID, "err", err)
return fmt.Errorf("setup tap device for extra ENI %s: %w", extra.ENIID, err)
}
}
}
if instance.MgmtMAC != "" && m.deps.NetworkPlumber != nil {
mgmtTap := MgmtTapName(instance.ID)
mgmtBridge := "br-mgmt"
if err := m.deps.NetworkPlumber.SetupTap(TapSpec{Name: mgmtTap, Bridge: mgmtBridge}); err != nil {
slog.Error("Failed to set up mgmt tap (direct-boot)", "tap", mgmtTap, "err", err)
return fmt.Errorf("setup mgmt tap: %w", err)
}
slog.Info("Mgmt tap configured (direct-boot)", "tap", mgmtTap, "mac", instance.MgmtMAC, "ip", instance.MgmtIP, "instanceId", instance.ID)
}
} else {
sshDebugAddr, err := viperblock.FindFreePort()
if err != nil {
slog.Error("Failed to find free port", "err", err)
return err
}
_, sshDebugPort, err := net.SplitHostPort(sshDebugAddr)
if err != nil {
slog.Error("Failed to parse port from address", "addr", sshDebugAddr, "err", err)
return fmt.Errorf("parse port from %s: %w", sshDebugAddr, err)
if instance.ENIId != "" && m.deps.NetworkPlumber != nil {
spec := VPCTapSpec(instance.ENIId, instance.ENIMac)
if err := m.deps.NetworkPlumber.SetupTap(spec); err != nil {
slog.Error("Failed to set up tap device", "eni", instance.ENIId, "err", err)
return fmt.Errorf("setup tap device: %w", err)
}
tapName := spec.Name
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{
Value: fmt.Sprintf("tap,id=net0,ifname=%s,script=no,downscript=no", tapName),
})
instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, "net0", instance.ENIMac))
slog.Info("VPC networking configured", "tap", tapName, "eni", instance.ENIId, "mac", instance.ENIMac)
if err := m.setupExtraENINICs(instance); err != nil {
return err
}
if m.deps.DevNetworking {
m.appendDevHostfwdNIC(instance)
}
} else {
sshDebugAddr, err := viperblock.FindFreePort()
if err != nil {
slog.Error("Failed to find free port", "err", err)
return err
}
_, sshDebugPort, err := net.SplitHostPort(sshDebugAddr)
if err != nil {
slog.Error("Failed to parse port from address", "addr", sshDebugAddr, "err", err)
return fmt.Errorf("parse port from %s: %w", sshDebugAddr, err)
}
bindIP := m.deps.BindHost
if bindIP == "" || bindIP == "0.0.0.0" {
bindIP = "127.0.0.1"
}
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{
Value: fmt.Sprintf("user,id=net0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort),
})
instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, "net0", ""))
}
bindIP := m.deps.BindHost
if bindIP == "" || bindIP == "0.0.0.0" {
bindIP = "127.0.0.1"
if instance.MgmtMAC != "" {
mgmtTap := MgmtTapName(instance.ID)
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{
Value: fmt.Sprintf("tap,id=mgmt0,ifname=%s,script=no,downscript=no", mgmtTap),
})
instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, "mgmt0", instance.MgmtMAC))
slog.Info("Management NIC configured", "tap", mgmtTap, "mac", instance.MgmtMAC, "ip", instance.MgmtIP, "instanceId", instance.ID)
}
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{
Value: fmt.Sprintf("user,id=net0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort),
})
instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, "net0", ""))
instance.Config.Devices = append(instance.Config.Devices, RngDevice(instance.Config.MachineType))
}
if instance.MgmtMAC != "" {
mgmtTap := MgmtTapName(instance.ID)
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{
Value: fmt.Sprintf("tap,id=mgmt0,ifname=%s,script=no,downscript=no", mgmtTap),
})
instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, "mgmt0", instance.MgmtMAC))
slog.Info("Management NIC configured", "tap", mgmtTap, "mac", instance.MgmtMAC, "ip", instance.MgmtIP, "instanceId", instance.ID)
}
instance.Config.Devices = append(instance.Config.Devices, RngDevice(instance.Config.MachineType))
for i, addr := range instance.GPUPCIAddresses {
xvga := "off"
if instance.GPUXVGAEnabled {
+6
View File
@@ -119,6 +119,12 @@ type VM struct {
// filters out tagged VMs from customer-facing listings.
ManagedBy string `json:"managed_by,omitempty"`
// DirectBoot signals that this VM was configured with a pre-built
// vm.Config by the launcher (microvm direct-boot path). When true,
// startQEMU skips buildBaseVMConfig and uses the existing Config
// instead. False for all PC-machine VMs.
DirectBoot bool `json:"direct_boot,omitempty"`
// GPUPCIAddresses holds the PCI addresses of all GPUs bound to this instance via
// VFIO (e.g. ["0000:03:00.0"]). Empty for non-GPU instances. Single-GPU instances
// have one entry; multi-GPU instances have one per assigned GPU.