mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-21 03:36:35 +00:00
feat(vm): direct-boot foundations + config/input extensions + image build pipeline
vm package (mulga-siv-62):
- Add KernelImage, Initrd, KernelCmdline, FwCfg/FwCfgEntry to vm.Config
- Emit -kernel/-initrd/-append/-fw_cfg args when fields set
- Relax drive invariant: accept kernel image in place of drives
- Add devices.go: IsMMIO predicate, NetDevice/BlkDevice/RngDevice helpers
- Refactor lifecycle.go + network_helpers.go call sites to use helpers
- shutdown.go: os.Remove fw_cfg tmpfiles after QEMU exits
- Add devices_test.go, microvm_test.go; extend vm_test.go
config/input extensions (mulga-siv-63):
- Add MicrovmConfig{ELBv2Enabled, ImagePath} to DaemonConfig
- ImagePath falls back to /usr/share/spinifex/microvm when empty
- Add commented [microvm] section to spinifex.toml template
- Add DirectBoot bool + NICs []NICConfig to SystemInstanceInput
image build pipeline (mulga-siv-64):
- scripts/build-microvm-image.sh: Alpine chroot build producing vmlinuz + initramfs.cpio.gz
- build/microvm/init.sh: fw_cfg-driven PID 1 init (mounts, modprobe, NIC config, lb-agent-env, ca-cert)
- build/microvm/inittab: BusyBox init with lb-agent respawn
- make build-microvm-image target
- setup.sh: install microvm/* to /usr/share/spinifex/microvm/
- .gitignore: exclude built vmlinuz + initramfs.cpio.gz artifacts
This commit is contained in:
joshsiv-mulga
@@ -17,6 +17,8 @@ go.work.sum
|
||||
# Build outputs
|
||||
bin/
|
||||
dist/
|
||||
build/microvm/vmlinuz
|
||||
build/microvm/initramfs.cpio.gz
|
||||
!spinifex/services/spinifexui/frontend/dist/
|
||||
tmp/
|
||||
*.test
|
||||
|
||||
@@ -82,6 +82,9 @@ endif
|
||||
build-lb-image: ## Build LB Alpine image
|
||||
$(MAKE) build-system-image IMAGE=lb
|
||||
|
||||
build-microvm-image: ## Build microVM kernel + initramfs
|
||||
./scripts/build-microvm-image.sh
|
||||
|
||||
go_run:
|
||||
@echo -e "\n....Running $(GO_PROJECT_NAME)...."
|
||||
$(GOPATH)/bin/$(GO_PROJECT_NAME)
|
||||
@@ -238,7 +241,7 @@ ansible-dev-install:
|
||||
ansible-dev-reset:
|
||||
cd scripts/ansible && ansible-playbook playbooks/dev-reset.yml
|
||||
|
||||
.PHONY: build build-ui build-installer build-lb-agent build-system-image build-lb-image go_build go_run preflight test test-cover test-race diff-coverage bench run \
|
||||
.PHONY: build build-ui build-installer build-lb-agent build-system-image build-lb-image build-microvm-image go_build go_run preflight test test-cover test-race diff-coverage bench run \
|
||||
deploy reinstall clean \
|
||||
install-system install-go install-aws quickinstall \
|
||||
lint fix govulncheck \
|
||||
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
mount -t proc proc /proc
|
||||
mount -t sysfs sys /sys
|
||||
mount -t devtmpfs dev /dev
|
||||
|
||||
modprobe qemu_fw_cfg 2>/dev/null || modprobe fw_cfg_sysfs || {
|
||||
echo "FATAL: qemu_fw_cfg module not available — cannot read fw_cfg blobs" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
. /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/netcfg/raw
|
||||
for n in 0 1 2 3 4 5; do
|
||||
eval mac=\$NIC${n}_MAC
|
||||
[ -z "$mac" ] && continue
|
||||
iface=$(ip -o link | awk -v m="$mac" 'tolower($0) ~ tolower(m) {print $2}' | tr -d :)
|
||||
eval cidr=\$NIC${n}_CIDR
|
||||
eval gw=\$NIC${n}_GW
|
||||
eval default=\$NIC${n}_DEFAULT
|
||||
eval rdst=\$NIC${n}_ROUTE_DST
|
||||
eval rvia=\$NIC${n}_ROUTE_VIA
|
||||
ip link set "$iface" up
|
||||
[ -n "$cidr" ] && ip addr add "$cidr" dev "$iface"
|
||||
[ "$default" = "1" ] && [ -n "$gw" ] && ip route add default via "$gw" dev "$iface"
|
||||
[ -n "$rdst" ] && [ -n "$rvia" ] && ip route add "$rdst" via "$rvia" dev "$iface"
|
||||
done
|
||||
|
||||
cp /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/lb-agent-env/raw /etc/conf.d/lb-agent
|
||||
mkdir -p /etc/ssl/certs
|
||||
cp /sys/firmware/qemu_fw_cfg/by_name/opt/spinifex/ca-cert/raw /etc/ssl/certs/ca-certificates.crt
|
||||
|
||||
exec /sbin/init
|
||||
@@ -0,0 +1,4 @@
|
||||
::sysinit:/etc/rc.local
|
||||
::respawn:/usr/local/bin/lb-agent
|
||||
::ctrlaltdel:/sbin/reboot
|
||||
::shutdown:/bin/umount -a -r
|
||||
@@ -72,6 +72,10 @@ dev_networking = true
|
||||
gpu_passthrough = true
|
||||
{{- end}}
|
||||
|
||||
# [nodes.{{.Node}}.daemon.microvm]
|
||||
# elbv2_enabled = false
|
||||
# image_path = "/usr/share/spinifex/microvm"
|
||||
|
||||
[nodes.{{.Node}}.awsgw]
|
||||
host = "{{.BindIP}}:9999"
|
||||
tlskey = "config/server.key"
|
||||
|
||||
Executable
+144
@@ -0,0 +1,144 @@
|
||||
#!/bin/bash
|
||||
# build-microvm-image.sh — build a minimal Alpine-based microVM kernel + initramfs
|
||||
# for use with QEMU microvm machine type and Spinifex lb-agent.
|
||||
#
|
||||
# Outputs:
|
||||
# $REPO_ROOT/build/microvm/vmlinuz
|
||||
# $REPO_ROOT/build/microvm/initramfs.cpio.gz
|
||||
#
|
||||
# Requirements: apk (Alpine package manager), cpio, gzip, find
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
BUILD_DIR="$REPO_ROOT/build/microvm"
|
||||
INIT_SH="$BUILD_DIR/init.sh"
|
||||
INITTAB="$BUILD_DIR/inittab"
|
||||
|
||||
ALPINE_VERSION="${ALPINE_VERSION:-3.20}"
|
||||
ALPINE_MIRROR="${ALPINE_MIRROR:-https://dl-cdn.alpinelinux.org/alpine}"
|
||||
|
||||
echo "[build-microvm-image] repo root: $REPO_ROOT"
|
||||
echo "[build-microvm-image] build dir: $BUILD_DIR"
|
||||
mkdir -p "$BUILD_DIR"
|
||||
|
||||
# --- Verify tooling ---
|
||||
for tool in apk cpio gzip find; do
|
||||
if ! command -v "$tool" >/dev/null 2>&1; then
|
||||
echo "ERROR: required tool not found: $tool" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# --- Create chroot ---
|
||||
CHROOT_DIR=$(mktemp -d)
|
||||
cleanup() {
|
||||
echo "[build-microvm-image] cleaning up chroot: $CHROOT_DIR"
|
||||
rm -rf "$CHROOT_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "[build-microvm-image] creating Alpine chroot in $CHROOT_DIR"
|
||||
|
||||
# Seed the Alpine package database
|
||||
mkdir -p \
|
||||
"$CHROOT_DIR/etc/apk" \
|
||||
"$CHROOT_DIR/lib/apk/db" \
|
||||
"$CHROOT_DIR/var/cache/apk"
|
||||
|
||||
cat > "$CHROOT_DIR/etc/apk/repositories" <<EOF
|
||||
${ALPINE_MIRROR}/v${ALPINE_VERSION}/main
|
||||
${ALPINE_MIRROR}/v${ALPINE_VERSION}/community
|
||||
EOF
|
||||
|
||||
# --- Install packages ---
|
||||
echo "[build-microvm-image] installing Alpine packages (busybox, linux-virt, haproxy)..."
|
||||
apk add \
|
||||
--root "$CHROOT_DIR" \
|
||||
--arch x86_64 \
|
||||
--no-cache \
|
||||
--allow-untrusted \
|
||||
--initdb \
|
||||
busybox \
|
||||
busybox-initscripts \
|
||||
linux-virt \
|
||||
haproxy \
|
||||
iproute2 \
|
||||
ca-certificates
|
||||
|
||||
# --- Place init script ---
|
||||
echo "[build-microvm-image] installing init.sh as /init..."
|
||||
if [ ! -f "$INIT_SH" ]; then
|
||||
echo "ERROR: $INIT_SH not found — cannot build initramfs" >&2
|
||||
exit 1
|
||||
fi
|
||||
install -m 0755 "$INIT_SH" "$CHROOT_DIR/init"
|
||||
|
||||
# --- Place inittab ---
|
||||
if [ -f "$INITTAB" ]; then
|
||||
echo "[build-microvm-image] installing inittab..."
|
||||
install -m 0644 "$INITTAB" "$CHROOT_DIR/etc/inittab"
|
||||
fi
|
||||
|
||||
# --- Copy lb-agent binary ---
|
||||
LB_AGENT_BIN="$REPO_ROOT/bin/lb-agent"
|
||||
if [ -f "$LB_AGENT_BIN" ]; then
|
||||
echo "[build-microvm-image] copying lb-agent binary..."
|
||||
mkdir -p "$CHROOT_DIR/usr/local/bin"
|
||||
install -m 0755 "$LB_AGENT_BIN" "$CHROOT_DIR/usr/local/bin/lb-agent"
|
||||
else
|
||||
echo "[build-microvm-image] WARNING: $LB_AGENT_BIN not found — lb-agent will be absent from initramfs" >&2
|
||||
fi
|
||||
|
||||
# --- Create required directories for init ---
|
||||
mkdir -p \
|
||||
"$CHROOT_DIR/proc" \
|
||||
"$CHROOT_DIR/sys" \
|
||||
"$CHROOT_DIR/dev" \
|
||||
"$CHROOT_DIR/etc/ssl/certs" \
|
||||
"$CHROOT_DIR/etc/conf.d"
|
||||
|
||||
# --- Locate kernel ---
|
||||
KERNEL_IMG=$(find "$CHROOT_DIR/boot" -name "vmlinuz*" | sort | tail -1)
|
||||
if [ -z "$KERNEL_IMG" ]; then
|
||||
echo "ERROR: no vmlinuz found in $CHROOT_DIR/boot" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[build-microvm-image] kernel image: $KERNEL_IMG"
|
||||
|
||||
# Locate kernel config (may be absent for built-in-only configs)
|
||||
KERNEL_VER=$(basename "$KERNEL_IMG" | sed 's/vmlinuz-//')
|
||||
KERNEL_CONFIG="$CHROOT_DIR/boot/config-${KERNEL_VER}"
|
||||
|
||||
# --- Assert fw_cfg module ---
|
||||
if ! find "$CHROOT_DIR/lib/modules" \
|
||||
-name "qemu_fw_cfg.ko*" \
|
||||
-o -name "fw_cfg_sysfs.ko*" \
|
||||
| grep -q .; then
|
||||
if ! grep -q "CONFIG_FW_CFG_SYSFS=y" "$KERNEL_CONFIG" 2>/dev/null; then
|
||||
echo "ERROR: qemu_fw_cfg module missing from initramfs — fw_cfg reads will fail at boot" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[build-microvm-image] fw_cfg: built-in (CONFIG_FW_CFG_SYSFS=y)"
|
||||
else
|
||||
echo "[build-microvm-image] fw_cfg: module found in lib/modules"
|
||||
fi
|
||||
|
||||
# --- Build initramfs ---
|
||||
INITRAMFS_OUT="$BUILD_DIR/initramfs.cpio.gz"
|
||||
echo "[build-microvm-image] building initramfs: $INITRAMFS_OUT"
|
||||
(
|
||||
cd "$CHROOT_DIR"
|
||||
find . | cpio --quiet -o -H newc | gzip -9 > "$INITRAMFS_OUT"
|
||||
)
|
||||
|
||||
# --- Copy vmlinuz ---
|
||||
VMLINUZ_OUT="$BUILD_DIR/vmlinuz"
|
||||
echo "[build-microvm-image] copying kernel: $VMLINUZ_OUT"
|
||||
cp "$KERNEL_IMG" "$VMLINUZ_OUT"
|
||||
|
||||
# --- Log artifact sizes ---
|
||||
echo ""
|
||||
echo "[build-microvm-image] artifacts:"
|
||||
ls -lh "$VMLINUZ_OUT" "$INITRAMFS_OUT"
|
||||
echo ""
|
||||
echo "[build-microvm-image] done."
|
||||
@@ -346,6 +346,13 @@ install_files() {
|
||||
$SUDO install -m 0755 "$EXTRACT_DIR/setup.sh" /usr/local/share/spinifex/setup.sh
|
||||
info " /usr/local/share/spinifex/setup.sh"
|
||||
fi
|
||||
|
||||
# microVM kernel + initramfs
|
||||
$SUDO install -d /usr/share/spinifex/microvm
|
||||
if [ -d "$EXTRACT_DIR/microvm" ]; then
|
||||
$SUDO cp "$EXTRACT_DIR/microvm/"* /usr/share/spinifex/microvm/
|
||||
info " /usr/share/spinifex/microvm/*"
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Create directories ---
|
||||
|
||||
@@ -138,6 +138,12 @@ type GPUModelOverride struct {
|
||||
XVGAOff bool `json:"XVGAOff" mapstructure:"xvga_off"`
|
||||
}
|
||||
|
||||
// MicrovmConfig holds configuration for microVM (Firecracker/QEMU microvm) support.
|
||||
type MicrovmConfig struct {
|
||||
ELBv2Enabled bool `json:"ELBv2Enabled" mapstructure:"elbv2_enabled"`
|
||||
ImagePath string `json:"ImagePath" mapstructure:"image_path"`
|
||||
}
|
||||
|
||||
// DaemonConfig holds the daemon configuration
|
||||
type DaemonConfig struct {
|
||||
Host string `json:"Host" mapstructure:"host"`
|
||||
@@ -147,6 +153,7 @@ type DaemonConfig struct {
|
||||
MgmtBridge string `json:"MgmtBridge" mapstructure:"mgmt_bridge"` // Linux bridge for system instance control plane (default "br-mgmt")
|
||||
GPUPassthrough bool `json:"GPUPassthrough" mapstructure:"gpu_passthrough"` // Enable VFIO GPU passthrough for g5.* instance types
|
||||
GPUModelOverrides []GPUModelOverride `json:"GPUModelOverrides" mapstructure:"gpu_model_overrides"` // Dev/test GPU mappings not in the production model list
|
||||
Microvm MicrovmConfig `json:"Microvm" mapstructure:"microvm"`
|
||||
}
|
||||
|
||||
// NATSConfig holds the NATS configuration
|
||||
@@ -242,6 +249,10 @@ func LoadConfig(configPath string) (*ClusterConfig, error) {
|
||||
local.Predastore.Host = strings.Replace(local.Predastore.Host, "0.0.0.0", "127.0.0.1", 1)
|
||||
config.Nodes[config.Node] = local
|
||||
}
|
||||
if local.Daemon.Microvm.ImagePath == "" {
|
||||
local.Daemon.Microvm.ImagePath = "/usr/share/spinifex/microvm"
|
||||
config.Nodes[config.Node] = local
|
||||
}
|
||||
}
|
||||
|
||||
return &config, nil
|
||||
|
||||
@@ -49,6 +49,16 @@ type SystemInstanceInput struct {
|
||||
// via the QEMU user-mode dev NIC (dev_networking mode only).
|
||||
// Each entry is a guest port (e.g. 80, 443). The host port is auto-assigned.
|
||||
HostfwdPorts []int
|
||||
|
||||
// DirectBoot skips AMI/volume setup and uses direct kernel boot instead.
|
||||
// When true, ImageID and volume attachment are ignored; the kernel and
|
||||
// initrd are sourced from the configured microvm image path.
|
||||
DirectBoot bool
|
||||
|
||||
// NICs defines the network interfaces for a direct-boot microVM launch.
|
||||
// Index 0 is the primary VPC ENI, index 1 is the management NIC, index 2+ are extra ENIs.
|
||||
// Ignored when DirectBoot is false.
|
||||
NICs []NICConfig
|
||||
}
|
||||
|
||||
// ExtraENIInput describes an additional pre-created ENI to attach to a
|
||||
@@ -60,6 +70,17 @@ type ExtraENIInput struct {
|
||||
SubnetID string
|
||||
}
|
||||
|
||||
// NICConfig describes a single network interface for a microVM direct-boot launch.
|
||||
// Index 0 is the primary VPC ENI, index 1 is the management NIC, index 2+ are extra ENIs.
|
||||
type NICConfig struct {
|
||||
MAC string // e.g. "02:0a:01:23:45:67"
|
||||
CIDR string // e.g. "10.0.1.5/24"
|
||||
Gateway string // e.g. "10.0.1.1"; empty for mgmt NIC
|
||||
IsDefault bool // true for exactly one NIC (primary VPC ENI)
|
||||
RouteDst string // specific host route dst, e.g. "10.20.0.5/32"
|
||||
RouteVia string // next-hop for RouteDst
|
||||
}
|
||||
|
||||
// SystemInstanceOutput contains the result of a successful launch.
|
||||
type SystemInstanceOutput struct {
|
||||
InstanceID string // e.g. "i-xxxxx"
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package vm
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// IsMMIO reports whether machineType is a memory-mapped I/O bus machine (e.g.
|
||||
// microvm). MMIO machines use virtio-*-device transports instead of
|
||||
// virtio-*-pci, because they have no PCI bus.
|
||||
func IsMMIO(machineType string) bool {
|
||||
return strings.HasPrefix(machineType, "microvm")
|
||||
}
|
||||
|
||||
// NetDevice returns the appropriate QEMU virtio-net device string for
|
||||
// machineType, wiring netdev and mac. mac is omitted when empty.
|
||||
func NetDevice(machineType, netdev, mac string) Device {
|
||||
var transport string
|
||||
if IsMMIO(machineType) {
|
||||
transport = "virtio-net-device"
|
||||
} else {
|
||||
transport = "virtio-net-pci"
|
||||
}
|
||||
if mac != "" {
|
||||
return Device{Value: fmt.Sprintf("%s,netdev=%s,mac=%s", transport, netdev, mac)}
|
||||
}
|
||||
return Device{Value: fmt.Sprintf("%s,netdev=%s", transport, netdev)}
|
||||
}
|
||||
|
||||
// BlkDevice returns the appropriate QEMU virtio-blk device string for
|
||||
// machineType. iothread and queues are omitted when empty/zero. bootIdx is
|
||||
// only emitted for PCI machines (MMIO has no bootindex concept).
|
||||
func BlkDevice(machineType, drive, iothread string, queues int, bootIdx int) Device {
|
||||
var b strings.Builder
|
||||
if IsMMIO(machineType) {
|
||||
b.WriteString("virtio-blk-device")
|
||||
} else {
|
||||
b.WriteString("virtio-blk-pci")
|
||||
}
|
||||
fmt.Fprintf(&b, ",drive=%s", drive)
|
||||
if iothread != "" {
|
||||
fmt.Fprintf(&b, ",iothread=%s", iothread)
|
||||
}
|
||||
if queues > 0 {
|
||||
fmt.Fprintf(&b, ",num-queues=%d", queues)
|
||||
}
|
||||
if !IsMMIO(machineType) && bootIdx > 0 {
|
||||
fmt.Fprintf(&b, ",bootindex=%d", bootIdx)
|
||||
}
|
||||
return Device{Value: b.String()}
|
||||
}
|
||||
|
||||
// RngDevice returns the appropriate QEMU virtio-rng device string for
|
||||
// machineType.
|
||||
func RngDevice(machineType string) Device {
|
||||
if IsMMIO(machineType) {
|
||||
return Device{Value: "virtio-rng-device"}
|
||||
}
|
||||
return Device{Value: "virtio-rng-pci"}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package vm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestIsMMIO(t *testing.T) {
|
||||
tests := []struct {
|
||||
machineType string
|
||||
want bool
|
||||
}{
|
||||
{"microvm", true},
|
||||
{"microvm,x-option-roms=off", true},
|
||||
{"q35", false},
|
||||
{"virt", false},
|
||||
{"", false},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.machineType, func(t *testing.T) {
|
||||
assert.Equal(t, tt.want, IsMMIO(tt.machineType))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetDevice_PCI(t *testing.T) {
|
||||
d := NetDevice("q35", "net0", "02:00:00:aa:bb:cc")
|
||||
assert.Equal(t, "virtio-net-pci,netdev=net0,mac=02:00:00:aa:bb:cc", d.Value)
|
||||
}
|
||||
|
||||
func TestNetDevice_PCI_NoMAC(t *testing.T) {
|
||||
d := NetDevice("q35", "net0", "")
|
||||
assert.Equal(t, "virtio-net-pci,netdev=net0", d.Value)
|
||||
}
|
||||
|
||||
func TestNetDevice_MMIO(t *testing.T) {
|
||||
d := NetDevice("microvm", "net0", "02:00:00:aa:bb:cc")
|
||||
assert.Equal(t, "virtio-net-device,netdev=net0,mac=02:00:00:aa:bb:cc", d.Value)
|
||||
}
|
||||
|
||||
func TestNetDevice_MMIO_NoMAC(t *testing.T) {
|
||||
d := NetDevice("microvm,x-option-roms=off", "net0", "")
|
||||
assert.Equal(t, "virtio-net-device,netdev=net0", d.Value)
|
||||
}
|
||||
|
||||
func TestBlkDevice_PCI(t *testing.T) {
|
||||
d := BlkDevice("q35", "os", "ioth-os", 4, 1)
|
||||
assert.Equal(t, "virtio-blk-pci,drive=os,iothread=ioth-os,num-queues=4,bootindex=1", d.Value)
|
||||
}
|
||||
|
||||
func TestBlkDevice_PCI_NoBootIdx(t *testing.T) {
|
||||
d := BlkDevice("q35", "data", "ioth-data", 2, 0)
|
||||
assert.Equal(t, "virtio-blk-pci,drive=data,iothread=ioth-data,num-queues=2", d.Value)
|
||||
}
|
||||
|
||||
func TestBlkDevice_PCI_NoIOThread(t *testing.T) {
|
||||
d := BlkDevice("q35", "data", "", 0, 0)
|
||||
assert.Equal(t, "virtio-blk-pci,drive=data", d.Value)
|
||||
}
|
||||
|
||||
func TestBlkDevice_MMIO(t *testing.T) {
|
||||
d := BlkDevice("microvm", "os", "ioth-os", 4, 1)
|
||||
// MMIO has no bootindex
|
||||
assert.Equal(t, "virtio-blk-device,drive=os,iothread=ioth-os,num-queues=4", d.Value)
|
||||
}
|
||||
|
||||
func TestBlkDevice_MMIO_NoIOThread(t *testing.T) {
|
||||
d := BlkDevice("microvm,x-option-roms=off", "data", "", 0, 0)
|
||||
assert.Equal(t, "virtio-blk-device,drive=data", d.Value)
|
||||
}
|
||||
|
||||
func TestRngDevice_PCI(t *testing.T) {
|
||||
d := RngDevice("q35")
|
||||
assert.Equal(t, "virtio-rng-pci", d.Value)
|
||||
}
|
||||
|
||||
func TestRngDevice_MMIO(t *testing.T) {
|
||||
d := RngDevice("microvm")
|
||||
assert.Equal(t, "virtio-rng-device", d.Value)
|
||||
}
|
||||
|
||||
func TestRngDevice_MMIO_WithOptions(t *testing.T) {
|
||||
d := RngDevice("microvm,x-option-roms=off,pic=off")
|
||||
assert.Equal(t, "virtio-rng-device", d.Value)
|
||||
}
|
||||
+13
-22
@@ -10,6 +10,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
@@ -194,7 +195,7 @@ func (m *Manager) startQEMU(instance *VM) error {
|
||||
instance.Config = buildBaseVMConfig(instance.ID, pidFile, consoleLogPath, serialSocket, spec.Architecture, spec.VCPUs, spec.MemoryMiB)
|
||||
|
||||
instance.EBSRequests.Mu.Lock()
|
||||
drives, iothreads, devices, err := buildDrives(instance.EBSRequests.Requests, spec.VCPUs)
|
||||
drives, iothreads, devices, err := buildDrives(instance.EBSRequests.Requests, spec.VCPUs, instance.Config.MachineType)
|
||||
instance.EBSRequests.Mu.Unlock()
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -214,9 +215,7 @@ func (m *Manager) startQEMU(instance *VM) error {
|
||||
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{
|
||||
Value: fmt.Sprintf("tap,id=net0,ifname=%s,script=no,downscript=no", tapName),
|
||||
})
|
||||
instance.Config.Devices = append(instance.Config.Devices, Device{
|
||||
Value: fmt.Sprintf("virtio-net-pci,netdev=net0,mac=%s", instance.ENIMac),
|
||||
})
|
||||
instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, "net0", instance.ENIMac))
|
||||
slog.Info("VPC networking configured", "tap", tapName, "eni", instance.ENIId, "mac", instance.ENIMac)
|
||||
|
||||
if err := m.setupExtraENINICs(instance); err != nil {
|
||||
@@ -245,9 +244,7 @@ func (m *Manager) startQEMU(instance *VM) error {
|
||||
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{
|
||||
Value: fmt.Sprintf("user,id=net0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort),
|
||||
})
|
||||
instance.Config.Devices = append(instance.Config.Devices, Device{
|
||||
Value: "virtio-net-pci,netdev=net0",
|
||||
})
|
||||
instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, "net0", ""))
|
||||
}
|
||||
|
||||
if instance.MgmtMAC != "" {
|
||||
@@ -255,13 +252,11 @@ func (m *Manager) startQEMU(instance *VM) error {
|
||||
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{
|
||||
Value: fmt.Sprintf("tap,id=mgmt0,ifname=%s,script=no,downscript=no", mgmtTap),
|
||||
})
|
||||
instance.Config.Devices = append(instance.Config.Devices, Device{
|
||||
Value: fmt.Sprintf("virtio-net-pci,netdev=mgmt0,mac=%s", instance.MgmtMAC),
|
||||
})
|
||||
instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, "mgmt0", instance.MgmtMAC))
|
||||
slog.Info("Management NIC configured", "tap", mgmtTap, "mac", instance.MgmtMAC, "ip", instance.MgmtIP, "instanceId", instance.ID)
|
||||
}
|
||||
|
||||
instance.Config.Devices = append(instance.Config.Devices, Device{Value: "virtio-rng-pci"})
|
||||
instance.Config.Devices = append(instance.Config.Devices, RngDevice(instance.Config.MachineType))
|
||||
|
||||
for i, addr := range instance.GPUPCIAddresses {
|
||||
xvga := "off"
|
||||
@@ -416,7 +411,8 @@ func (m *Manager) appendDevHostfwdNIC(instance *VM) {
|
||||
if bindIP == "" || bindIP == "0.0.0.0" {
|
||||
bindIP = "127.0.0.1"
|
||||
}
|
||||
netdevVal := fmt.Sprintf("user,id=dev0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort)
|
||||
var nb strings.Builder
|
||||
fmt.Fprintf(&nb, "user,id=dev0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort)
|
||||
|
||||
if instance.ExtraHostfwd != nil {
|
||||
for guestPort := range instance.ExtraHostfwd {
|
||||
@@ -435,17 +431,15 @@ func (m *Manager) appendDevHostfwdNIC(instance *VM) {
|
||||
slog.Warn("DEV_NETWORKING: failed to convert extra hostfwd port", "hostPort", hostPort, "err", convErr)
|
||||
continue
|
||||
}
|
||||
netdevVal += fmt.Sprintf(",hostfwd=tcp:%s:%s-:%d", bindIP, hostPort, guestPort)
|
||||
fmt.Fprintf(&nb, ",hostfwd=tcp:%s:%s-:%d", bindIP, hostPort, guestPort)
|
||||
instance.ExtraHostfwd[guestPort] = hostPortInt
|
||||
slog.Info("DEV_NETWORKING: extra hostfwd", "guestPort", guestPort, "hostPort", hostPort, "instanceId", instance.ID)
|
||||
}
|
||||
}
|
||||
|
||||
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{Value: netdevVal})
|
||||
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{Value: nb.String()})
|
||||
devMac := GenerateDevMAC(instance.ID)
|
||||
instance.Config.Devices = append(instance.Config.Devices, Device{
|
||||
Value: fmt.Sprintf("virtio-net-pci,netdev=dev0,mac=%s", devMac),
|
||||
})
|
||||
instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, "dev0", devMac))
|
||||
slog.Info("DEV_NETWORKING: added dev NIC with SSH hostfwd",
|
||||
"bindIP", bindIP, "port", sshDebugPort, "mac", devMac, "instanceId", instance.ID)
|
||||
}
|
||||
@@ -586,7 +580,7 @@ func buildBaseVMConfig(instanceID, pidFile, consoleLogPath, serialSocket, archit
|
||||
// buildDrives converts EBS volume requests into QEMU drive, iothread, and
|
||||
// device configurations. Returns an error if any non-EFI volume is missing
|
||||
// its NBDURI.
|
||||
func buildDrives(requests []types.EBSRequest, cpuCount int) ([]Drive, []IOThread, []Device, error) {
|
||||
func buildDrives(requests []types.EBSRequest, cpuCount int, machineType string) ([]Drive, []IOThread, []Device, error) {
|
||||
var drives []Drive
|
||||
var iothreads []IOThread
|
||||
var devices []Device
|
||||
@@ -611,10 +605,7 @@ func buildDrives(requests []types.EBSRequest, cpuCount int) ([]Drive, []IOThread
|
||||
|
||||
iothreadID := "ioth-os"
|
||||
iothreads = append(iothreads, IOThread{ID: iothreadID})
|
||||
devices = append(devices, Device{
|
||||
Value: fmt.Sprintf("virtio-blk-pci,drive=%s,iothread=%s,num-queues=%d,bootindex=1",
|
||||
drive.ID, iothreadID, cpuCount),
|
||||
})
|
||||
devices = append(devices, BlkDevice(machineType, drive.ID, iothreadID, cpuCount, 1))
|
||||
}
|
||||
|
||||
if v.CloudInit {
|
||||
|
||||
@@ -42,6 +42,7 @@ func TestBuildDrives(t *testing.T) {
|
||||
name string
|
||||
requests []types.EBSRequest
|
||||
cpuCount int
|
||||
machineType string
|
||||
wantDrives []Drive
|
||||
wantIOThreads []IOThread
|
||||
wantDevices []Device
|
||||
@@ -112,7 +113,11 @@ func TestBuildDrives(t *testing.T) {
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
drives, iothreads, devices, err := buildDrives(tt.requests, tt.cpuCount)
|
||||
machineType := tt.machineType
|
||||
if machineType == "" {
|
||||
machineType = "q35"
|
||||
}
|
||||
drives, iothreads, devices, err := buildDrives(tt.requests, tt.cpuCount, machineType)
|
||||
|
||||
if tt.wantErr != "" {
|
||||
require.Error(t, err)
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package vm
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestMicroVMDirectBoot asserts that a microvm Config with direct-boot fields
|
||||
// set produces the expected QEMU args and does not include PCI-specific flags
|
||||
// such as -bios or pcie-root-port.
|
||||
func TestMicroVMDirectBoot(t *testing.T) {
|
||||
cfg := Config{
|
||||
CPUCount: 2,
|
||||
Memory: 512,
|
||||
Architecture: "x86_64",
|
||||
MachineType: "microvm,x-option-roms=off,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on",
|
||||
KernelImage: "/path/vmlinuz",
|
||||
Initrd: "/path/initramfs.cpio.gz",
|
||||
KernelCmdline: "console=ttyS0 panic=1",
|
||||
FwCfg: []FwCfgEntry{
|
||||
{Name: "opt/spinifex/netcfg", File: "/tmp/nc.tmp"},
|
||||
},
|
||||
}
|
||||
|
||||
cmd, err := cfg.Execute()
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, cmd)
|
||||
|
||||
args := cmd.Args[1:]
|
||||
|
||||
// Machine type must be set
|
||||
assert.Equal(t, "microvm,x-option-roms=off,pic=off,pit=off,rtc=on,acpi=on,isa-serial=on", argValue(args, "-M"))
|
||||
|
||||
// Direct-boot flags must be present
|
||||
assert.Equal(t, "/path/vmlinuz", argValue(args, "-kernel"))
|
||||
assert.Equal(t, "/path/initramfs.cpio.gz", argValue(args, "-initrd"))
|
||||
assert.Equal(t, "console=ttyS0 panic=1", argValue(args, "-append"))
|
||||
assert.Equal(t, "name=opt/spinifex/netcfg,file=/tmp/nc.tmp", argValue(args, "-fw_cfg"))
|
||||
|
||||
// PCI-specific flags must NOT appear
|
||||
assert.False(t, argExists(args, "-bios"), "-bios must not appear for microvm direct-boot")
|
||||
for _, a := range args {
|
||||
assert.NotContains(t, a, "pcie-root-port", "pcie-root-port must not appear for microvm")
|
||||
}
|
||||
}
|
||||
@@ -86,9 +86,7 @@ func (m *Manager) setupExtraENINICs(instance *VM) error {
|
||||
instance.Config.NetDevs = append(instance.Config.NetDevs, NetDev{
|
||||
Value: fmt.Sprintf("tap,id=%s,ifname=%s,script=no,downscript=no", netID, extraTapName),
|
||||
})
|
||||
instance.Config.Devices = append(instance.Config.Devices, Device{
|
||||
Value: fmt.Sprintf("virtio-net-pci,netdev=%s,mac=%s", netID, extra.ENIMac),
|
||||
})
|
||||
instance.Config.Devices = append(instance.Config.Devices, NetDevice(instance.Config.MachineType, netID, extra.ENIMac))
|
||||
slog.Info("Extra VPC NIC configured",
|
||||
"tap", extraTapName, "eni", extra.ENIID, "mac", extra.ENIMac, "subnet", extra.SubnetID)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@ package vm
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -333,6 +334,12 @@ func (m *Manager) shutdownAndUnmount(instance *VM) {
|
||||
slog.Error("Volume unmount failed", "id", instance.ID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, fw := range instance.Config.FwCfg {
|
||||
if err := os.Remove(fw.File); err != nil && !os.IsNotExist(err) {
|
||||
slog.Warn("Failed to remove fw_cfg temp file", "file", fw.File, "id", instance.ID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupTapDevices removes the primary VPC tap, every extra ENI tap, and
|
||||
|
||||
+29
-2
@@ -195,6 +195,17 @@ type Config struct {
|
||||
// UseUEFI requests OVMF firmware instead of the default SeaBIOS for x86_64 VMs.
|
||||
// Falls back silently to SeaBIOS if OVMF is not installed on the host.
|
||||
UseUEFI bool `json:"use_uefi,omitempty"`
|
||||
|
||||
KernelImage string `json:"kernel_image,omitempty"` // path to vmlinuz; emits -kernel when set
|
||||
Initrd string `json:"initrd,omitempty"` // path to initramfs; emits -initrd when set
|
||||
KernelCmdline string `json:"kernel_cmdline,omitempty"` // emits -append when set
|
||||
FwCfg []FwCfgEntry `json:"fw_cfg,omitempty"` // each emits -fw_cfg name=<name>,file=<path>
|
||||
}
|
||||
|
||||
// FwCfgEntry describes a single QEMU firmware configuration file entry.
|
||||
type FwCfgEntry struct {
|
||||
Name string `json:"name"`
|
||||
File string `json:"file"`
|
||||
}
|
||||
|
||||
func (cfg *Config) Execute() (*exec.Cmd, error) {
|
||||
@@ -252,8 +263,8 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
|
||||
args = append(args, "-object", fmt.Sprintf("iothread,id=%s", iot.ID))
|
||||
}
|
||||
|
||||
if len(cfg.Drives) == 0 {
|
||||
return nil, fmt.Errorf("at least one drive is required")
|
||||
if len(cfg.Drives) == 0 && cfg.KernelImage == "" {
|
||||
return nil, fmt.Errorf("at least one drive or a kernel image is required")
|
||||
}
|
||||
|
||||
for _, drive := range cfg.Drives {
|
||||
@@ -296,6 +307,22 @@ func (cfg *Config) Execute() (*exec.Cmd, error) {
|
||||
args = append(args, "-netdev", netdev.Value)
|
||||
}
|
||||
|
||||
if cfg.KernelImage != "" {
|
||||
args = append(args, "-kernel", cfg.KernelImage)
|
||||
}
|
||||
|
||||
if cfg.Initrd != "" {
|
||||
args = append(args, "-initrd", cfg.Initrd)
|
||||
}
|
||||
|
||||
if cfg.KernelCmdline != "" {
|
||||
args = append(args, "-append", cfg.KernelCmdline)
|
||||
}
|
||||
|
||||
for _, fw := range cfg.FwCfg {
|
||||
args = append(args, "-fw_cfg", fmt.Sprintf("name=%s,file=%s", fw.Name, fw.File))
|
||||
}
|
||||
|
||||
var qemuArchitecture string
|
||||
|
||||
switch cfg.Architecture {
|
||||
|
||||
+34
-2
@@ -34,9 +34,9 @@ func TestExecute(t *testing.T) {
|
||||
|
||||
cmd, err = cfg.Execute()
|
||||
|
||||
// Expect error, at least one drive required
|
||||
// Expect error, at least one drive or kernel image required
|
||||
assert.Error(t, err)
|
||||
assert.ErrorContains(t, err, "at least one drive is required")
|
||||
assert.ErrorContains(t, err, "at least one drive or a kernel image is required")
|
||||
assert.Nil(t, cmd)
|
||||
|
||||
cfg.Drives = []Drive{
|
||||
@@ -231,6 +231,38 @@ func argExists(args []string, flag string) bool {
|
||||
return slices.Contains(args, flag)
|
||||
}
|
||||
|
||||
func TestExecute_DriveAndKernelInvariant(t *testing.T) {
|
||||
base := Config{
|
||||
CPUCount: 2,
|
||||
Memory: 1024,
|
||||
Architecture: "x86_64",
|
||||
}
|
||||
|
||||
t.Run("drives-only is OK", func(t *testing.T) {
|
||||
cfg := base
|
||||
cfg.Drives = []Drive{{File: "disk.img", Format: "raw"}}
|
||||
cmd, err := cfg.Execute()
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, cmd)
|
||||
})
|
||||
|
||||
t.Run("kernel-only is OK", func(t *testing.T) {
|
||||
cfg := base
|
||||
cfg.KernelImage = "/boot/vmlinuz"
|
||||
cmd, err := cfg.Execute()
|
||||
assert.NoError(t, err)
|
||||
assert.NotNil(t, cmd)
|
||||
})
|
||||
|
||||
t.Run("both-empty returns error", func(t *testing.T) {
|
||||
cfg := base
|
||||
cmd, err := cfg.Execute()
|
||||
assert.Error(t, err)
|
||||
assert.ErrorContains(t, err, "at least one drive or a kernel image")
|
||||
assert.Nil(t, cmd)
|
||||
})
|
||||
}
|
||||
|
||||
func TestResetNodeLocalState(t *testing.T) {
|
||||
v := &VM{
|
||||
ID: "i-abc123",
|
||||
|
||||
Reference in New Issue
Block a user