feat(elbv2): wire microvm.elbv2_enabled flag into ELBv2 service

When ELBv2Enabled=true:
- Drop lbVMUserData cloud-config; emit KEY=value LBAgentEnv blob instead
- Set SystemInstanceInput.DirectBoot=true; omit ImageID
- Populate NICs []NICConfig (index 0=primary ENI IsDefault, 1=mgmt, 2+=extra ENIs)
- Validate exactly one IsDefault NIC at service layer

When ELBv2Enabled=false: existing PC-machine path unchanged.

Adds buildLBAgentEnv, buildMicrovmNICs, subnetCIDRForIP, subnetGatewayIP helpers.
Tests in arn_test.go cover both flag-on and flag-off paths.

Closes mulga-siv-66
This commit is contained in:
joshsiv-mulga
2026-05-17 00:59:55 +10:00
parent 228e428ec7
commit 92a67115c3
2 changed files with 425 additions and 24 deletions
+259
View File
@@ -4,6 +4,12 @@ import (
"strings"
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/elbv2"
"github.com/mulgadc/spinifex/spinifex/config"
handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc"
"github.com/mulgadc/spinifex/spinifex/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -125,6 +131,259 @@ func TestLbVMUserData_Structural(t *testing.T) {
assert.NotContains(t, ud, "bootcmd:")
}
// TestBuildLBAgentEnv verifies that buildLBAgentEnv produces a KEY=value blob
// containing all five env vars with the correct values.
func TestBuildLBAgentEnv(t *testing.T) {
svc := &ELBv2ServiceImpl{
GatewayURL: "https://10.0.0.1:9999",
SystemAccessKey: "AKIAIOSFODNN7EXAMPLE",
SystemSecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
region: "ap-southeast-2",
}
env := svc.buildLBAgentEnv("lb-abc123")
lines := strings.Split(strings.TrimRight(env, "\n"), "\n")
kvs := make(map[string]string, len(lines))
for _, l := range lines {
k, v, ok := strings.Cut(l, "=")
require.True(t, ok, "env line missing '=': %q", l)
kvs[k] = v
}
assert.Equal(t, "lb-abc123", kvs["LB_LB_ID"])
assert.Equal(t, "https://10.0.0.1:9999", kvs["LB_GATEWAY_URL"])
assert.Equal(t, "AKIAIOSFODNN7EXAMPLE", kvs["LB_ACCESS_KEY"])
assert.Equal(t, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", kvs["LB_SECRET_KEY"])
assert.Equal(t, "ap-southeast-2", kvs["LB_REGION"])
}
// TestSubnetCIDRForIP and TestSubnetGatewayIP cover the CIDR helpers.
func TestSubnetCIDRForIP(t *testing.T) {
assert.Equal(t, "10.0.1.5/24", subnetCIDRForIP("10.0.1.5", "10.0.1.0/24"))
assert.Equal(t, "172.16.2.10/20", subnetCIDRForIP("172.16.2.10", "172.16.0.0/20"))
assert.Equal(t, "", subnetCIDRForIP("10.0.1.5", "bad-cidr"))
}
func TestSubnetGatewayIP(t *testing.T) {
assert.Equal(t, "10.0.1.1", subnetGatewayIP("10.0.1.0/24"))
assert.Equal(t, "172.16.0.1", subnetGatewayIP("172.16.0.0/20"))
assert.Equal(t, "", subnetGatewayIP("not-a-cidr"))
}
// setupMicrovmTestService creates an ELBv2 service with ELBv2Enabled=true wired
// to a real VPC service. The VPC has one subnet (10.0.1.0/24) pre-created.
func setupMicrovmTestService(t *testing.T) (*ELBv2ServiceImpl, *handlers_ec2_vpc.VPCServiceImpl, string) {
t.Helper()
_, nc, _ := testutil.StartTestJetStream(t)
testutil.StubVpcdSGResponder(t, nc)
vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(nil, nc)
require.NoError(t, err)
cfg := &config.Config{
Daemon: config.DaemonConfig{
DevNetworking: true,
Microvm: config.MicrovmConfig{ELBv2Enabled: true},
},
}
elbv2Svc, err := NewELBv2ServiceImplWithNATS(cfg, nc)
require.NoError(t, err)
elbv2Svc.VPCService = vpcSvc
elbv2Svc.GatewayURL = "https://10.20.0.5:9999"
elbv2Svc.SystemAccessKey = "AKID"
elbv2Svc.SystemSecretKey = "SECRET"
elbv2Svc.region = "us-east-1"
elbv2Svc.CACert = "-----BEGIN CERTIFICATE-----\nMOCK\n-----END CERTIFICATE-----\n"
vpcOut, err := vpcSvc.CreateVpc(&ec2.CreateVpcInput{
CidrBlock: aws.String("10.0.0.0/16"),
}, testAccountID)
require.NoError(t, err)
vpcID := *vpcOut.Vpc.VpcId
subnetOut, err := vpcSvc.CreateSubnet(&ec2.CreateSubnetInput{
VpcId: aws.String(vpcID),
CidrBlock: aws.String("10.0.1.0/24"),
AvailabilityZone: aws.String("us-east-1a"),
}, testAccountID)
require.NoError(t, err)
subnetID := *subnetOut.Subnet.SubnetId
return elbv2Svc, vpcSvc, subnetID
}
// TestCreateLoadBalancer_Microvm_DirectBoot asserts the microvm launch path
// sets DirectBoot=true, leaves ImageID empty, and delivers lb-agent env vars.
func TestCreateLoadBalancer_Microvm_DirectBoot(t *testing.T) {
svc, _, subnetID := setupMicrovmTestService(t)
mock := &mockSystemInstanceLauncher{
launchResult: &SystemInstanceOutput{
InstanceID: "i-microvm-test",
PrivateIP: "10.0.1.5",
},
}
svc.InstanceLauncher = mock
_, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
Name: aws.String("mv-alb"),
Subnets: []*string{aws.String(subnetID)},
}, testAccountID)
require.NoError(t, err)
require.Len(t, mock.launchCalls, 1)
inp := mock.launchCalls[0]
assert.True(t, inp.DirectBoot, "DirectBoot must be true in microvm mode")
assert.Equal(t, "", inp.ImageID, "ImageID must be empty in microvm mode")
assert.Equal(t, "-----BEGIN CERTIFICATE-----\nMOCK\n-----END CERTIFICATE-----\n", inp.CACert)
}
// TestCreateLoadBalancer_Microvm_LBAgentEnv asserts all five lb-agent env vars
// appear in LBAgentEnv with correct values.
func TestCreateLoadBalancer_Microvm_LBAgentEnv(t *testing.T) {
svc, _, subnetID := setupMicrovmTestService(t)
mock := &mockSystemInstanceLauncher{
launchResult: &SystemInstanceOutput{InstanceID: "i-env-test", PrivateIP: "10.0.1.6"},
}
svc.InstanceLauncher = mock
_, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
Name: aws.String("env-alb"),
Subnets: []*string{aws.String(subnetID)},
}, testAccountID)
require.NoError(t, err)
require.Len(t, mock.launchCalls, 1)
envBlob := mock.launchCalls[0].LBAgentEnv
kvs := parseEnvBlob(t, envBlob)
assert.NotEmpty(t, kvs["LB_LB_ID"])
assert.Equal(t, "https://10.20.0.5:9999", kvs["LB_GATEWAY_URL"])
assert.Equal(t, "AKID", kvs["LB_ACCESS_KEY"])
assert.Equal(t, "SECRET", kvs["LB_SECRET_KEY"])
assert.Equal(t, "us-east-1", kvs["LB_REGION"])
}
// TestCreateLoadBalancer_Microvm_NICs asserts NIC[0].IsDefault=true and
// NIC[1].IsDefault=false, and that CIDR/Gateway are derived from the subnet.
func TestCreateLoadBalancer_Microvm_NICs(t *testing.T) {
svc, _, subnetID := setupMicrovmTestService(t)
mock := &mockSystemInstanceLauncher{
launchResult: &SystemInstanceOutput{InstanceID: "i-nic-test", PrivateIP: "10.0.1.7"},
}
svc.InstanceLauncher = mock
_, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
Name: aws.String("nic-alb"),
Subnets: []*string{aws.String(subnetID)},
}, testAccountID)
require.NoError(t, err)
require.Len(t, mock.launchCalls, 1)
nics := mock.launchCalls[0].NICs
require.GreaterOrEqual(t, len(nics), 2, "must have at least primary ENI NIC and mgmt NIC")
// NIC[0]: primary VPC ENI — must be the default route owner.
assert.True(t, nics[0].IsDefault, "NIC[0] must have IsDefault=true")
assert.NotEmpty(t, nics[0].MAC, "NIC[0] MAC must be populated from ENI")
assert.Contains(t, nics[0].CIDR, "/24", "NIC[0] CIDR must include prefix from subnet")
assert.Equal(t, "10.0.1.1", nics[0].Gateway, "NIC[0] gateway must be subnet .1 address")
// NIC[1]: management NIC — daemon allocates IP/MAC.
assert.False(t, nics[1].IsDefault, "NIC[1] must have IsDefault=false")
}
// TestCreateLoadBalancer_Microvm_MissingCredentials_SetsStateFailed verifies
// that missing system credentials on the microvm path result in StateFailed.
func TestCreateLoadBalancer_Microvm_MissingCredentials_SetsStateFailed(t *testing.T) {
svc, _, subnetID := setupMicrovmTestService(t)
// Clear credentials to trigger the failure path.
svc.GatewayURL = ""
mock := &mockSystemInstanceLauncher{
launchResult: &SystemInstanceOutput{InstanceID: "i-should-not-launch"},
}
svc.InstanceLauncher = mock
out, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
Name: aws.String("mv-nocreds-alb"),
Subnets: []*string{aws.String(subnetID)},
}, testAccountID)
require.NoError(t, err)
assert.Equal(t, StateFailed, *out.LoadBalancers[0].State.Code)
assert.Empty(t, mock.launchCalls, "launcher must not be invoked when credentials are missing")
}
// TestCreateLoadBalancer_PCMachine_Unchanged verifies that when ELBv2Enabled=false
// the existing PC-machine cloud-config path is used and DirectBoot is not set.
func TestCreateLoadBalancer_PCMachine_Unchanged(t *testing.T) {
_, nc, _ := testutil.StartTestJetStream(t)
testutil.StubVpcdSGResponder(t, nc)
vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(nil, nc)
require.NoError(t, err)
cfg := &config.Config{
Daemon: config.DaemonConfig{
DevNetworking: true,
Microvm: config.MicrovmConfig{ELBv2Enabled: false},
},
}
svc, err := NewELBv2ServiceImplWithNATS(cfg, nc)
require.NoError(t, err)
svc.VPCService = vpcSvc
svc.GatewayURL = "https://10.0.0.1:9999"
svc.SystemAccessKey = "AKID"
svc.SystemSecretKey = "SECRET"
svc.SetSystemAMIFunc(func() (string, error) { return "ami-lb-test", nil })
vpcOut, err := vpcSvc.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String("10.0.0.0/16")}, testAccountID)
require.NoError(t, err)
subnetOut, err := vpcSvc.CreateSubnet(&ec2.CreateSubnetInput{
VpcId: vpcOut.Vpc.VpcId,
CidrBlock: aws.String("10.0.2.0/24"),
AvailabilityZone: aws.String("us-east-1a"),
}, testAccountID)
require.NoError(t, err)
mock := &mockSystemInstanceLauncher{
launchResult: &SystemInstanceOutput{InstanceID: "i-pc-test", PrivateIP: "10.0.2.5"},
}
svc.InstanceLauncher = mock
_, err = svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{
Name: aws.String("pc-alb"),
Subnets: []*string{subnetOut.Subnet.SubnetId},
}, testAccountID)
require.NoError(t, err)
require.Len(t, mock.launchCalls, 1)
inp := mock.launchCalls[0]
assert.False(t, inp.DirectBoot, "DirectBoot must be false on PC-machine path")
assert.Equal(t, "ami-lb-test", inp.ImageID, "ImageID must be set on PC-machine path")
assert.NotEmpty(t, inp.UserData, "UserData (cloud-config) must be set on PC-machine path")
assert.True(t, strings.HasPrefix(inp.UserData, "#cloud-config\n"),
"UserData must be cloud-config on PC-machine path")
}
// parseEnvBlob parses a KEY=value newline-separated blob into a map.
func parseEnvBlob(t *testing.T, blob string) map[string]string {
t.Helper()
kvs := make(map[string]string)
for line := range strings.SplitSeq(strings.TrimRight(blob, "\n"), "\n") {
if line == "" {
continue
}
k, v, ok := strings.Cut(line, "=")
require.True(t, ok, "env blob line missing '=': %q", line)
kvs[k] = v
}
return kvs
}
// TestLbVMUserData_MissingCredentials covers the three required-field error
// paths (gateway URL, access key, secret key) in one table-driven test.
func TestLbVMUserData_MissingCredentials(t *testing.T) {
+166 -24
View File
@@ -7,6 +7,7 @@ import (
"fmt"
"log/slog"
"maps"
"net"
"regexp"
"slices"
"strings"
@@ -141,6 +142,7 @@ type ELBv2ServiceImpl struct {
MgmtRouteTarget string // AWSGW bind IP to route via mgmt NIC
MgmtBridgeIP string // br-mgmt IP, populated whenever br-mgmt exists (single + multi node) for the internal-scheme fallback route
AdvertiseIP string // AdvertiseIP / WAN gateway, populated whenever set; used as the internal-scheme fallback route target on single-node
CACert string // PEM-encoded CA certificate delivered to microvm guests via fw_cfg (direct-boot path only)
nodeID string
region string
systemAMI string // AMI ID for system VMs (ALB VMs); resolved lazily via systemAMIFunc
@@ -255,6 +257,109 @@ func (s *ELBv2ServiceImpl) getSystemInstanceType() string {
return s.systemInstanceType
}
// buildLBAgentEnv returns the KEY=value blob written to /etc/conf.d/lb-agent
// via fw_cfg on the direct-boot path.
func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID string) string {
return fmt.Sprintf("LB_LB_ID=%s\nLB_GATEWAY_URL=%s\nLB_ACCESS_KEY=%s\nLB_SECRET_KEY=%s\nLB_REGION=%s\n",
lbID, s.GatewayURL, s.SystemAccessKey, s.SystemSecretKey, s.region)
}
// subnetCIDRForIP computes "ip/prefixlen" given a host IP and the subnet's
// CIDR block (e.g. "10.0.1.0/24" → "10.0.1.5/24"). Returns empty string on
// parse failure so the caller can log and continue without panicking.
func subnetCIDRForIP(hostIP, subnetCIDR string) string {
_, ipNet, err := net.ParseCIDR(subnetCIDR)
if err != nil {
return ""
}
ones, _ := ipNet.Mask.Size()
return fmt.Sprintf("%s/%d", hostIP, ones)
}
// subnetGatewayIP derives the gateway IP (network address +1) from a subnet
// CIDR block. Returns empty string on parse failure.
func subnetGatewayIP(subnetCIDR string) string {
_, ipNet, err := net.ParseCIDR(subnetCIDR)
if err != nil {
return ""
}
gw := ipNet.IP.To4()
if gw == nil {
return ""
}
gwCopy := make(net.IP, len(gw))
copy(gwCopy, gw)
gwCopy[3]++
return gwCopy.String()
}
// buildMicrovmNICs constructs the NIC slice for a direct-boot microvm launch.
//
// NIC[0] is the primary VPC ENI (IsDefault=true): MAC/IP from the ENI record,
// CIDR from ip+subnet prefix, gateway = subnet network address +1.
//
// NIC[1] is the management NIC (IsDefault=false): MAC and CIDR are left empty
// here — the daemon injects them after allocating the per-instance mgmt IP.
// RouteDst/RouteVia encode the host route the lb-agent needs to reach AWSGW
// via the management interface (same logic as resolveMgmtRoute).
//
// NIC[2+] are extra VPC ENIs for multi-subnet ALBs.
func (s *ELBv2ServiceImpl) buildMicrovmNICs(primaryIP, primaryMAC, primarySubnetID, _, scheme string, extraENIs []ExtraENIInput, accountID string) []NICConfig {
// Resolve primary subnet CIDR for NIC[0] prefix and gateway.
primaryCIDR := ""
primaryGateway := ""
if s.VPCService != nil && primarySubnetID != "" {
if subnet, err := s.VPCService.GetSubnet(accountID, primarySubnetID); err == nil && subnet != nil {
primaryCIDR = subnetCIDRForIP(primaryIP, subnet.CidrBlock)
primaryGateway = subnetGatewayIP(subnet.CidrBlock)
} else {
slog.Warn("buildMicrovmNICs: could not look up primary subnet", "subnetId", primarySubnetID, "err", err)
}
}
nics := []NICConfig{
// NIC[0]: primary VPC ENI — default route owner.
{
MAC: primaryMAC,
CIDR: primaryCIDR,
Gateway: primaryGateway,
IsDefault: true,
},
}
// NIC[1]: management NIC — daemon allocates IP/MAC; we provide routing only.
// RouteDst and RouteVia reproduce the bootcmd host route from the PC-machine
// cloud-config path (resolveMgmtRoute) so lb-agent can reach AWSGW.
mgmtRouteDst, mgmtRouteVia := s.resolveMgmtRoute(scheme)
nics = append(nics, NICConfig{
IsDefault: false,
RouteDst: mgmtRouteDst,
RouteVia: mgmtRouteVia,
})
// NIC[2+]: extra ENIs for multi-subnet ALBs.
for _, extra := range extraENIs {
extraCIDR := ""
extraGateway := ""
if s.VPCService != nil && extra.SubnetID != "" {
if subnet, err := s.VPCService.GetSubnet(accountID, extra.SubnetID); err == nil && subnet != nil {
extraCIDR = subnetCIDRForIP(extra.ENIIP, subnet.CidrBlock)
extraGateway = subnetGatewayIP(subnet.CidrBlock)
} else {
slog.Warn("buildMicrovmNICs: could not look up extra subnet", "subnetId", extra.SubnetID, "err", err)
}
}
nics = append(nics, NICConfig{
MAC: extra.ENIMac,
CIDR: extraCIDR,
Gateway: extraGateway,
IsDefault: false,
})
}
return nics
}
// resolveENIBindAddr looks up the private IP of the first ENI belonging to the ALB.
// Returns empty string if VPC service is unavailable or no ENIs exist.
func (s *ELBv2ServiceImpl) resolveENIBindAddr(lb *LoadBalancerRecord) string {
@@ -671,15 +776,24 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
var hostPorts map[int]int
var launchFailed bool
if s.InstanceLauncher != nil && len(eniIDs) > 0 && len(subnets) > 0 {
systemAMI, amiErr := s.getSystemAMI()
if amiErr != nil {
slog.Error("CreateLoadBalancer: cannot resolve LB system AMI", "lbId", lbID, "err", amiErr)
return nil, errors.New(awserrors.ErrorServerInternal)
}
if systemAMI == "" {
slog.Error("CreateLoadBalancer: LB system AMI not resolved", "lbId", lbID)
return nil, errors.New(awserrors.ErrorServerInternal)
microvmEnabled := s.config != nil && s.config.Daemon.Microvm.ELBv2Enabled
// For the PC-machine path, AMI must be resolved before continuing.
// The microvm (direct-boot) path skips AMI entirely.
var systemAMI string
if !microvmEnabled {
var amiErr error
systemAMI, amiErr = s.getSystemAMI()
if amiErr != nil {
slog.Error("CreateLoadBalancer: cannot resolve LB system AMI", "lbId", lbID, "err", amiErr)
return nil, errors.New(awserrors.ErrorServerInternal)
}
if systemAMI == "" {
slog.Error("CreateLoadBalancer: LB system AMI not resolved", "lbId", lbID)
return nil, errors.New(awserrors.ErrorServerInternal)
}
}
// Resolve MAC/IP for every ENI in one describe call.
eniDetails := make(map[string]*ec2.NetworkInterface, len(eniIDs))
if s.VPCService != nil {
@@ -721,23 +835,51 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp
})
}
userData, udErr := s.lbVMUserData(lbID, scheme)
if udErr != nil {
slog.Error("CreateLoadBalancer: system credentials not configured — cannot launch ALB VM", "lbId", lbID, "err", udErr)
launchFailed = true
} else {
launchInput := &SystemInstanceInput{
InstanceType: s.getSystemInstanceType(),
ImageID: systemAMI,
SubnetID: subnets[0],
UserData: userData,
ENIID: eniIDs[0],
ENIMac: primaryMAC,
ENIIP: primaryIP,
ExtraENIs: extraENIInputs,
Scheme: scheme,
AccountID: accountID,
var launchInput *SystemInstanceInput
if microvmEnabled {
// Direct-boot microvm path: skip AMI/volumes, deliver config via fw_cfg.
if s.GatewayURL == "" || s.SystemAccessKey == "" || s.SystemSecretKey == "" {
slog.Error("CreateLoadBalancer: system credentials not configured for microvm path — cannot launch ALB VM", "lbId", lbID)
launchFailed = true
} else {
nics := s.buildMicrovmNICs(primaryIP, primaryMAC, subnets[0], eniIDs[0], scheme, extraENIInputs, accountID)
launchInput = &SystemInstanceInput{
InstanceType: s.getSystemInstanceType(),
SubnetID: subnets[0],
ENIID: eniIDs[0],
ENIMac: primaryMAC,
ENIIP: primaryIP,
ExtraENIs: extraENIInputs,
Scheme: scheme,
AccountID: accountID,
DirectBoot: true,
NICs: nics,
LBAgentEnv: s.buildLBAgentEnv(lbID),
CACert: s.CACert,
}
}
} else {
// PC-machine path: cloud-init user-data carries the lb-agent config.
userData, udErr := s.lbVMUserData(lbID, scheme)
if udErr != nil {
slog.Error("CreateLoadBalancer: system credentials not configured — cannot launch ALB VM", "lbId", lbID, "err", udErr)
launchFailed = true
} else {
launchInput = &SystemInstanceInput{
InstanceType: s.getSystemInstanceType(),
ImageID: systemAMI,
SubnetID: subnets[0],
UserData: userData,
ENIID: eniIDs[0],
ENIMac: primaryMAC,
ENIIP: primaryIP,
ExtraENIs: extraENIInputs,
Scheme: scheme,
AccountID: accountID,
}
}
}
if !launchFailed && launchInput != nil {
// Dev-mode only: forward HTTP/HTTPS ports from host for local testing.
// In production (VPC networking), traffic reaches the ALB VM's VPC IP directly.
if s.config != nil && s.config.Daemon.DevNetworking {