diff --git a/spinifex/handlers/elbv2/arn_test.go b/spinifex/handlers/elbv2/arn_test.go index 06dca3024..24de00a39 100644 --- a/spinifex/handlers/elbv2/arn_test.go +++ b/spinifex/handlers/elbv2/arn_test.go @@ -4,6 +4,12 @@ import ( "strings" "testing" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/mulgadc/spinifex/spinifex/config" + handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + "github.com/mulgadc/spinifex/spinifex/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -125,6 +131,259 @@ func TestLbVMUserData_Structural(t *testing.T) { assert.NotContains(t, ud, "bootcmd:") } +// TestBuildLBAgentEnv verifies that buildLBAgentEnv produces a KEY=value blob +// containing all five env vars with the correct values. +func TestBuildLBAgentEnv(t *testing.T) { + svc := &ELBv2ServiceImpl{ + GatewayURL: "https://10.0.0.1:9999", + SystemAccessKey: "AKIAIOSFODNN7EXAMPLE", + SystemSecretKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + region: "ap-southeast-2", + } + env := svc.buildLBAgentEnv("lb-abc123") + + lines := strings.Split(strings.TrimRight(env, "\n"), "\n") + kvs := make(map[string]string, len(lines)) + for _, l := range lines { + k, v, ok := strings.Cut(l, "=") + require.True(t, ok, "env line missing '=': %q", l) + kvs[k] = v + } + + assert.Equal(t, "lb-abc123", kvs["LB_LB_ID"]) + assert.Equal(t, "https://10.0.0.1:9999", kvs["LB_GATEWAY_URL"]) + assert.Equal(t, "AKIAIOSFODNN7EXAMPLE", kvs["LB_ACCESS_KEY"]) + assert.Equal(t, "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", kvs["LB_SECRET_KEY"]) + assert.Equal(t, "ap-southeast-2", kvs["LB_REGION"]) +} + +// TestSubnetCIDRForIP and TestSubnetGatewayIP cover the CIDR helpers. +func TestSubnetCIDRForIP(t *testing.T) { + assert.Equal(t, "10.0.1.5/24", subnetCIDRForIP("10.0.1.5", "10.0.1.0/24")) + assert.Equal(t, "172.16.2.10/20", subnetCIDRForIP("172.16.2.10", "172.16.0.0/20")) + assert.Equal(t, "", subnetCIDRForIP("10.0.1.5", "bad-cidr")) +} + +func TestSubnetGatewayIP(t *testing.T) { + assert.Equal(t, "10.0.1.1", subnetGatewayIP("10.0.1.0/24")) + assert.Equal(t, "172.16.0.1", subnetGatewayIP("172.16.0.0/20")) + assert.Equal(t, "", subnetGatewayIP("not-a-cidr")) +} + +// setupMicrovmTestService creates an ELBv2 service with ELBv2Enabled=true wired +// to a real VPC service. The VPC has one subnet (10.0.1.0/24) pre-created. +func setupMicrovmTestService(t *testing.T) (*ELBv2ServiceImpl, *handlers_ec2_vpc.VPCServiceImpl, string) { + t.Helper() + _, nc, _ := testutil.StartTestJetStream(t) + testutil.StubVpcdSGResponder(t, nc) + + vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(nil, nc) + require.NoError(t, err) + + cfg := &config.Config{ + Daemon: config.DaemonConfig{ + DevNetworking: true, + Microvm: config.MicrovmConfig{ELBv2Enabled: true}, + }, + } + elbv2Svc, err := NewELBv2ServiceImplWithNATS(cfg, nc) + require.NoError(t, err) + elbv2Svc.VPCService = vpcSvc + elbv2Svc.GatewayURL = "https://10.20.0.5:9999" + elbv2Svc.SystemAccessKey = "AKID" + elbv2Svc.SystemSecretKey = "SECRET" + elbv2Svc.region = "us-east-1" + elbv2Svc.CACert = "-----BEGIN CERTIFICATE-----\nMOCK\n-----END CERTIFICATE-----\n" + + vpcOut, err := vpcSvc.CreateVpc(&ec2.CreateVpcInput{ + CidrBlock: aws.String("10.0.0.0/16"), + }, testAccountID) + require.NoError(t, err) + vpcID := *vpcOut.Vpc.VpcId + + subnetOut, err := vpcSvc.CreateSubnet(&ec2.CreateSubnetInput{ + VpcId: aws.String(vpcID), + CidrBlock: aws.String("10.0.1.0/24"), + AvailabilityZone: aws.String("us-east-1a"), + }, testAccountID) + require.NoError(t, err) + subnetID := *subnetOut.Subnet.SubnetId + + return elbv2Svc, vpcSvc, subnetID +} + +// TestCreateLoadBalancer_Microvm_DirectBoot asserts the microvm launch path +// sets DirectBoot=true, leaves ImageID empty, and delivers lb-agent env vars. +func TestCreateLoadBalancer_Microvm_DirectBoot(t *testing.T) { + svc, _, subnetID := setupMicrovmTestService(t) + + mock := &mockSystemInstanceLauncher{ + launchResult: &SystemInstanceOutput{ + InstanceID: "i-microvm-test", + PrivateIP: "10.0.1.5", + }, + } + svc.InstanceLauncher = mock + + _, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{ + Name: aws.String("mv-alb"), + Subnets: []*string{aws.String(subnetID)}, + }, testAccountID) + require.NoError(t, err) + + require.Len(t, mock.launchCalls, 1) + inp := mock.launchCalls[0] + + assert.True(t, inp.DirectBoot, "DirectBoot must be true in microvm mode") + assert.Equal(t, "", inp.ImageID, "ImageID must be empty in microvm mode") + assert.Equal(t, "-----BEGIN CERTIFICATE-----\nMOCK\n-----END CERTIFICATE-----\n", inp.CACert) +} + +// TestCreateLoadBalancer_Microvm_LBAgentEnv asserts all five lb-agent env vars +// appear in LBAgentEnv with correct values. +func TestCreateLoadBalancer_Microvm_LBAgentEnv(t *testing.T) { + svc, _, subnetID := setupMicrovmTestService(t) + + mock := &mockSystemInstanceLauncher{ + launchResult: &SystemInstanceOutput{InstanceID: "i-env-test", PrivateIP: "10.0.1.6"}, + } + svc.InstanceLauncher = mock + + _, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{ + Name: aws.String("env-alb"), + Subnets: []*string{aws.String(subnetID)}, + }, testAccountID) + require.NoError(t, err) + require.Len(t, mock.launchCalls, 1) + + envBlob := mock.launchCalls[0].LBAgentEnv + kvs := parseEnvBlob(t, envBlob) + + assert.NotEmpty(t, kvs["LB_LB_ID"]) + assert.Equal(t, "https://10.20.0.5:9999", kvs["LB_GATEWAY_URL"]) + assert.Equal(t, "AKID", kvs["LB_ACCESS_KEY"]) + assert.Equal(t, "SECRET", kvs["LB_SECRET_KEY"]) + assert.Equal(t, "us-east-1", kvs["LB_REGION"]) +} + +// TestCreateLoadBalancer_Microvm_NICs asserts NIC[0].IsDefault=true and +// NIC[1].IsDefault=false, and that CIDR/Gateway are derived from the subnet. +func TestCreateLoadBalancer_Microvm_NICs(t *testing.T) { + svc, _, subnetID := setupMicrovmTestService(t) + + mock := &mockSystemInstanceLauncher{ + launchResult: &SystemInstanceOutput{InstanceID: "i-nic-test", PrivateIP: "10.0.1.7"}, + } + svc.InstanceLauncher = mock + + _, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{ + Name: aws.String("nic-alb"), + Subnets: []*string{aws.String(subnetID)}, + }, testAccountID) + require.NoError(t, err) + require.Len(t, mock.launchCalls, 1) + + nics := mock.launchCalls[0].NICs + require.GreaterOrEqual(t, len(nics), 2, "must have at least primary ENI NIC and mgmt NIC") + + // NIC[0]: primary VPC ENI — must be the default route owner. + assert.True(t, nics[0].IsDefault, "NIC[0] must have IsDefault=true") + assert.NotEmpty(t, nics[0].MAC, "NIC[0] MAC must be populated from ENI") + assert.Contains(t, nics[0].CIDR, "/24", "NIC[0] CIDR must include prefix from subnet") + assert.Equal(t, "10.0.1.1", nics[0].Gateway, "NIC[0] gateway must be subnet .1 address") + + // NIC[1]: management NIC — daemon allocates IP/MAC. + assert.False(t, nics[1].IsDefault, "NIC[1] must have IsDefault=false") +} + +// TestCreateLoadBalancer_Microvm_MissingCredentials_SetsStateFailed verifies +// that missing system credentials on the microvm path result in StateFailed. +func TestCreateLoadBalancer_Microvm_MissingCredentials_SetsStateFailed(t *testing.T) { + svc, _, subnetID := setupMicrovmTestService(t) + // Clear credentials to trigger the failure path. + svc.GatewayURL = "" + + mock := &mockSystemInstanceLauncher{ + launchResult: &SystemInstanceOutput{InstanceID: "i-should-not-launch"}, + } + svc.InstanceLauncher = mock + + out, err := svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{ + Name: aws.String("mv-nocreds-alb"), + Subnets: []*string{aws.String(subnetID)}, + }, testAccountID) + require.NoError(t, err) + assert.Equal(t, StateFailed, *out.LoadBalancers[0].State.Code) + assert.Empty(t, mock.launchCalls, "launcher must not be invoked when credentials are missing") +} + +// TestCreateLoadBalancer_PCMachine_Unchanged verifies that when ELBv2Enabled=false +// the existing PC-machine cloud-config path is used and DirectBoot is not set. +func TestCreateLoadBalancer_PCMachine_Unchanged(t *testing.T) { + _, nc, _ := testutil.StartTestJetStream(t) + testutil.StubVpcdSGResponder(t, nc) + + vpcSvc, err := handlers_ec2_vpc.NewVPCServiceImplWithNATS(nil, nc) + require.NoError(t, err) + + cfg := &config.Config{ + Daemon: config.DaemonConfig{ + DevNetworking: true, + Microvm: config.MicrovmConfig{ELBv2Enabled: false}, + }, + } + svc, err := NewELBv2ServiceImplWithNATS(cfg, nc) + require.NoError(t, err) + svc.VPCService = vpcSvc + svc.GatewayURL = "https://10.0.0.1:9999" + svc.SystemAccessKey = "AKID" + svc.SystemSecretKey = "SECRET" + svc.SetSystemAMIFunc(func() (string, error) { return "ami-lb-test", nil }) + + vpcOut, err := vpcSvc.CreateVpc(&ec2.CreateVpcInput{CidrBlock: aws.String("10.0.0.0/16")}, testAccountID) + require.NoError(t, err) + subnetOut, err := vpcSvc.CreateSubnet(&ec2.CreateSubnetInput{ + VpcId: vpcOut.Vpc.VpcId, + CidrBlock: aws.String("10.0.2.0/24"), + AvailabilityZone: aws.String("us-east-1a"), + }, testAccountID) + require.NoError(t, err) + + mock := &mockSystemInstanceLauncher{ + launchResult: &SystemInstanceOutput{InstanceID: "i-pc-test", PrivateIP: "10.0.2.5"}, + } + svc.InstanceLauncher = mock + + _, err = svc.CreateLoadBalancer(&elbv2.CreateLoadBalancerInput{ + Name: aws.String("pc-alb"), + Subnets: []*string{subnetOut.Subnet.SubnetId}, + }, testAccountID) + require.NoError(t, err) + require.Len(t, mock.launchCalls, 1) + + inp := mock.launchCalls[0] + assert.False(t, inp.DirectBoot, "DirectBoot must be false on PC-machine path") + assert.Equal(t, "ami-lb-test", inp.ImageID, "ImageID must be set on PC-machine path") + assert.NotEmpty(t, inp.UserData, "UserData (cloud-config) must be set on PC-machine path") + assert.True(t, strings.HasPrefix(inp.UserData, "#cloud-config\n"), + "UserData must be cloud-config on PC-machine path") +} + +// parseEnvBlob parses a KEY=value newline-separated blob into a map. +func parseEnvBlob(t *testing.T, blob string) map[string]string { + t.Helper() + kvs := make(map[string]string) + for line := range strings.SplitSeq(strings.TrimRight(blob, "\n"), "\n") { + if line == "" { + continue + } + k, v, ok := strings.Cut(line, "=") + require.True(t, ok, "env blob line missing '=': %q", line) + kvs[k] = v + } + return kvs +} + // TestLbVMUserData_MissingCredentials covers the three required-field error // paths (gateway URL, access key, secret key) in one table-driven test. func TestLbVMUserData_MissingCredentials(t *testing.T) { diff --git a/spinifex/handlers/elbv2/service_impl.go b/spinifex/handlers/elbv2/service_impl.go index 641e063f2..f83090ffb 100644 --- a/spinifex/handlers/elbv2/service_impl.go +++ b/spinifex/handlers/elbv2/service_impl.go @@ -7,6 +7,7 @@ import ( "fmt" "log/slog" "maps" + "net" "regexp" "slices" "strings" @@ -141,6 +142,7 @@ type ELBv2ServiceImpl struct { MgmtRouteTarget string // AWSGW bind IP to route via mgmt NIC MgmtBridgeIP string // br-mgmt IP, populated whenever br-mgmt exists (single + multi node) for the internal-scheme fallback route AdvertiseIP string // AdvertiseIP / WAN gateway, populated whenever set; used as the internal-scheme fallback route target on single-node + CACert string // PEM-encoded CA certificate delivered to microvm guests via fw_cfg (direct-boot path only) nodeID string region string systemAMI string // AMI ID for system VMs (ALB VMs); resolved lazily via systemAMIFunc @@ -255,6 +257,109 @@ func (s *ELBv2ServiceImpl) getSystemInstanceType() string { return s.systemInstanceType } +// buildLBAgentEnv returns the KEY=value blob written to /etc/conf.d/lb-agent +// via fw_cfg on the direct-boot path. +func (s *ELBv2ServiceImpl) buildLBAgentEnv(lbID string) string { + return fmt.Sprintf("LB_LB_ID=%s\nLB_GATEWAY_URL=%s\nLB_ACCESS_KEY=%s\nLB_SECRET_KEY=%s\nLB_REGION=%s\n", + lbID, s.GatewayURL, s.SystemAccessKey, s.SystemSecretKey, s.region) +} + +// subnetCIDRForIP computes "ip/prefixlen" given a host IP and the subnet's +// CIDR block (e.g. "10.0.1.0/24" → "10.0.1.5/24"). Returns empty string on +// parse failure so the caller can log and continue without panicking. +func subnetCIDRForIP(hostIP, subnetCIDR string) string { + _, ipNet, err := net.ParseCIDR(subnetCIDR) + if err != nil { + return "" + } + ones, _ := ipNet.Mask.Size() + return fmt.Sprintf("%s/%d", hostIP, ones) +} + +// subnetGatewayIP derives the gateway IP (network address +1) from a subnet +// CIDR block. Returns empty string on parse failure. +func subnetGatewayIP(subnetCIDR string) string { + _, ipNet, err := net.ParseCIDR(subnetCIDR) + if err != nil { + return "" + } + gw := ipNet.IP.To4() + if gw == nil { + return "" + } + gwCopy := make(net.IP, len(gw)) + copy(gwCopy, gw) + gwCopy[3]++ + return gwCopy.String() +} + +// buildMicrovmNICs constructs the NIC slice for a direct-boot microvm launch. +// +// NIC[0] is the primary VPC ENI (IsDefault=true): MAC/IP from the ENI record, +// CIDR from ip+subnet prefix, gateway = subnet network address +1. +// +// NIC[1] is the management NIC (IsDefault=false): MAC and CIDR are left empty +// here — the daemon injects them after allocating the per-instance mgmt IP. +// RouteDst/RouteVia encode the host route the lb-agent needs to reach AWSGW +// via the management interface (same logic as resolveMgmtRoute). +// +// NIC[2+] are extra VPC ENIs for multi-subnet ALBs. +func (s *ELBv2ServiceImpl) buildMicrovmNICs(primaryIP, primaryMAC, primarySubnetID, _, scheme string, extraENIs []ExtraENIInput, accountID string) []NICConfig { + // Resolve primary subnet CIDR for NIC[0] prefix and gateway. + primaryCIDR := "" + primaryGateway := "" + if s.VPCService != nil && primarySubnetID != "" { + if subnet, err := s.VPCService.GetSubnet(accountID, primarySubnetID); err == nil && subnet != nil { + primaryCIDR = subnetCIDRForIP(primaryIP, subnet.CidrBlock) + primaryGateway = subnetGatewayIP(subnet.CidrBlock) + } else { + slog.Warn("buildMicrovmNICs: could not look up primary subnet", "subnetId", primarySubnetID, "err", err) + } + } + + nics := []NICConfig{ + // NIC[0]: primary VPC ENI — default route owner. + { + MAC: primaryMAC, + CIDR: primaryCIDR, + Gateway: primaryGateway, + IsDefault: true, + }, + } + + // NIC[1]: management NIC — daemon allocates IP/MAC; we provide routing only. + // RouteDst and RouteVia reproduce the bootcmd host route from the PC-machine + // cloud-config path (resolveMgmtRoute) so lb-agent can reach AWSGW. + mgmtRouteDst, mgmtRouteVia := s.resolveMgmtRoute(scheme) + nics = append(nics, NICConfig{ + IsDefault: false, + RouteDst: mgmtRouteDst, + RouteVia: mgmtRouteVia, + }) + + // NIC[2+]: extra ENIs for multi-subnet ALBs. + for _, extra := range extraENIs { + extraCIDR := "" + extraGateway := "" + if s.VPCService != nil && extra.SubnetID != "" { + if subnet, err := s.VPCService.GetSubnet(accountID, extra.SubnetID); err == nil && subnet != nil { + extraCIDR = subnetCIDRForIP(extra.ENIIP, subnet.CidrBlock) + extraGateway = subnetGatewayIP(subnet.CidrBlock) + } else { + slog.Warn("buildMicrovmNICs: could not look up extra subnet", "subnetId", extra.SubnetID, "err", err) + } + } + nics = append(nics, NICConfig{ + MAC: extra.ENIMac, + CIDR: extraCIDR, + Gateway: extraGateway, + IsDefault: false, + }) + } + + return nics +} + // resolveENIBindAddr looks up the private IP of the first ENI belonging to the ALB. // Returns empty string if VPC service is unavailable or no ENIs exist. func (s *ELBv2ServiceImpl) resolveENIBindAddr(lb *LoadBalancerRecord) string { @@ -671,15 +776,24 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp var hostPorts map[int]int var launchFailed bool if s.InstanceLauncher != nil && len(eniIDs) > 0 && len(subnets) > 0 { - systemAMI, amiErr := s.getSystemAMI() - if amiErr != nil { - slog.Error("CreateLoadBalancer: cannot resolve LB system AMI", "lbId", lbID, "err", amiErr) - return nil, errors.New(awserrors.ErrorServerInternal) - } - if systemAMI == "" { - slog.Error("CreateLoadBalancer: LB system AMI not resolved", "lbId", lbID) - return nil, errors.New(awserrors.ErrorServerInternal) + microvmEnabled := s.config != nil && s.config.Daemon.Microvm.ELBv2Enabled + + // For the PC-machine path, AMI must be resolved before continuing. + // The microvm (direct-boot) path skips AMI entirely. + var systemAMI string + if !microvmEnabled { + var amiErr error + systemAMI, amiErr = s.getSystemAMI() + if amiErr != nil { + slog.Error("CreateLoadBalancer: cannot resolve LB system AMI", "lbId", lbID, "err", amiErr) + return nil, errors.New(awserrors.ErrorServerInternal) + } + if systemAMI == "" { + slog.Error("CreateLoadBalancer: LB system AMI not resolved", "lbId", lbID) + return nil, errors.New(awserrors.ErrorServerInternal) + } } + // Resolve MAC/IP for every ENI in one describe call. eniDetails := make(map[string]*ec2.NetworkInterface, len(eniIDs)) if s.VPCService != nil { @@ -721,23 +835,51 @@ func (s *ELBv2ServiceImpl) CreateLoadBalancer(input *elbv2.CreateLoadBalancerInp }) } - userData, udErr := s.lbVMUserData(lbID, scheme) - if udErr != nil { - slog.Error("CreateLoadBalancer: system credentials not configured — cannot launch ALB VM", "lbId", lbID, "err", udErr) - launchFailed = true - } else { - launchInput := &SystemInstanceInput{ - InstanceType: s.getSystemInstanceType(), - ImageID: systemAMI, - SubnetID: subnets[0], - UserData: userData, - ENIID: eniIDs[0], - ENIMac: primaryMAC, - ENIIP: primaryIP, - ExtraENIs: extraENIInputs, - Scheme: scheme, - AccountID: accountID, + var launchInput *SystemInstanceInput + if microvmEnabled { + // Direct-boot microvm path: skip AMI/volumes, deliver config via fw_cfg. + if s.GatewayURL == "" || s.SystemAccessKey == "" || s.SystemSecretKey == "" { + slog.Error("CreateLoadBalancer: system credentials not configured for microvm path — cannot launch ALB VM", "lbId", lbID) + launchFailed = true + } else { + nics := s.buildMicrovmNICs(primaryIP, primaryMAC, subnets[0], eniIDs[0], scheme, extraENIInputs, accountID) + launchInput = &SystemInstanceInput{ + InstanceType: s.getSystemInstanceType(), + SubnetID: subnets[0], + ENIID: eniIDs[0], + ENIMac: primaryMAC, + ENIIP: primaryIP, + ExtraENIs: extraENIInputs, + Scheme: scheme, + AccountID: accountID, + DirectBoot: true, + NICs: nics, + LBAgentEnv: s.buildLBAgentEnv(lbID), + CACert: s.CACert, + } } + } else { + // PC-machine path: cloud-init user-data carries the lb-agent config. + userData, udErr := s.lbVMUserData(lbID, scheme) + if udErr != nil { + slog.Error("CreateLoadBalancer: system credentials not configured — cannot launch ALB VM", "lbId", lbID, "err", udErr) + launchFailed = true + } else { + launchInput = &SystemInstanceInput{ + InstanceType: s.getSystemInstanceType(), + ImageID: systemAMI, + SubnetID: subnets[0], + UserData: userData, + ENIID: eniIDs[0], + ENIMac: primaryMAC, + ENIIP: primaryIP, + ExtraENIs: extraENIInputs, + Scheme: scheme, + AccountID: accountID, + } + } + } + if !launchFailed && launchInput != nil { // Dev-mode only: forward HTTP/HTTPS ports from host for local testing. // In production (VPC networking), traffic reaches the ALB VM's VPC IP directly. if s.config != nil && s.config.Daemon.DevNetworking {