diff --git a/spinifex/admin/network_detect.go b/spinifex/admin/network_detect.go index 023b110a6..532097cd9 100644 --- a/spinifex/admin/network_detect.go +++ b/spinifex/admin/network_detect.go @@ -188,6 +188,10 @@ func SuggestPoolRange(wan *DetectedInterface) (start, end string) { // isVirtualInterface returns true for interfaces that shouldn't be // considered as physical NICs. func isVirtualInterface(name string) bool { + // Allow br-external (OVS WAN bridge used for external networking) + if name == "br-external" { + return false + } prefixes := []string{ "docker", "br-", "veth", "ovs", "virbr", "tun", "tap", "tailscale", "wg", "lo", diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index eaf954600..6030e9d21 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -105,6 +105,7 @@ type Daemon struct { eigwService *handlers_ec2_eigw.EgressOnlyIGWServiceImpl igwService *handlers_ec2_igw.IGWServiceImpl vpcService *handlers_ec2_vpc.VPCServiceImpl + externalIPAM *handlers_ec2_vpc.ExternalIPAM ctx context.Context cancel context.CancelFunc shutdownWg sync.WaitGroup @@ -512,6 +513,34 @@ func (d *Daemon) Start() error { return fmt.Errorf("failed to initialize VPC service: %w", err) } + // Initialize external IPAM if external networking is configured + if d.clusterConfig != nil && d.clusterConfig.Network.ExternalMode != "" { + js, jsErr := d.natsConn.JetStream() + if jsErr != nil { + slog.Warn("Failed to get JetStream for external IPAM", "err", jsErr) + } else { + var pools []handlers_ec2_vpc.ExternalPoolConfig + for _, p := range d.clusterConfig.Network.ExternalPools { + pools = append(pools, handlers_ec2_vpc.ExternalPoolConfig{ + Name: p.Name, + RangeStart: p.RangeStart, + RangeEnd: p.RangeEnd, + Gateway: p.Gateway, + GatewayIP: p.GatewayIP, + PrefixLen: p.PrefixLen, + Region: p.Region, + AZ: p.AZ, + }) + } + d.externalIPAM, err = handlers_ec2_vpc.NewExternalIPAM(js, pools) + if err != nil { + slog.Warn("Failed to initialize external IPAM", "err", err) + } else { + slog.Info("External IPAM initialized", "mode", d.clusterConfig.Network.ExternalMode, "pools", len(pools)) + } + } + } + d.accountService, err = initServiceWithRetry("account settings service", func() (*handlers_ec2_account.AccountSettingsServiceImpl, error) { return handlers_ec2_account.NewAccountSettingsServiceImplWithNATS(d.config, d.natsConn) }) @@ -1565,6 +1594,24 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er } } + // Release public IP before deleting ENI + if deleteVolume && instance.PublicIP != "" && instance.PublicIPPool != "" && d.externalIPAM != nil { + // Publish vpc.delete-nat to remove dnat_and_snat rule + portName := "port-" + instance.ENIId + vpcId := "" + if instance.Instance != nil && instance.Instance.VpcId != nil { + vpcId = *instance.Instance.VpcId + } + d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, "", portName, "") + + // Release IP back to pool + if err := d.externalIPAM.ReleaseIP(instance.PublicIPPool, instance.PublicIP); err != nil { + slog.Warn("Failed to release public IP on termination", "ip", instance.PublicIP, "pool", instance.PublicIPPool, "err", err) + } else { + slog.Info("Released public IP on termination", "ip", instance.PublicIP, "instanceId", instance.ID) + } + } + // On termination, delete the auto-created ENI (releases IP back to IPAM, // publishes vpc.delete-port for vpcd). On stop, ENI persists (AWS behavior). if deleteVolume && instance.ENIId != "" && d.vpcService != nil { diff --git a/spinifex/daemon/daemon_handlers_instance.go b/spinifex/daemon/daemon_handlers_instance.go index b6409567d..a56903c56 100644 --- a/spinifex/daemon/daemon_handlers_instance.go +++ b/spinifex/daemon/daemon_handlers_instance.go @@ -196,6 +196,41 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) { "privateIp", *eni.PrivateIpAddress, "mac", instance.ENIMac, ) + + // Auto-assign public IP if subnet has MapPublicIpOnLaunch and external IPAM is available + if d.externalIPAM != nil { + subnet, subErr := d.vpcService.GetSubnet(accountID, *runInstancesInput.SubnetId) + if subErr == nil && subnet.MapPublicIpOnLaunch { + region := "" + az := "" + if d.config != nil { + region = d.config.Region + az = d.config.AZ + } + publicIP, poolName, allocErr := d.externalIPAM.AllocateIP(region, az, "auto_assign", *eni.NetworkInterfaceId, instance.ID) + if allocErr != nil { + slog.Warn("Failed to allocate public IP for instance", "instanceId", instance.ID, "err", allocErr) + } else { + // Update ENI record with public IP + if updateErr := d.vpcService.UpdateENIPublicIP(accountID, *eni.NetworkInterfaceId, publicIP, poolName); updateErr != nil { + slog.Warn("Failed to update ENI with public IP", "eniId", *eni.NetworkInterfaceId, "err", updateErr) + } + // Publish vpc.add-nat for dnat_and_snat rule + portName := "port-" + *eni.NetworkInterfaceId + d.publishNATEvent("vpc.add-nat", *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress) + // Set on ec2Instance response + ec2Instance.PublicIpAddress = aws.String(publicIP) + instance.PublicIP = publicIP + instance.PublicIPPool = poolName + slog.Info("Auto-assigned public IP", + "instanceId", instance.ID, + "publicIp", publicIP, + "privateIp", *eni.PrivateIpAddress, + "pool", poolName, + ) + } + } + } } instances = append(instances, instance) @@ -628,6 +663,11 @@ func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) { instanceCopy := *instance.Instance instanceCopy.State = &ec2.InstanceState{} + // Populate PublicIpAddress from VM if stored + if instance.PublicIP != "" && instanceCopy.PublicIpAddress == nil { + instanceCopy.PublicIpAddress = aws.String(instance.PublicIP) + } + // Map internal status to EC2 state codes using the centralized mapping if info, ok := vm.EC2StateCodes[instance.Status]; ok { instanceCopy.State.SetCode(info.Code) @@ -899,6 +939,34 @@ func (d *Daemon) handleEC2TerminateStoppedInstance(msg *nats.Msg) { } instance.EBSRequests.Mu.Unlock() + // Release public IP before termination + if instance.PublicIP != "" && instance.PublicIPPool != "" && d.externalIPAM != nil { + portName := "port-" + instance.ENIId + vpcId := "" + if instance.Instance != nil && instance.Instance.VpcId != nil { + vpcId = *instance.Instance.VpcId + } + d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, "", portName, "") + + if err := d.externalIPAM.ReleaseIP(instance.PublicIPPool, instance.PublicIP); err != nil { + slog.Warn("handleEC2TerminateStoppedInstance: failed to release public IP", "ip", instance.PublicIP, "pool", instance.PublicIPPool, "err", err) + } else { + slog.Info("handleEC2TerminateStoppedInstance: released public IP", "ip", instance.PublicIP, "instanceId", req.InstanceID) + } + } + + // Delete ENI if present + if instance.ENIId != "" && d.vpcService != nil { + _, eniErr := d.vpcService.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{ + NetworkInterfaceId: &instance.ENIId, + }, instance.AccountID) + if eniErr != nil { + slog.Error("handleEC2TerminateStoppedInstance: failed to delete ENI", "eni", instance.ENIId, "err", eniErr) + } else { + slog.Info("handleEC2TerminateStoppedInstance: deleted ENI", "eni", instance.ENIId, "instanceId", req.InstanceID) + } + } + // Write to terminated KV bucket FIRST so the instance is visible in DescribeInstances. // If this fails, the instance remains in the stopped bucket (safe to retry). instance.Status = vm.StateTerminated @@ -1115,3 +1183,25 @@ func (d *Daemon) handleEC2ModifyInstanceAttribute(msg *nats.Msg) { slog.Error("Failed to respond to NATS request", "err", err) } } + +// publishNATEvent publishes a NAT lifecycle event (vpc.add-nat or vpc.delete-nat) to NATS. +func (d *Daemon) publishNATEvent(topic, vpcId, externalIP, logicalIP, portName, mac string) { + if d.natsConn == nil { + return + } + evt := struct { + VpcId string `json:"vpc_id"` + ExternalIP string `json:"external_ip"` + LogicalIP string `json:"logical_ip"` + PortName string `json:"port_name"` + MAC string `json:"mac"` + }{VpcId: vpcId, ExternalIP: externalIP, LogicalIP: logicalIP, PortName: portName, MAC: mac} + data, err := json.Marshal(evt) + if err != nil { + slog.Error("Failed to marshal NAT event", "topic", topic, "err", err) + return + } + if err := d.natsConn.Publish(topic, data); err != nil { + slog.Error("Failed to publish NAT event", "topic", topic, "err", err) + } +} diff --git a/spinifex/handlers/ec2/vpc/eni.go b/spinifex/handlers/ec2/vpc/eni.go index e3049e72d..741ab0951 100644 --- a/spinifex/handlers/ec2/vpc/eni.go +++ b/spinifex/handlers/ec2/vpc/eni.go @@ -325,6 +325,34 @@ func (s *VPCServiceImpl) DetachENI(accountID, eniId string) error { return nil } +// UpdateENIPublicIP updates the PublicIpAddress and PublicIpPool on an ENI record. +func (s *VPCServiceImpl) UpdateENIPublicIP(accountID, eniId, publicIP, poolName string) error { + key := utils.AccountKey(accountID, eniId) + entry, err := s.eniKV.Get(key) + if err != nil { + return fmt.Errorf("ENI %s not found: %w", eniId, err) + } + + var record ENIRecord + if err := json.Unmarshal(entry.Value(), &record); err != nil { + return fmt.Errorf("unmarshal ENI record: %w", err) + } + + record.PublicIpAddress = publicIP + record.PublicIpPool = poolName + + data, err := json.Marshal(record) + if err != nil { + return fmt.Errorf("marshal ENI record: %w", err) + } + if _, err := s.eniKV.Update(key, data, entry.Revision()); err != nil { + return fmt.Errorf("update ENI record: %w", err) + } + + slog.Info("Updated ENI with public IP", "eniId", eniId, "publicIp", publicIP, "pool", poolName) + return nil +} + // eniRecordToEC2 converts an ENI record to an EC2 NetworkInterface func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string) *ec2.NetworkInterface { eni := &ec2.NetworkInterface{ diff --git a/spinifex/handlers/ec2/vpc/service_impl.go b/spinifex/handlers/ec2/vpc/service_impl.go index 89cd4635c..f32665037 100644 --- a/spinifex/handlers/ec2/vpc/service_impl.go +++ b/spinifex/handlers/ec2/vpc/service_impl.go @@ -837,6 +837,20 @@ func (s *VPCServiceImpl) GetDefaultSubnet(accountID string) (*SubnetRecord, erro return nil, fmt.Errorf("no default subnet found") } +// GetSubnet looks up a SubnetRecord by ID. +func (s *VPCServiceImpl) GetSubnet(accountID, subnetId string) (*SubnetRecord, error) { + key := utils.AccountKey(accountID, subnetId) + entry, err := s.subnetKV.Get(key) + if err != nil { + return nil, fmt.Errorf("subnet %s not found: %w", subnetId, err) + } + var record SubnetRecord + if err := json.Unmarshal(entry.Value(), &record); err != nil { + return nil, fmt.Errorf("unmarshal subnet record: %w", err) + } + return &record, nil +} + // publishVPCEvent publishes a VPC lifecycle event to NATS for vpcd consumption. // This is fire-and-forget; errors are logged but do not fail the API response. func (s *VPCServiceImpl) publishVPCEvent(topic, vpcId, cidrBlock string, vni int64) { diff --git a/spinifex/vm/vm.go b/spinifex/vm/vm.go index a967161d0..ae1aa5a21 100644 --- a/spinifex/vm/vm.go +++ b/spinifex/vm/vm.go @@ -68,6 +68,10 @@ type VM struct { ENIId string `json:"eni_id,omitempty"` ENIMac string `json:"eni_mac,omitempty"` + // Public IP auto-assigned from external IPAM pool (released on termination) + PublicIP string `json:"public_ip,omitempty"` + PublicIPPool string `json:"public_ip_pool,omitempty"` + // DevMAC is the MAC for the dev/hostfwd NIC (DEV_NETWORKING mode). // Set before cloud-init ISO generation so netplan can suppress its default route. DevMAC string `json:"dev_mac,omitempty"`