feat: Phase 8c — daemon auto-assign public IP on RunInstances

Wire public IP auto-assignment end-to-end:
- RunInstances checks MapPublicIpOnLaunch on subnet, allocates external
  IP from ExternalIPAM, updates ENI record, publishes vpc.add-nat for
  dnat_and_snat rule, returns PublicIpAddress in response
- TerminateInstances releases public IP back to pool, publishes
  vpc.delete-nat to remove NAT rule before ENI deletion
- DescribeInstances populates PublicIpAddress from VM struct

Add ExternalIPAM initialization in daemon startup from cluster
NetworkConfig. Add GetSubnet and UpdateENIPublicIP helpers. Add
PublicIP/PublicIPPool fields to vm.VM struct. Fix network auto-detect
to recognize br-external as WAN interface.
This commit is contained in:
Ben Duncan
2026-03-20 06:41:32 +00:00
parent a554fe4ecf
commit 946429d792
6 changed files with 187 additions and 0 deletions
+4
View File
@@ -188,6 +188,10 @@ func SuggestPoolRange(wan *DetectedInterface) (start, end string) {
// isVirtualInterface returns true for interfaces that shouldn't be
// considered as physical NICs.
func isVirtualInterface(name string) bool {
// Allow br-external (OVS WAN bridge used for external networking)
if name == "br-external" {
return false
}
prefixes := []string{
"docker", "br-", "veth", "ovs", "virbr",
"tun", "tap", "tailscale", "wg", "lo",
+47
View File
@@ -105,6 +105,7 @@ type Daemon struct {
eigwService *handlers_ec2_eigw.EgressOnlyIGWServiceImpl
igwService *handlers_ec2_igw.IGWServiceImpl
vpcService *handlers_ec2_vpc.VPCServiceImpl
externalIPAM *handlers_ec2_vpc.ExternalIPAM
ctx context.Context
cancel context.CancelFunc
shutdownWg sync.WaitGroup
@@ -512,6 +513,34 @@ func (d *Daemon) Start() error {
return fmt.Errorf("failed to initialize VPC service: %w", err)
}
// Initialize external IPAM if external networking is configured
if d.clusterConfig != nil && d.clusterConfig.Network.ExternalMode != "" {
js, jsErr := d.natsConn.JetStream()
if jsErr != nil {
slog.Warn("Failed to get JetStream for external IPAM", "err", jsErr)
} else {
var pools []handlers_ec2_vpc.ExternalPoolConfig
for _, p := range d.clusterConfig.Network.ExternalPools {
pools = append(pools, handlers_ec2_vpc.ExternalPoolConfig{
Name: p.Name,
RangeStart: p.RangeStart,
RangeEnd: p.RangeEnd,
Gateway: p.Gateway,
GatewayIP: p.GatewayIP,
PrefixLen: p.PrefixLen,
Region: p.Region,
AZ: p.AZ,
})
}
d.externalIPAM, err = handlers_ec2_vpc.NewExternalIPAM(js, pools)
if err != nil {
slog.Warn("Failed to initialize external IPAM", "err", err)
} else {
slog.Info("External IPAM initialized", "mode", d.clusterConfig.Network.ExternalMode, "pools", len(pools))
}
}
}
d.accountService, err = initServiceWithRetry("account settings service", func() (*handlers_ec2_account.AccountSettingsServiceImpl, error) {
return handlers_ec2_account.NewAccountSettingsServiceImplWithNATS(d.config, d.natsConn)
})
@@ -1565,6 +1594,24 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er
}
}
// Release public IP before deleting ENI
if deleteVolume && instance.PublicIP != "" && instance.PublicIPPool != "" && d.externalIPAM != nil {
// Publish vpc.delete-nat to remove dnat_and_snat rule
portName := "port-" + instance.ENIId
vpcId := ""
if instance.Instance != nil && instance.Instance.VpcId != nil {
vpcId = *instance.Instance.VpcId
}
d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, "", portName, "")
// Release IP back to pool
if err := d.externalIPAM.ReleaseIP(instance.PublicIPPool, instance.PublicIP); err != nil {
slog.Warn("Failed to release public IP on termination", "ip", instance.PublicIP, "pool", instance.PublicIPPool, "err", err)
} else {
slog.Info("Released public IP on termination", "ip", instance.PublicIP, "instanceId", instance.ID)
}
}
// On termination, delete the auto-created ENI (releases IP back to IPAM,
// publishes vpc.delete-port for vpcd). On stop, ENI persists (AWS behavior).
if deleteVolume && instance.ENIId != "" && d.vpcService != nil {
@@ -196,6 +196,41 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
"privateIp", *eni.PrivateIpAddress,
"mac", instance.ENIMac,
)
// Auto-assign public IP if subnet has MapPublicIpOnLaunch and external IPAM is available
if d.externalIPAM != nil {
subnet, subErr := d.vpcService.GetSubnet(accountID, *runInstancesInput.SubnetId)
if subErr == nil && subnet.MapPublicIpOnLaunch {
region := ""
az := ""
if d.config != nil {
region = d.config.Region
az = d.config.AZ
}
publicIP, poolName, allocErr := d.externalIPAM.AllocateIP(region, az, "auto_assign", *eni.NetworkInterfaceId, instance.ID)
if allocErr != nil {
slog.Warn("Failed to allocate public IP for instance", "instanceId", instance.ID, "err", allocErr)
} else {
// Update ENI record with public IP
if updateErr := d.vpcService.UpdateENIPublicIP(accountID, *eni.NetworkInterfaceId, publicIP, poolName); updateErr != nil {
slog.Warn("Failed to update ENI with public IP", "eniId", *eni.NetworkInterfaceId, "err", updateErr)
}
// Publish vpc.add-nat for dnat_and_snat rule
portName := "port-" + *eni.NetworkInterfaceId
d.publishNATEvent("vpc.add-nat", *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress)
// Set on ec2Instance response
ec2Instance.PublicIpAddress = aws.String(publicIP)
instance.PublicIP = publicIP
instance.PublicIPPool = poolName
slog.Info("Auto-assigned public IP",
"instanceId", instance.ID,
"publicIp", publicIP,
"privateIp", *eni.PrivateIpAddress,
"pool", poolName,
)
}
}
}
}
instances = append(instances, instance)
@@ -628,6 +663,11 @@ func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) {
instanceCopy := *instance.Instance
instanceCopy.State = &ec2.InstanceState{}
// Populate PublicIpAddress from VM if stored
if instance.PublicIP != "" && instanceCopy.PublicIpAddress == nil {
instanceCopy.PublicIpAddress = aws.String(instance.PublicIP)
}
// Map internal status to EC2 state codes using the centralized mapping
if info, ok := vm.EC2StateCodes[instance.Status]; ok {
instanceCopy.State.SetCode(info.Code)
@@ -899,6 +939,34 @@ func (d *Daemon) handleEC2TerminateStoppedInstance(msg *nats.Msg) {
}
instance.EBSRequests.Mu.Unlock()
// Release public IP before termination
if instance.PublicIP != "" && instance.PublicIPPool != "" && d.externalIPAM != nil {
portName := "port-" + instance.ENIId
vpcId := ""
if instance.Instance != nil && instance.Instance.VpcId != nil {
vpcId = *instance.Instance.VpcId
}
d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, "", portName, "")
if err := d.externalIPAM.ReleaseIP(instance.PublicIPPool, instance.PublicIP); err != nil {
slog.Warn("handleEC2TerminateStoppedInstance: failed to release public IP", "ip", instance.PublicIP, "pool", instance.PublicIPPool, "err", err)
} else {
slog.Info("handleEC2TerminateStoppedInstance: released public IP", "ip", instance.PublicIP, "instanceId", req.InstanceID)
}
}
// Delete ENI if present
if instance.ENIId != "" && d.vpcService != nil {
_, eniErr := d.vpcService.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
NetworkInterfaceId: &instance.ENIId,
}, instance.AccountID)
if eniErr != nil {
slog.Error("handleEC2TerminateStoppedInstance: failed to delete ENI", "eni", instance.ENIId, "err", eniErr)
} else {
slog.Info("handleEC2TerminateStoppedInstance: deleted ENI", "eni", instance.ENIId, "instanceId", req.InstanceID)
}
}
// Write to terminated KV bucket FIRST so the instance is visible in DescribeInstances.
// If this fails, the instance remains in the stopped bucket (safe to retry).
instance.Status = vm.StateTerminated
@@ -1115,3 +1183,25 @@ func (d *Daemon) handleEC2ModifyInstanceAttribute(msg *nats.Msg) {
slog.Error("Failed to respond to NATS request", "err", err)
}
}
// publishNATEvent publishes a NAT lifecycle event (vpc.add-nat or vpc.delete-nat) to NATS.
func (d *Daemon) publishNATEvent(topic, vpcId, externalIP, logicalIP, portName, mac string) {
if d.natsConn == nil {
return
}
evt := struct {
VpcId string `json:"vpc_id"`
ExternalIP string `json:"external_ip"`
LogicalIP string `json:"logical_ip"`
PortName string `json:"port_name"`
MAC string `json:"mac"`
}{VpcId: vpcId, ExternalIP: externalIP, LogicalIP: logicalIP, PortName: portName, MAC: mac}
data, err := json.Marshal(evt)
if err != nil {
slog.Error("Failed to marshal NAT event", "topic", topic, "err", err)
return
}
if err := d.natsConn.Publish(topic, data); err != nil {
slog.Error("Failed to publish NAT event", "topic", topic, "err", err)
}
}
+28
View File
@@ -325,6 +325,34 @@ func (s *VPCServiceImpl) DetachENI(accountID, eniId string) error {
return nil
}
// UpdateENIPublicIP updates the PublicIpAddress and PublicIpPool on an ENI record.
func (s *VPCServiceImpl) UpdateENIPublicIP(accountID, eniId, publicIP, poolName string) error {
key := utils.AccountKey(accountID, eniId)
entry, err := s.eniKV.Get(key)
if err != nil {
return fmt.Errorf("ENI %s not found: %w", eniId, err)
}
var record ENIRecord
if err := json.Unmarshal(entry.Value(), &record); err != nil {
return fmt.Errorf("unmarshal ENI record: %w", err)
}
record.PublicIpAddress = publicIP
record.PublicIpPool = poolName
data, err := json.Marshal(record)
if err != nil {
return fmt.Errorf("marshal ENI record: %w", err)
}
if _, err := s.eniKV.Update(key, data, entry.Revision()); err != nil {
return fmt.Errorf("update ENI record: %w", err)
}
slog.Info("Updated ENI with public IP", "eniId", eniId, "publicIp", publicIP, "pool", poolName)
return nil
}
// eniRecordToEC2 converts an ENI record to an EC2 NetworkInterface
func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string) *ec2.NetworkInterface {
eni := &ec2.NetworkInterface{
+14
View File
@@ -837,6 +837,20 @@ func (s *VPCServiceImpl) GetDefaultSubnet(accountID string) (*SubnetRecord, erro
return nil, fmt.Errorf("no default subnet found")
}
// GetSubnet looks up a SubnetRecord by ID.
func (s *VPCServiceImpl) GetSubnet(accountID, subnetId string) (*SubnetRecord, error) {
key := utils.AccountKey(accountID, subnetId)
entry, err := s.subnetKV.Get(key)
if err != nil {
return nil, fmt.Errorf("subnet %s not found: %w", subnetId, err)
}
var record SubnetRecord
if err := json.Unmarshal(entry.Value(), &record); err != nil {
return nil, fmt.Errorf("unmarshal subnet record: %w", err)
}
return &record, nil
}
// publishVPCEvent publishes a VPC lifecycle event to NATS for vpcd consumption.
// This is fire-and-forget; errors are logged but do not fail the API response.
func (s *VPCServiceImpl) publishVPCEvent(topic, vpcId, cidrBlock string, vni int64) {
+4
View File
@@ -68,6 +68,10 @@ type VM struct {
ENIId string `json:"eni_id,omitempty"`
ENIMac string `json:"eni_mac,omitempty"`
// Public IP auto-assigned from external IPAM pool (released on termination)
PublicIP string `json:"public_ip,omitempty"`
PublicIPPool string `json:"public_ip_pool,omitempty"`
// DevMAC is the MAC for the dev/hostfwd NIC (DEV_NETWORKING mode).
// Set before cloud-init ISO generation so netplan can suppress its default route.
DevMAC string `json:"dev_mac,omitempty"`