mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 10:52:04 +00:00
feat: Phase 8c — daemon auto-assign public IP on RunInstances
Wire public IP auto-assignment end-to-end: - RunInstances checks MapPublicIpOnLaunch on subnet, allocates external IP from ExternalIPAM, updates ENI record, publishes vpc.add-nat for dnat_and_snat rule, returns PublicIpAddress in response - TerminateInstances releases public IP back to pool, publishes vpc.delete-nat to remove NAT rule before ENI deletion - DescribeInstances populates PublicIpAddress from VM struct Add ExternalIPAM initialization in daemon startup from cluster NetworkConfig. Add GetSubnet and UpdateENIPublicIP helpers. Add PublicIP/PublicIPPool fields to vm.VM struct. Fix network auto-detect to recognize br-external as WAN interface.
This commit is contained in:
@@ -188,6 +188,10 @@ func SuggestPoolRange(wan *DetectedInterface) (start, end string) {
|
||||
// isVirtualInterface returns true for interfaces that shouldn't be
|
||||
// considered as physical NICs.
|
||||
func isVirtualInterface(name string) bool {
|
||||
// Allow br-external (OVS WAN bridge used for external networking)
|
||||
if name == "br-external" {
|
||||
return false
|
||||
}
|
||||
prefixes := []string{
|
||||
"docker", "br-", "veth", "ovs", "virbr",
|
||||
"tun", "tap", "tailscale", "wg", "lo",
|
||||
|
||||
@@ -105,6 +105,7 @@ type Daemon struct {
|
||||
eigwService *handlers_ec2_eigw.EgressOnlyIGWServiceImpl
|
||||
igwService *handlers_ec2_igw.IGWServiceImpl
|
||||
vpcService *handlers_ec2_vpc.VPCServiceImpl
|
||||
externalIPAM *handlers_ec2_vpc.ExternalIPAM
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
shutdownWg sync.WaitGroup
|
||||
@@ -512,6 +513,34 @@ func (d *Daemon) Start() error {
|
||||
return fmt.Errorf("failed to initialize VPC service: %w", err)
|
||||
}
|
||||
|
||||
// Initialize external IPAM if external networking is configured
|
||||
if d.clusterConfig != nil && d.clusterConfig.Network.ExternalMode != "" {
|
||||
js, jsErr := d.natsConn.JetStream()
|
||||
if jsErr != nil {
|
||||
slog.Warn("Failed to get JetStream for external IPAM", "err", jsErr)
|
||||
} else {
|
||||
var pools []handlers_ec2_vpc.ExternalPoolConfig
|
||||
for _, p := range d.clusterConfig.Network.ExternalPools {
|
||||
pools = append(pools, handlers_ec2_vpc.ExternalPoolConfig{
|
||||
Name: p.Name,
|
||||
RangeStart: p.RangeStart,
|
||||
RangeEnd: p.RangeEnd,
|
||||
Gateway: p.Gateway,
|
||||
GatewayIP: p.GatewayIP,
|
||||
PrefixLen: p.PrefixLen,
|
||||
Region: p.Region,
|
||||
AZ: p.AZ,
|
||||
})
|
||||
}
|
||||
d.externalIPAM, err = handlers_ec2_vpc.NewExternalIPAM(js, pools)
|
||||
if err != nil {
|
||||
slog.Warn("Failed to initialize external IPAM", "err", err)
|
||||
} else {
|
||||
slog.Info("External IPAM initialized", "mode", d.clusterConfig.Network.ExternalMode, "pools", len(pools))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
d.accountService, err = initServiceWithRetry("account settings service", func() (*handlers_ec2_account.AccountSettingsServiceImpl, error) {
|
||||
return handlers_ec2_account.NewAccountSettingsServiceImplWithNATS(d.config, d.natsConn)
|
||||
})
|
||||
@@ -1565,6 +1594,24 @@ func (d *Daemon) stopInstance(instances map[string]*vm.VM, deleteVolume bool) er
|
||||
}
|
||||
}
|
||||
|
||||
// Release public IP before deleting ENI
|
||||
if deleteVolume && instance.PublicIP != "" && instance.PublicIPPool != "" && d.externalIPAM != nil {
|
||||
// Publish vpc.delete-nat to remove dnat_and_snat rule
|
||||
portName := "port-" + instance.ENIId
|
||||
vpcId := ""
|
||||
if instance.Instance != nil && instance.Instance.VpcId != nil {
|
||||
vpcId = *instance.Instance.VpcId
|
||||
}
|
||||
d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, "", portName, "")
|
||||
|
||||
// Release IP back to pool
|
||||
if err := d.externalIPAM.ReleaseIP(instance.PublicIPPool, instance.PublicIP); err != nil {
|
||||
slog.Warn("Failed to release public IP on termination", "ip", instance.PublicIP, "pool", instance.PublicIPPool, "err", err)
|
||||
} else {
|
||||
slog.Info("Released public IP on termination", "ip", instance.PublicIP, "instanceId", instance.ID)
|
||||
}
|
||||
}
|
||||
|
||||
// On termination, delete the auto-created ENI (releases IP back to IPAM,
|
||||
// publishes vpc.delete-port for vpcd). On stop, ENI persists (AWS behavior).
|
||||
if deleteVolume && instance.ENIId != "" && d.vpcService != nil {
|
||||
|
||||
@@ -196,6 +196,41 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
|
||||
"privateIp", *eni.PrivateIpAddress,
|
||||
"mac", instance.ENIMac,
|
||||
)
|
||||
|
||||
// Auto-assign public IP if subnet has MapPublicIpOnLaunch and external IPAM is available
|
||||
if d.externalIPAM != nil {
|
||||
subnet, subErr := d.vpcService.GetSubnet(accountID, *runInstancesInput.SubnetId)
|
||||
if subErr == nil && subnet.MapPublicIpOnLaunch {
|
||||
region := ""
|
||||
az := ""
|
||||
if d.config != nil {
|
||||
region = d.config.Region
|
||||
az = d.config.AZ
|
||||
}
|
||||
publicIP, poolName, allocErr := d.externalIPAM.AllocateIP(region, az, "auto_assign", *eni.NetworkInterfaceId, instance.ID)
|
||||
if allocErr != nil {
|
||||
slog.Warn("Failed to allocate public IP for instance", "instanceId", instance.ID, "err", allocErr)
|
||||
} else {
|
||||
// Update ENI record with public IP
|
||||
if updateErr := d.vpcService.UpdateENIPublicIP(accountID, *eni.NetworkInterfaceId, publicIP, poolName); updateErr != nil {
|
||||
slog.Warn("Failed to update ENI with public IP", "eniId", *eni.NetworkInterfaceId, "err", updateErr)
|
||||
}
|
||||
// Publish vpc.add-nat for dnat_and_snat rule
|
||||
portName := "port-" + *eni.NetworkInterfaceId
|
||||
d.publishNATEvent("vpc.add-nat", *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress)
|
||||
// Set on ec2Instance response
|
||||
ec2Instance.PublicIpAddress = aws.String(publicIP)
|
||||
instance.PublicIP = publicIP
|
||||
instance.PublicIPPool = poolName
|
||||
slog.Info("Auto-assigned public IP",
|
||||
"instanceId", instance.ID,
|
||||
"publicIp", publicIP,
|
||||
"privateIp", *eni.PrivateIpAddress,
|
||||
"pool", poolName,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
instances = append(instances, instance)
|
||||
@@ -628,6 +663,11 @@ func (d *Daemon) handleEC2DescribeInstances(msg *nats.Msg) {
|
||||
instanceCopy := *instance.Instance
|
||||
instanceCopy.State = &ec2.InstanceState{}
|
||||
|
||||
// Populate PublicIpAddress from VM if stored
|
||||
if instance.PublicIP != "" && instanceCopy.PublicIpAddress == nil {
|
||||
instanceCopy.PublicIpAddress = aws.String(instance.PublicIP)
|
||||
}
|
||||
|
||||
// Map internal status to EC2 state codes using the centralized mapping
|
||||
if info, ok := vm.EC2StateCodes[instance.Status]; ok {
|
||||
instanceCopy.State.SetCode(info.Code)
|
||||
@@ -899,6 +939,34 @@ func (d *Daemon) handleEC2TerminateStoppedInstance(msg *nats.Msg) {
|
||||
}
|
||||
instance.EBSRequests.Mu.Unlock()
|
||||
|
||||
// Release public IP before termination
|
||||
if instance.PublicIP != "" && instance.PublicIPPool != "" && d.externalIPAM != nil {
|
||||
portName := "port-" + instance.ENIId
|
||||
vpcId := ""
|
||||
if instance.Instance != nil && instance.Instance.VpcId != nil {
|
||||
vpcId = *instance.Instance.VpcId
|
||||
}
|
||||
d.publishNATEvent("vpc.delete-nat", vpcId, instance.PublicIP, "", portName, "")
|
||||
|
||||
if err := d.externalIPAM.ReleaseIP(instance.PublicIPPool, instance.PublicIP); err != nil {
|
||||
slog.Warn("handleEC2TerminateStoppedInstance: failed to release public IP", "ip", instance.PublicIP, "pool", instance.PublicIPPool, "err", err)
|
||||
} else {
|
||||
slog.Info("handleEC2TerminateStoppedInstance: released public IP", "ip", instance.PublicIP, "instanceId", req.InstanceID)
|
||||
}
|
||||
}
|
||||
|
||||
// Delete ENI if present
|
||||
if instance.ENIId != "" && d.vpcService != nil {
|
||||
_, eniErr := d.vpcService.DeleteNetworkInterface(&ec2.DeleteNetworkInterfaceInput{
|
||||
NetworkInterfaceId: &instance.ENIId,
|
||||
}, instance.AccountID)
|
||||
if eniErr != nil {
|
||||
slog.Error("handleEC2TerminateStoppedInstance: failed to delete ENI", "eni", instance.ENIId, "err", eniErr)
|
||||
} else {
|
||||
slog.Info("handleEC2TerminateStoppedInstance: deleted ENI", "eni", instance.ENIId, "instanceId", req.InstanceID)
|
||||
}
|
||||
}
|
||||
|
||||
// Write to terminated KV bucket FIRST so the instance is visible in DescribeInstances.
|
||||
// If this fails, the instance remains in the stopped bucket (safe to retry).
|
||||
instance.Status = vm.StateTerminated
|
||||
@@ -1115,3 +1183,25 @@ func (d *Daemon) handleEC2ModifyInstanceAttribute(msg *nats.Msg) {
|
||||
slog.Error("Failed to respond to NATS request", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// publishNATEvent publishes a NAT lifecycle event (vpc.add-nat or vpc.delete-nat) to NATS.
|
||||
func (d *Daemon) publishNATEvent(topic, vpcId, externalIP, logicalIP, portName, mac string) {
|
||||
if d.natsConn == nil {
|
||||
return
|
||||
}
|
||||
evt := struct {
|
||||
VpcId string `json:"vpc_id"`
|
||||
ExternalIP string `json:"external_ip"`
|
||||
LogicalIP string `json:"logical_ip"`
|
||||
PortName string `json:"port_name"`
|
||||
MAC string `json:"mac"`
|
||||
}{VpcId: vpcId, ExternalIP: externalIP, LogicalIP: logicalIP, PortName: portName, MAC: mac}
|
||||
data, err := json.Marshal(evt)
|
||||
if err != nil {
|
||||
slog.Error("Failed to marshal NAT event", "topic", topic, "err", err)
|
||||
return
|
||||
}
|
||||
if err := d.natsConn.Publish(topic, data); err != nil {
|
||||
slog.Error("Failed to publish NAT event", "topic", topic, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -325,6 +325,34 @@ func (s *VPCServiceImpl) DetachENI(accountID, eniId string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpdateENIPublicIP updates the PublicIpAddress and PublicIpPool on an ENI record.
|
||||
func (s *VPCServiceImpl) UpdateENIPublicIP(accountID, eniId, publicIP, poolName string) error {
|
||||
key := utils.AccountKey(accountID, eniId)
|
||||
entry, err := s.eniKV.Get(key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ENI %s not found: %w", eniId, err)
|
||||
}
|
||||
|
||||
var record ENIRecord
|
||||
if err := json.Unmarshal(entry.Value(), &record); err != nil {
|
||||
return fmt.Errorf("unmarshal ENI record: %w", err)
|
||||
}
|
||||
|
||||
record.PublicIpAddress = publicIP
|
||||
record.PublicIpPool = poolName
|
||||
|
||||
data, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal ENI record: %w", err)
|
||||
}
|
||||
if _, err := s.eniKV.Update(key, data, entry.Revision()); err != nil {
|
||||
return fmt.Errorf("update ENI record: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("Updated ENI with public IP", "eniId", eniId, "publicIp", publicIP, "pool", poolName)
|
||||
return nil
|
||||
}
|
||||
|
||||
// eniRecordToEC2 converts an ENI record to an EC2 NetworkInterface
|
||||
func (s *VPCServiceImpl) eniRecordToEC2(record *ENIRecord, accountID string) *ec2.NetworkInterface {
|
||||
eni := &ec2.NetworkInterface{
|
||||
|
||||
@@ -837,6 +837,20 @@ func (s *VPCServiceImpl) GetDefaultSubnet(accountID string) (*SubnetRecord, erro
|
||||
return nil, fmt.Errorf("no default subnet found")
|
||||
}
|
||||
|
||||
// GetSubnet looks up a SubnetRecord by ID.
|
||||
func (s *VPCServiceImpl) GetSubnet(accountID, subnetId string) (*SubnetRecord, error) {
|
||||
key := utils.AccountKey(accountID, subnetId)
|
||||
entry, err := s.subnetKV.Get(key)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("subnet %s not found: %w", subnetId, err)
|
||||
}
|
||||
var record SubnetRecord
|
||||
if err := json.Unmarshal(entry.Value(), &record); err != nil {
|
||||
return nil, fmt.Errorf("unmarshal subnet record: %w", err)
|
||||
}
|
||||
return &record, nil
|
||||
}
|
||||
|
||||
// publishVPCEvent publishes a VPC lifecycle event to NATS for vpcd consumption.
|
||||
// This is fire-and-forget; errors are logged but do not fail the API response.
|
||||
func (s *VPCServiceImpl) publishVPCEvent(topic, vpcId, cidrBlock string, vni int64) {
|
||||
|
||||
@@ -68,6 +68,10 @@ type VM struct {
|
||||
ENIId string `json:"eni_id,omitempty"`
|
||||
ENIMac string `json:"eni_mac,omitempty"`
|
||||
|
||||
// Public IP auto-assigned from external IPAM pool (released on termination)
|
||||
PublicIP string `json:"public_ip,omitempty"`
|
||||
PublicIPPool string `json:"public_ip_pool,omitempty"`
|
||||
|
||||
// DevMAC is the MAC for the dev/hostfwd NIC (DEV_NETWORKING mode).
|
||||
// Set before cloud-init ISO generation so netplan can suppress its default route.
|
||||
DevMAC string `json:"dev_mac,omitempty"`
|
||||
|
||||
Reference in New Issue
Block a user