diff --git a/cmd/spinifex/cmd/templates/spinifex.toml b/cmd/spinifex/cmd/templates/spinifex.toml index 484305878..adf48f259 100644 --- a/cmd/spinifex/cmd/templates/spinifex.toml +++ b/cmd/spinifex/cmd/templates/spinifex.toml @@ -3,6 +3,20 @@ version = "1.0" epoch = 1 node = "{{.Node}}" +# External network for public subnets (disabled by default). +# Set external_mode = "pool" and define pools to enable public IPs. +# See docs/development/feature/vpc-public-subnet.md for details. +# +# [network] +# external_mode = "pool" # "pool" (public IPs), "nat" (outbound-only), or "" (disabled) +# +# [[network.external_pools]] +# name = "wan" +# range_start = "192.168.1.150" +# range_end = "192.168.1.250" +# gateway = "192.168.1.1" +# prefix_len = 24 + [nodes.{{.Node}}] node = "{{.Node}}" host = "{{.BindIP}}" @@ -41,6 +55,7 @@ shardwal = false [nodes.{{.Node}}.vpcd] ovn_nb_addr = "{{.OVNNBAddr}}" ovn_sb_addr = "{{.OVNSBAddr}}" +# external_interface = "eth1" # WAN NIC for br-external (public subnets) [nodes.{{.Node}}.predastore] host = "{{.BindIP}}:8443" diff --git a/scripts/setup-ovn.sh b/scripts/setup-ovn.sh index 1a78ec5c1..5125cb7ca 100755 --- a/scripts/setup-ovn.sh +++ b/scripts/setup-ovn.sh @@ -13,15 +13,20 @@ # ./scripts/setup-ovn.sh [options] # # Options: -# --management Also start OVN central services (NB DB, SB DB, ovn-northd) -# --ovn-remote OVN SB DB address (default: tcp:127.0.0.1:6642) -# --encap-ip Geneve tunnel endpoint IP (default: auto-detect) -# --chassis-id OVN chassis identifier (default: hostname) +# --management Also start OVN central services (NB DB, SB DB, ovn-northd) +# --external-bridge Create br-external for public subnet WAN uplink +# --external-iface WAN NIC to add to br-external (default: eth1) +# --ovn-remote OVN SB DB address (default: tcp:127.0.0.1:6642) +# --encap-ip Geneve tunnel endpoint IP (default: auto-detect) +# --chassis-id OVN chassis identifier (default: hostname) # # Examples: # # Single-node development (management + compute on same host): # ./scripts/setup-ovn.sh --management # +# # With external bridge for public subnets: +# ./scripts/setup-ovn.sh --management --external-bridge --external-iface=eth1 +# # # Compute node joining an existing cluster: # ./scripts/setup-ovn.sh --ovn-remote=tcp:10.0.0.1:6642 --encap-ip=10.0.0.2 # @@ -34,6 +39,8 @@ set -e # Defaults MANAGEMENT=false +EXTERNAL_BRIDGE=false +EXTERNAL_IFACE="eth1" OVN_REMOTE="tcp:127.0.0.1:6642" ENCAP_IP="" CHASSIS_ID="" @@ -41,12 +48,14 @@ CHASSIS_ID="" # Parse arguments for arg in "$@"; do case "$arg" in - --management) MANAGEMENT=true ;; - --ovn-remote=*) OVN_REMOTE="${arg#*=}" ;; - --encap-ip=*) ENCAP_IP="${arg#*=}" ;; - --chassis-id=*) CHASSIS_ID="${arg#*=}" ;; + --management) MANAGEMENT=true ;; + --external-bridge) EXTERNAL_BRIDGE=true ;; + --external-iface=*) EXTERNAL_IFACE="${arg#*=}" ;; + --ovn-remote=*) OVN_REMOTE="${arg#*=}" ;; + --encap-ip=*) ENCAP_IP="${arg#*=}" ;; + --chassis-id=*) CHASSIS_ID="${arg#*=}" ;; --help|-h) - head -30 "$0" | tail -28 + head -34 "$0" | tail -32 exit 0 ;; *) @@ -72,10 +81,14 @@ if [ -z "$CHASSIS_ID" ]; then fi echo "=== Spinifex OVN Compute Node Setup ===" -echo " Management node: $MANAGEMENT" -echo " OVN Remote (SB): $OVN_REMOTE" -echo " Encap IP: $ENCAP_IP" -echo " Chassis ID: $CHASSIS_ID" +echo " Management node: $MANAGEMENT" +echo " External bridge: $EXTERNAL_BRIDGE" +if [ "$EXTERNAL_BRIDGE" = true ]; then +echo " External iface: $EXTERNAL_IFACE" +fi +echo " OVN Remote (SB): $OVN_REMOTE" +echo " Encap IP: $ENCAP_IP" +echo " Chassis ID: $CHASSIS_ID" echo "" # --- Step 1: Install packages --- @@ -135,15 +148,61 @@ sudo ovs-vsctl set Bridge br-int other-config:disable-in-band=true sudo ip link set br-int up echo " br-int: created, fail-mode=secure, up" +# --- Step 3b: Create br-external for WAN uplink (optional) --- +if [ "$EXTERNAL_BRIDGE" = true ]; then + echo "" + echo "Step 3b: Configuring br-external for public subnet WAN uplink..." + + # Verify the WAN NIC exists + if ! ip link show "$EXTERNAL_IFACE" >/dev/null 2>&1; then + echo " ERROR: interface $EXTERNAL_IFACE does not exist" + echo " Available interfaces:" + ip -o link show | awk -F': ' '{print " " $2}' + exit 1 + fi + + # Create br-external and add the WAN NIC + sudo ovs-vsctl --may-exist add-br br-external + sudo ovs-vsctl --may-exist add-port br-external "$EXTERNAL_IFACE" + sudo ip link set br-external up + + # If the WAN NIC has an IP, move it to br-external so the host keeps + # connectivity. This is the standard OpenStack/OVN approach for sharing + # a NIC between host and OVN localnet traffic. + WAN_IP=$(ip -4 addr show dev "$EXTERNAL_IFACE" 2>/dev/null | awk '/inet /{print $2}' | head -1) + if [ -n "$WAN_IP" ]; then + WAN_GW=$(ip -4 route show default dev "$EXTERNAL_IFACE" 2>/dev/null | awk '{print $3}' | head -1) + sudo ip addr del "$WAN_IP" dev "$EXTERNAL_IFACE" 2>/dev/null || true + sudo ip addr add "$WAN_IP" dev br-external + if [ -n "$WAN_GW" ]; then + sudo ip route add default via "$WAN_GW" dev br-external 2>/dev/null || true + fi + echo " Migrated $WAN_IP from $EXTERNAL_IFACE to br-external" + fi + + echo " br-external: created with port $EXTERNAL_IFACE" +fi + # --- Step 4: Configure OVN external_ids --- echo "" echo "Step 4: Setting OVS external_ids for OVN..." -sudo ovs-vsctl set Open_vSwitch . \ - external_ids:system-id="$CHASSIS_ID" \ - external_ids:ovn-remote="$OVN_REMOTE" \ - external_ids:ovn-encap-ip="$ENCAP_IP" \ - external_ids:ovn-encap-type="geneve" +BRIDGE_MAPPINGS="external:br-external" +if [ "$EXTERNAL_BRIDGE" = true ]; then + sudo ovs-vsctl set Open_vSwitch . \ + external_ids:system-id="$CHASSIS_ID" \ + external_ids:ovn-remote="$OVN_REMOTE" \ + external_ids:ovn-encap-ip="$ENCAP_IP" \ + external_ids:ovn-encap-type="geneve" \ + external_ids:ovn-bridge-mappings="$BRIDGE_MAPPINGS" + echo " ovn-bridge-mappings: $BRIDGE_MAPPINGS" +else + sudo ovs-vsctl set Open_vSwitch . \ + external_ids:system-id="$CHASSIS_ID" \ + external_ids:ovn-remote="$OVN_REMOTE" \ + external_ids:ovn-encap-ip="$ENCAP_IP" \ + external_ids:ovn-encap-type="geneve" +fi echo " system-id: $CHASSIS_ID" echo " ovn-remote: $OVN_REMOTE" @@ -311,6 +370,16 @@ else OK=false fi +# Check br-external (only if --external-bridge was used) +if [ "$EXTERNAL_BRIDGE" = true ]; then + if sudo ovs-vsctl br-exists br-external; then + echo " br-external: OK" + else + echo " br-external: FAILED" + OK=false + fi +fi + # Check ovn-controller if sudo ovs-appctl -t ovn-controller version >/dev/null 2>&1 || systemctl is-active --quiet ovn-controller 2>/dev/null; then echo " ovn-controller: OK" diff --git a/spinifex/handlers/ec2/vpc/external_ipam.go b/spinifex/handlers/ec2/vpc/external_ipam.go new file mode 100644 index 000000000..e2bfaa7b7 --- /dev/null +++ b/spinifex/handlers/ec2/vpc/external_ipam.go @@ -0,0 +1,327 @@ +package handlers_ec2_vpc + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "math/big" + "net" + + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" +) + +const ( + KVBucketExternalIPAM = "spinifex-external-ipam" + KVBucketExternalIPAMVersion = 1 +) + +// ExternalIPAllocation describes how an external IP is being used. +type ExternalIPAllocation struct { + Type string `json:"type"` // "gateway", "auto_assign", "elastic_ip" + AllocationID string `json:"allocation_id,omitempty"` // For elastic IPs + Association string `json:"association,omitempty"` // ENI ID for elastic IPs + ENIId string `json:"eni_id,omitempty"` // ENI owning this IP + InstanceId string `json:"instance_id,omitempty"` // Instance owning this IP + Note string `json:"note,omitempty"` // Human-readable note +} + +// ExternalIPAMRecord tracks allocated external IPs for a single pool. +type ExternalIPAMRecord struct { + PoolName string `json:"pool_name"` + RangeStart string `json:"range_start"` + RangeEnd string `json:"range_end"` + Gateway string `json:"gateway"` + GatewayIP string `json:"gateway_ip"` + PrefixLen int `json:"prefix_len"` + Region string `json:"region,omitempty"` + AZ string `json:"az,omitempty"` + Allocated map[string]ExternalIPAllocation `json:"allocated"` +} + +// ExternalPoolConfig is the admin-defined pool from spinifex.toml. +type ExternalPoolConfig struct { + Name string + RangeStart string + RangeEnd string + Gateway string + GatewayIP string + PrefixLen int + Region string + AZ string +} + +// ExternalIPAM manages external IP allocation from admin-defined pools using NATS KV with CAS. +type ExternalIPAM struct { + kv nats.KeyValue + pools []ExternalPoolConfig +} + +// NewExternalIPAM creates a new ExternalIPAM backed by NATS JetStream KV. +func NewExternalIPAM(js nats.JetStreamContext, pools []ExternalPoolConfig) (*ExternalIPAM, error) { + kv, err := utils.GetOrCreateKVBucket(js, KVBucketExternalIPAM, 5) + if err != nil { + return nil, fmt.Errorf("create external IPAM KV bucket: %w", err) + } + if err := utils.WriteVersion(kv, KVBucketExternalIPAMVersion); err != nil { + return nil, fmt.Errorf("write version to %s: %w", KVBucketExternalIPAM, err) + } + ipam := &ExternalIPAM{kv: kv, pools: pools} + if err := ipam.initPools(); err != nil { + return nil, fmt.Errorf("init external IPAM pools: %w", err) + } + return ipam, nil +} + +// NewExternalIPAMWithKV creates an ExternalIPAM with an existing KV bucket (for testing). +func NewExternalIPAMWithKV(kv nats.KeyValue, pools []ExternalPoolConfig) *ExternalIPAM { + return &ExternalIPAM{kv: kv, pools: pools} +} + +// initPools ensures each configured pool has a KV record. Idempotent — safe to call +// on every vpcd startup. Reserves the gateway IP in each pool. +func (m *ExternalIPAM) initPools() error { + for _, pool := range m.pools { + if err := m.initPool(pool); err != nil { + return fmt.Errorf("init pool %q: %w", pool.Name, err) + } + } + return nil +} + +func (m *ExternalIPAM) initPool(pool ExternalPoolConfig) error { + _, _, err := m.getRecord(pool.Name) + if err == nil { + slog.Debug("external IPAM pool already initialized", "pool", pool.Name) + return nil // Already exists + } + if !errors.Is(err, nats.ErrKeyNotFound) { + return err + } + + gwIP := pool.GatewayIP + if gwIP == "" { + gwIP = pool.RangeStart + } + + record := &ExternalIPAMRecord{ + PoolName: pool.Name, + RangeStart: pool.RangeStart, + RangeEnd: pool.RangeEnd, + Gateway: pool.Gateway, + GatewayIP: gwIP, + PrefixLen: pool.PrefixLen, + Region: pool.Region, + AZ: pool.AZ, + Allocated: map[string]ExternalIPAllocation{ + gwIP: {Type: "gateway", Note: "OVN router SNAT address"}, + }, + } + + data, err := json.Marshal(record) + if err != nil { + return fmt.Errorf("marshal pool record: %w", err) + } + + if _, err := m.kv.Create(pool.Name, data); err != nil { + // Another instance may have initialized concurrently — that's fine. + if errors.Is(err, nats.ErrKeyExists) { + return nil + } + return fmt.Errorf("create pool KV entry: %w", err) + } + + slog.Info("external IPAM pool initialized", "pool", pool.Name, "gateway_ip", gwIP) + return nil +} + +// AllocateIP allocates the next available external IP from the best pool +// matching the given region/AZ. Returns the allocated IP and pool name. +func (m *ExternalIPAM) AllocateIP(region, az, allocType, eniID, instanceID string) (string, string, error) { + pool := m.findPool(region, az) + if pool == nil { + return "", "", fmt.Errorf("InsufficientAddressCapacity: no external pool available for region=%q az=%q", region, az) + } + ip, err := m.allocateFromPool(pool.Name, allocType, eniID, instanceID) + if err != nil { + return "", "", err + } + return ip, pool.Name, nil +} + +// AllocateFromPool allocates an IP from a specific named pool. +func (m *ExternalIPAM) AllocateFromPool(poolName, allocType, eniID, instanceID string) (string, error) { + return m.allocateFromPool(poolName, allocType, eniID, instanceID) +} + +func (m *ExternalIPAM) allocateFromPool(poolName, allocType, eniID, instanceID string) (string, error) { + for attempt := range 5 { + record, revision, err := m.getRecord(poolName) + if err != nil { + return "", fmt.Errorf("get external IPAM record: %w", err) + } + + ip, err := nextAvailableExternalIP(record) + if err != nil { + return "", err + } + + record.Allocated[ip] = ExternalIPAllocation{ + Type: allocType, + ENIId: eniID, + InstanceId: instanceID, + } + + data, err := json.Marshal(record) + if err != nil { + return "", fmt.Errorf("marshal external IPAM record: %w", err) + } + + if _, err := m.kv.Update(poolName, data, revision); err != nil { + slog.Debug("external IPAM CAS conflict, retrying", "pool", poolName, "attempt", attempt) + continue + } + + slog.Info("external IPAM allocated IP", "pool", poolName, "ip", ip, "type", allocType) + return ip, nil + } + + return "", fmt.Errorf("external IPAM allocation failed after CAS retries for pool %s", poolName) +} + +// ReleaseIP releases a previously allocated external IP back to its pool. +func (m *ExternalIPAM) ReleaseIP(poolName, ip string) error { + for attempt := range 5 { + record, revision, err := m.getRecord(poolName) + if err != nil { + return fmt.Errorf("get external IPAM record for release: %w", err) + } + + alloc, ok := record.Allocated[ip] + if !ok { + return fmt.Errorf("IP %s not allocated in pool %s", ip, poolName) + } + if alloc.Type == "gateway" { + return fmt.Errorf("cannot release gateway IP %s in pool %s", ip, poolName) + } + + delete(record.Allocated, ip) + + data, err := json.Marshal(record) + if err != nil { + return fmt.Errorf("marshal external IPAM record: %w", err) + } + + if _, err := m.kv.Update(poolName, data, revision); err != nil { + slog.Debug("external IPAM release CAS conflict, retrying", "pool", poolName, "attempt", attempt) + continue + } + + slog.Info("external IPAM released IP", "pool", poolName, "ip", ip) + return nil + } + + return fmt.Errorf("external IPAM release failed after CAS retries for pool %s", poolName) +} + +// GetPoolRecord returns the current IPAM record for a pool. +func (m *ExternalIPAM) GetPoolRecord(poolName string) (*ExternalIPAMRecord, error) { + record, _, err := m.getRecord(poolName) + return record, err +} + +// findPool returns the best pool for the given region/AZ using the same +// fallback order as topology.go: AZ-scoped → region-scoped → unscoped. +func (m *ExternalIPAM) findPool(region, az string) *ExternalPoolConfig { + // 1. AZ-scoped match + for i := range m.pools { + p := &m.pools[i] + if p.AZ != "" && p.AZ == az && p.Region == region { + return p + } + } + // 2. Region-scoped (no AZ) + for i := range m.pools { + p := &m.pools[i] + if p.AZ == "" && p.Region != "" && p.Region == region { + return p + } + } + // 3. Unscoped (global pool) + for i := range m.pools { + p := &m.pools[i] + if p.Region == "" && p.AZ == "" { + return p + } + } + return nil +} + +func (m *ExternalIPAM) getRecord(poolName string) (*ExternalIPAMRecord, uint64, error) { + entry, err := m.kv.Get(poolName) + if err != nil { + return nil, 0, err + } + + var record ExternalIPAMRecord + if err := json.Unmarshal(entry.Value(), &record); err != nil { + return nil, 0, fmt.Errorf("unmarshal external IPAM record: %w", err) + } + + return &record, entry.Revision(), nil +} + +// nextAvailableExternalIP finds the next unallocated IP in the pool's range. +func nextAvailableExternalIP(record *ExternalIPAMRecord) (string, error) { + startIP := net.ParseIP(record.RangeStart).To4() + endIP := net.ParseIP(record.RangeEnd).To4() + if startIP == nil || endIP == nil { + return "", fmt.Errorf("invalid IP range: %s - %s", record.RangeStart, record.RangeEnd) + } + + startInt := ipToInt(startIP) + endInt := ipToInt(endIP) + + for i := startInt.Int64(); i <= endInt.Int64(); i++ { + candidate := intToIP(intFromInt64(i)).String() + if _, taken := record.Allocated[candidate]; !taken { + return candidate, nil + } + } + + return "", fmt.Errorf("InsufficientAddressCapacity: pool %s exhausted", record.PoolName) +} + +// ValidatePoolConfig checks that a pool config is valid. +func ValidatePoolConfig(pool ExternalPoolConfig) error { + if pool.Name == "" { + return fmt.Errorf("pool name is required") + } + startIP := net.ParseIP(pool.RangeStart) + if startIP == nil { + return fmt.Errorf("invalid range_start: %q", pool.RangeStart) + } + endIP := net.ParseIP(pool.RangeEnd) + if endIP == nil { + return fmt.Errorf("invalid range_end: %q", pool.RangeEnd) + } + if ipToInt(startIP.To4()).Cmp(ipToInt(endIP.To4())) > 0 { + return fmt.Errorf("range_start %s is greater than range_end %s", pool.RangeStart, pool.RangeEnd) + } + if pool.Gateway == "" { + return fmt.Errorf("gateway is required for pool %q", pool.Name) + } + if net.ParseIP(pool.Gateway) == nil { + return fmt.Errorf("invalid gateway IP: %q", pool.Gateway) + } + if pool.GatewayIP != "" && net.ParseIP(pool.GatewayIP) == nil { + return fmt.Errorf("invalid gateway_ip: %q", pool.GatewayIP) + } + return nil +} + +// intFromInt64 wraps a plain int64 into a *big.Int-compatible IP conversion. +func intFromInt64(v int64) *big.Int { + return big.NewInt(v) +} diff --git a/spinifex/handlers/ec2/vpc/external_ipam_test.go b/spinifex/handlers/ec2/vpc/external_ipam_test.go new file mode 100644 index 000000000..42d8bb074 --- /dev/null +++ b/spinifex/handlers/ec2/vpc/external_ipam_test.go @@ -0,0 +1,421 @@ +package handlers_ec2_vpc + +import ( + "sync" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupTestExternalIPAM(t *testing.T, pools []ExternalPoolConfig) *ExternalIPAM { + t.Helper() + opts := &server.Options{ + Host: "127.0.0.1", + Port: -1, + JetStream: true, + StoreDir: t.TempDir(), + NoLog: true, + NoSigs: true, + } + ns, err := server.NewServer(opts) + require.NoError(t, err) + go ns.Start() + require.True(t, ns.ReadyForConnections(5*time.Second)) + t.Cleanup(func() { ns.Shutdown() }) + + nc, err := nats.Connect(ns.ClientURL()) + require.NoError(t, err) + t.Cleanup(func() { nc.Close() }) + + js, err := nc.JetStream() + require.NoError(t, err) + + ipam, err := NewExternalIPAM(js, pools) + require.NoError(t, err) + return ipam +} + +func testPool() ExternalPoolConfig { + return ExternalPoolConfig{ + Name: "wan", + RangeStart: "192.168.1.150", + RangeEnd: "192.168.1.160", + Gateway: "192.168.1.1", + PrefixLen: 24, + } +} + +func TestExternalIPAM_AllocateSequential(t *testing.T) { + pool := testPool() + ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) + + // .150 is reserved for gateway, so first allocable is .151 + ip1, poolName, err := ipam.AllocateIP("", "", "auto_assign", "eni-1", "i-1") + require.NoError(t, err) + assert.Equal(t, "192.168.1.151", ip1) + assert.Equal(t, "wan", poolName) + + ip2, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-2", "i-2") + require.NoError(t, err) + assert.Equal(t, "192.168.1.152", ip2) + + ip3, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-3", "i-3") + require.NoError(t, err) + assert.Equal(t, "192.168.1.153", ip3) +} + +func TestExternalIPAM_GatewayIPReserved(t *testing.T) { + pool := testPool() + ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) + + record, err := ipam.GetPoolRecord("wan") + require.NoError(t, err) + + // Gateway IP (.150) should be reserved + alloc, ok := record.Allocated["192.168.1.150"] + assert.True(t, ok, "gateway IP should be in allocated map") + assert.Equal(t, "gateway", alloc.Type) + + // Cannot release gateway IP + err = ipam.ReleaseIP("wan", "192.168.1.150") + assert.ErrorContains(t, err, "cannot release gateway IP") +} + +func TestExternalIPAM_ExplicitGatewayIP(t *testing.T) { + pool := testPool() + pool.GatewayIP = "192.168.1.155" + ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) + + record, err := ipam.GetPoolRecord("wan") + require.NoError(t, err) + + // Explicit gateway IP (.155) reserved, not .150 + _, ok := record.Allocated["192.168.1.155"] + assert.True(t, ok) + _, ok = record.Allocated["192.168.1.150"] + assert.False(t, ok) + + // First allocable is .150 since .155 is the gateway + ip1, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-1", "i-1") + require.NoError(t, err) + assert.Equal(t, "192.168.1.150", ip1) +} + +func TestExternalIPAM_Release(t *testing.T) { + pool := testPool() + ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) + + ip1, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-1", "i-1") + require.NoError(t, err) + assert.Equal(t, "192.168.1.151", ip1) + + ip2, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-2", "i-2") + require.NoError(t, err) + assert.Equal(t, "192.168.1.152", ip2) + + // Release first + err = ipam.ReleaseIP("wan", "192.168.1.151") + require.NoError(t, err) + + // Next allocation reuses released IP + ip3, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-3", "i-3") + require.NoError(t, err) + assert.Equal(t, "192.168.1.151", ip3) +} + +func TestExternalIPAM_Exhaustion(t *testing.T) { + pool := ExternalPoolConfig{ + Name: "tiny", + RangeStart: "10.0.0.1", + RangeEnd: "10.0.0.3", + Gateway: "10.0.0.254", + PrefixLen: 24, + } + ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) + + // .1 reserved for gateway, .2 and .3 allocable + ip1, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-1", "i-1") + require.NoError(t, err) + assert.Equal(t, "10.0.0.2", ip1) + + ip2, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-2", "i-2") + require.NoError(t, err) + assert.Equal(t, "10.0.0.3", ip2) + + // Pool exhausted + _, _, err = ipam.AllocateIP("", "", "auto_assign", "eni-3", "i-3") + assert.ErrorContains(t, err, "InsufficientAddressCapacity") +} + +func TestExternalIPAM_CASConflict(t *testing.T) { + pool := testPool() + ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) + + // Concurrent allocations should all succeed (CAS retry handles conflicts) + var wg sync.WaitGroup + results := make([]string, 5) + errs := make([]error, 5) + + for i := range 5 { + wg.Add(1) + go func(idx int) { + defer wg.Done() + ip, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-"+itoa(idx), "i-"+itoa(idx)) + results[idx] = ip + errs[idx] = err + }(i) + } + wg.Wait() + + // All should succeed + for i, err := range errs { + assert.NoError(t, err, "concurrent allocation %d should succeed", i) + } + + // All IPs should be unique + seen := make(map[string]bool) + for _, ip := range results { + assert.False(t, seen[ip], "duplicate IP: %s", ip) + seen[ip] = true + } +} + +func TestExternalIPAM_MultiPool(t *testing.T) { + pools := []ExternalPoolConfig{ + { + Name: "us-east", + RangeStart: "203.0.113.2", + RangeEnd: "203.0.113.10", + Gateway: "203.0.113.1", + PrefixLen: 24, + Region: "us-east-1", + AZ: "us-east-1a", + }, + { + Name: "eu-west", + RangeStart: "198.51.100.2", + RangeEnd: "198.51.100.10", + Gateway: "198.51.100.1", + PrefixLen: 24, + Region: "eu-west-1", + AZ: "eu-west-1a", + }, + } + ipam := setupTestExternalIPAM(t, pools) + + // Allocate from US pool + ip1, poolName1, err := ipam.AllocateIP("us-east-1", "us-east-1a", "auto_assign", "eni-1", "i-1") + require.NoError(t, err) + assert.Equal(t, "203.0.113.3", ip1) // .2 reserved for gateway + assert.Equal(t, "us-east", poolName1) + + // Allocate from EU pool + ip2, poolName2, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", "auto_assign", "eni-2", "i-2") + require.NoError(t, err) + assert.Equal(t, "198.51.100.3", ip2) + assert.Equal(t, "eu-west", poolName2) +} + +func TestExternalIPAM_PoolFallback(t *testing.T) { + pools := []ExternalPoolConfig{ + { + Name: "az-pool", + RangeStart: "10.0.0.1", + RangeEnd: "10.0.0.2", + Gateway: "10.0.0.254", + PrefixLen: 24, + Region: "us-east-1", + AZ: "us-east-1a", + }, + { + Name: "region-pool", + RangeStart: "10.0.1.1", + RangeEnd: "10.0.1.10", + Gateway: "10.0.1.254", + PrefixLen: 24, + Region: "us-east-1", + }, + { + Name: "global-pool", + RangeStart: "10.0.2.1", + RangeEnd: "10.0.2.10", + Gateway: "10.0.2.254", + PrefixLen: 24, + }, + } + ipam := setupTestExternalIPAM(t, pools) + + // AZ match: us-east-1a → az-pool + ip1, pool1, err := ipam.AllocateIP("us-east-1", "us-east-1a", "auto_assign", "eni-1", "i-1") + require.NoError(t, err) + assert.Equal(t, "az-pool", pool1) + assert.Equal(t, "10.0.0.2", ip1) // .1 reserved for gw + + // Region match (different AZ): us-east-1b → region-pool + ip2, pool2, err := ipam.AllocateIP("us-east-1", "us-east-1b", "auto_assign", "eni-2", "i-2") + require.NoError(t, err) + assert.Equal(t, "region-pool", pool2) + assert.Equal(t, "10.0.1.2", ip2) + + // No match: eu-west-1 → global-pool + ip3, pool3, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", "auto_assign", "eni-3", "i-3") + require.NoError(t, err) + assert.Equal(t, "global-pool", pool3) + assert.Equal(t, "10.0.2.2", ip3) +} + +func TestExternalIPAM_SpecificPool(t *testing.T) { + pools := []ExternalPoolConfig{ + { + Name: "pool-a", + RangeStart: "10.0.0.1", + RangeEnd: "10.0.0.10", + Gateway: "10.0.0.254", + PrefixLen: 24, + }, + { + Name: "pool-b", + RangeStart: "10.0.1.1", + RangeEnd: "10.0.1.10", + Gateway: "10.0.1.254", + PrefixLen: 24, + }, + } + ipam := setupTestExternalIPAM(t, pools) + + // Allocate specifically from pool-b + ip, err := ipam.AllocateFromPool("pool-b", "elastic_ip", "", "") + require.NoError(t, err) + assert.Equal(t, "10.0.1.2", ip) // .1 reserved for gw +} + +func TestExternalIPAM_RangeValidation(t *testing.T) { + tests := []struct { + name string + pool ExternalPoolConfig + wantErr string + }{ + { + name: "empty name", + pool: ExternalPoolConfig{RangeStart: "10.0.0.1", RangeEnd: "10.0.0.10", Gateway: "10.0.0.254"}, + wantErr: "pool name is required", + }, + { + name: "bad range_start", + pool: ExternalPoolConfig{Name: "bad", RangeStart: "not-an-ip", RangeEnd: "10.0.0.10", Gateway: "10.0.0.254"}, + wantErr: "invalid range_start", + }, + { + name: "bad range_end", + pool: ExternalPoolConfig{Name: "bad", RangeStart: "10.0.0.1", RangeEnd: "not-an-ip", Gateway: "10.0.0.254"}, + wantErr: "invalid range_end", + }, + { + name: "start > end", + pool: ExternalPoolConfig{Name: "bad", RangeStart: "10.0.0.10", RangeEnd: "10.0.0.1", Gateway: "10.0.0.254"}, + wantErr: "greater than range_end", + }, + { + name: "missing gateway", + pool: ExternalPoolConfig{Name: "bad", RangeStart: "10.0.0.1", RangeEnd: "10.0.0.10"}, + wantErr: "gateway is required", + }, + { + name: "bad gateway", + pool: ExternalPoolConfig{Name: "bad", RangeStart: "10.0.0.1", RangeEnd: "10.0.0.10", Gateway: "nope"}, + wantErr: "invalid gateway IP", + }, + { + name: "bad gateway_ip", + pool: ExternalPoolConfig{Name: "ok", RangeStart: "10.0.0.1", RangeEnd: "10.0.0.10", Gateway: "10.0.0.254", GatewayIP: "nope"}, + wantErr: "invalid gateway_ip", + }, + { + name: "valid", + pool: ExternalPoolConfig{Name: "ok", RangeStart: "10.0.0.1", RangeEnd: "10.0.0.10", Gateway: "10.0.0.254"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := ValidatePoolConfig(tt.pool) + if tt.wantErr != "" { + assert.ErrorContains(t, err, tt.wantErr) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestExternalIPAM_InitFromConfig(t *testing.T) { + pool := testPool() + // Create IPAM twice — second init should be idempotent + opts := &server.Options{ + Host: "127.0.0.1", + Port: -1, + JetStream: true, + StoreDir: t.TempDir(), + NoLog: true, + NoSigs: true, + } + ns, err := server.NewServer(opts) + require.NoError(t, err) + go ns.Start() + require.True(t, ns.ReadyForConnections(5*time.Second)) + t.Cleanup(func() { ns.Shutdown() }) + + nc, err := nats.Connect(ns.ClientURL()) + require.NoError(t, err) + t.Cleanup(func() { nc.Close() }) + + js, err := nc.JetStream() + require.NoError(t, err) + + // First init + ipam1, err := NewExternalIPAM(js, []ExternalPoolConfig{pool}) + require.NoError(t, err) + + // Allocate an IP + ip, _, err := ipam1.AllocateIP("", "", "auto_assign", "eni-1", "i-1") + require.NoError(t, err) + assert.Equal(t, "192.168.1.151", ip) + + // Second init (simulating restart) — should not lose allocation + ipam2, err := NewExternalIPAM(js, []ExternalPoolConfig{pool}) + require.NoError(t, err) + + record, err := ipam2.GetPoolRecord("wan") + require.NoError(t, err) + assert.Contains(t, record.Allocated, "192.168.1.151") + assert.Contains(t, record.Allocated, "192.168.1.150") // gateway still reserved +} + +func TestExternalIPAM_ReleaseNotAllocated(t *testing.T) { + pool := testPool() + ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) + + err := ipam.ReleaseIP("wan", "192.168.1.200") + assert.ErrorContains(t, err, "not allocated") +} + +func TestExternalIPAM_NoPoolAvailable(t *testing.T) { + pool := ExternalPoolConfig{ + Name: "us-only", + RangeStart: "10.0.0.1", + RangeEnd: "10.0.0.10", + Gateway: "10.0.0.254", + PrefixLen: 24, + Region: "us-east-1", + AZ: "us-east-1a", + } + ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) + + // No pool matches eu-west + _, _, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", "auto_assign", "eni-1", "i-1") + assert.ErrorContains(t, err, "InsufficientAddressCapacity") +}