mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-21 19:56:33 +00:00
feat: Phase 1 completion — external IPAM, br-external setup, config template
Add external IPAM module (CAS-based IP allocation from admin-defined pools in NATS KV). Supports multi-pool with region/AZ scoping, pool fallback logic, gateway IP reservation, and concurrent allocation via CAS retries. 12 unit tests covering allocation, release, exhaustion, CAS conflict, multi-pool, fallback, validation, and idempotent init. Add --external-bridge flag to setup-ovn.sh: creates OVS bridge br-external, adds WAN NIC, migrates host IP to bridge, and sets ovn-bridge-mappings=external:br-external for OVN localnet ports. Add commented [network] section and external_interface to spinifex.toml config template for discoverability.
This commit is contained in:
Ben Duncan
@@ -3,6 +3,20 @@ version = "1.0"
|
||||
epoch = 1
|
||||
node = "{{.Node}}"
|
||||
|
||||
# External network for public subnets (disabled by default).
|
||||
# Set external_mode = "pool" and define pools to enable public IPs.
|
||||
# See docs/development/feature/vpc-public-subnet.md for details.
|
||||
#
|
||||
# [network]
|
||||
# external_mode = "pool" # "pool" (public IPs), "nat" (outbound-only), or "" (disabled)
|
||||
#
|
||||
# [[network.external_pools]]
|
||||
# name = "wan"
|
||||
# range_start = "192.168.1.150"
|
||||
# range_end = "192.168.1.250"
|
||||
# gateway = "192.168.1.1"
|
||||
# prefix_len = 24
|
||||
|
||||
[nodes.{{.Node}}]
|
||||
node = "{{.Node}}"
|
||||
host = "{{.BindIP}}"
|
||||
@@ -41,6 +55,7 @@ shardwal = false
|
||||
[nodes.{{.Node}}.vpcd]
|
||||
ovn_nb_addr = "{{.OVNNBAddr}}"
|
||||
ovn_sb_addr = "{{.OVNSBAddr}}"
|
||||
# external_interface = "eth1" # WAN NIC for br-external (public subnets)
|
||||
|
||||
[nodes.{{.Node}}.predastore]
|
||||
host = "{{.BindIP}}:8443"
|
||||
|
||||
+87
-18
@@ -13,15 +13,20 @@
|
||||
# ./scripts/setup-ovn.sh [options]
|
||||
#
|
||||
# Options:
|
||||
# --management Also start OVN central services (NB DB, SB DB, ovn-northd)
|
||||
# --ovn-remote OVN SB DB address (default: tcp:127.0.0.1:6642)
|
||||
# --encap-ip Geneve tunnel endpoint IP (default: auto-detect)
|
||||
# --chassis-id OVN chassis identifier (default: hostname)
|
||||
# --management Also start OVN central services (NB DB, SB DB, ovn-northd)
|
||||
# --external-bridge Create br-external for public subnet WAN uplink
|
||||
# --external-iface WAN NIC to add to br-external (default: eth1)
|
||||
# --ovn-remote OVN SB DB address (default: tcp:127.0.0.1:6642)
|
||||
# --encap-ip Geneve tunnel endpoint IP (default: auto-detect)
|
||||
# --chassis-id OVN chassis identifier (default: hostname)
|
||||
#
|
||||
# Examples:
|
||||
# # Single-node development (management + compute on same host):
|
||||
# ./scripts/setup-ovn.sh --management
|
||||
#
|
||||
# # With external bridge for public subnets:
|
||||
# ./scripts/setup-ovn.sh --management --external-bridge --external-iface=eth1
|
||||
#
|
||||
# # Compute node joining an existing cluster:
|
||||
# ./scripts/setup-ovn.sh --ovn-remote=tcp:10.0.0.1:6642 --encap-ip=10.0.0.2
|
||||
#
|
||||
@@ -34,6 +39,8 @@ set -e
|
||||
|
||||
# Defaults
|
||||
MANAGEMENT=false
|
||||
EXTERNAL_BRIDGE=false
|
||||
EXTERNAL_IFACE="eth1"
|
||||
OVN_REMOTE="tcp:127.0.0.1:6642"
|
||||
ENCAP_IP=""
|
||||
CHASSIS_ID=""
|
||||
@@ -41,12 +48,14 @@ CHASSIS_ID=""
|
||||
# Parse arguments
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--management) MANAGEMENT=true ;;
|
||||
--ovn-remote=*) OVN_REMOTE="${arg#*=}" ;;
|
||||
--encap-ip=*) ENCAP_IP="${arg#*=}" ;;
|
||||
--chassis-id=*) CHASSIS_ID="${arg#*=}" ;;
|
||||
--management) MANAGEMENT=true ;;
|
||||
--external-bridge) EXTERNAL_BRIDGE=true ;;
|
||||
--external-iface=*) EXTERNAL_IFACE="${arg#*=}" ;;
|
||||
--ovn-remote=*) OVN_REMOTE="${arg#*=}" ;;
|
||||
--encap-ip=*) ENCAP_IP="${arg#*=}" ;;
|
||||
--chassis-id=*) CHASSIS_ID="${arg#*=}" ;;
|
||||
--help|-h)
|
||||
head -30 "$0" | tail -28
|
||||
head -34 "$0" | tail -32
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
@@ -72,10 +81,14 @@ if [ -z "$CHASSIS_ID" ]; then
|
||||
fi
|
||||
|
||||
echo "=== Spinifex OVN Compute Node Setup ==="
|
||||
echo " Management node: $MANAGEMENT"
|
||||
echo " OVN Remote (SB): $OVN_REMOTE"
|
||||
echo " Encap IP: $ENCAP_IP"
|
||||
echo " Chassis ID: $CHASSIS_ID"
|
||||
echo " Management node: $MANAGEMENT"
|
||||
echo " External bridge: $EXTERNAL_BRIDGE"
|
||||
if [ "$EXTERNAL_BRIDGE" = true ]; then
|
||||
echo " External iface: $EXTERNAL_IFACE"
|
||||
fi
|
||||
echo " OVN Remote (SB): $OVN_REMOTE"
|
||||
echo " Encap IP: $ENCAP_IP"
|
||||
echo " Chassis ID: $CHASSIS_ID"
|
||||
echo ""
|
||||
|
||||
# --- Step 1: Install packages ---
|
||||
@@ -135,15 +148,61 @@ sudo ovs-vsctl set Bridge br-int other-config:disable-in-band=true
|
||||
sudo ip link set br-int up
|
||||
echo " br-int: created, fail-mode=secure, up"
|
||||
|
||||
# --- Step 3b: Create br-external for WAN uplink (optional) ---
|
||||
if [ "$EXTERNAL_BRIDGE" = true ]; then
|
||||
echo ""
|
||||
echo "Step 3b: Configuring br-external for public subnet WAN uplink..."
|
||||
|
||||
# Verify the WAN NIC exists
|
||||
if ! ip link show "$EXTERNAL_IFACE" >/dev/null 2>&1; then
|
||||
echo " ERROR: interface $EXTERNAL_IFACE does not exist"
|
||||
echo " Available interfaces:"
|
||||
ip -o link show | awk -F': ' '{print " " $2}'
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create br-external and add the WAN NIC
|
||||
sudo ovs-vsctl --may-exist add-br br-external
|
||||
sudo ovs-vsctl --may-exist add-port br-external "$EXTERNAL_IFACE"
|
||||
sudo ip link set br-external up
|
||||
|
||||
# If the WAN NIC has an IP, move it to br-external so the host keeps
|
||||
# connectivity. This is the standard OpenStack/OVN approach for sharing
|
||||
# a NIC between host and OVN localnet traffic.
|
||||
WAN_IP=$(ip -4 addr show dev "$EXTERNAL_IFACE" 2>/dev/null | awk '/inet /{print $2}' | head -1)
|
||||
if [ -n "$WAN_IP" ]; then
|
||||
WAN_GW=$(ip -4 route show default dev "$EXTERNAL_IFACE" 2>/dev/null | awk '{print $3}' | head -1)
|
||||
sudo ip addr del "$WAN_IP" dev "$EXTERNAL_IFACE" 2>/dev/null || true
|
||||
sudo ip addr add "$WAN_IP" dev br-external
|
||||
if [ -n "$WAN_GW" ]; then
|
||||
sudo ip route add default via "$WAN_GW" dev br-external 2>/dev/null || true
|
||||
fi
|
||||
echo " Migrated $WAN_IP from $EXTERNAL_IFACE to br-external"
|
||||
fi
|
||||
|
||||
echo " br-external: created with port $EXTERNAL_IFACE"
|
||||
fi
|
||||
|
||||
# --- Step 4: Configure OVN external_ids ---
|
||||
echo ""
|
||||
echo "Step 4: Setting OVS external_ids for OVN..."
|
||||
|
||||
sudo ovs-vsctl set Open_vSwitch . \
|
||||
external_ids:system-id="$CHASSIS_ID" \
|
||||
external_ids:ovn-remote="$OVN_REMOTE" \
|
||||
external_ids:ovn-encap-ip="$ENCAP_IP" \
|
||||
external_ids:ovn-encap-type="geneve"
|
||||
BRIDGE_MAPPINGS="external:br-external"
|
||||
if [ "$EXTERNAL_BRIDGE" = true ]; then
|
||||
sudo ovs-vsctl set Open_vSwitch . \
|
||||
external_ids:system-id="$CHASSIS_ID" \
|
||||
external_ids:ovn-remote="$OVN_REMOTE" \
|
||||
external_ids:ovn-encap-ip="$ENCAP_IP" \
|
||||
external_ids:ovn-encap-type="geneve" \
|
||||
external_ids:ovn-bridge-mappings="$BRIDGE_MAPPINGS"
|
||||
echo " ovn-bridge-mappings: $BRIDGE_MAPPINGS"
|
||||
else
|
||||
sudo ovs-vsctl set Open_vSwitch . \
|
||||
external_ids:system-id="$CHASSIS_ID" \
|
||||
external_ids:ovn-remote="$OVN_REMOTE" \
|
||||
external_ids:ovn-encap-ip="$ENCAP_IP" \
|
||||
external_ids:ovn-encap-type="geneve"
|
||||
fi
|
||||
|
||||
echo " system-id: $CHASSIS_ID"
|
||||
echo " ovn-remote: $OVN_REMOTE"
|
||||
@@ -311,6 +370,16 @@ else
|
||||
OK=false
|
||||
fi
|
||||
|
||||
# Check br-external (only if --external-bridge was used)
|
||||
if [ "$EXTERNAL_BRIDGE" = true ]; then
|
||||
if sudo ovs-vsctl br-exists br-external; then
|
||||
echo " br-external: OK"
|
||||
else
|
||||
echo " br-external: FAILED"
|
||||
OK=false
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check ovn-controller
|
||||
if sudo ovs-appctl -t ovn-controller version >/dev/null 2>&1 || systemctl is-active --quiet ovn-controller 2>/dev/null; then
|
||||
echo " ovn-controller: OK"
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
package handlers_ec2_vpc
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math/big"
|
||||
"net"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/utils"
|
||||
"github.com/nats-io/nats.go"
|
||||
)
|
||||
|
||||
const (
|
||||
KVBucketExternalIPAM = "spinifex-external-ipam"
|
||||
KVBucketExternalIPAMVersion = 1
|
||||
)
|
||||
|
||||
// ExternalIPAllocation describes how an external IP is being used.
|
||||
type ExternalIPAllocation struct {
|
||||
Type string `json:"type"` // "gateway", "auto_assign", "elastic_ip"
|
||||
AllocationID string `json:"allocation_id,omitempty"` // For elastic IPs
|
||||
Association string `json:"association,omitempty"` // ENI ID for elastic IPs
|
||||
ENIId string `json:"eni_id,omitempty"` // ENI owning this IP
|
||||
InstanceId string `json:"instance_id,omitempty"` // Instance owning this IP
|
||||
Note string `json:"note,omitempty"` // Human-readable note
|
||||
}
|
||||
|
||||
// ExternalIPAMRecord tracks allocated external IPs for a single pool.
|
||||
type ExternalIPAMRecord struct {
|
||||
PoolName string `json:"pool_name"`
|
||||
RangeStart string `json:"range_start"`
|
||||
RangeEnd string `json:"range_end"`
|
||||
Gateway string `json:"gateway"`
|
||||
GatewayIP string `json:"gateway_ip"`
|
||||
PrefixLen int `json:"prefix_len"`
|
||||
Region string `json:"region,omitempty"`
|
||||
AZ string `json:"az,omitempty"`
|
||||
Allocated map[string]ExternalIPAllocation `json:"allocated"`
|
||||
}
|
||||
|
||||
// ExternalPoolConfig is the admin-defined pool from spinifex.toml.
|
||||
type ExternalPoolConfig struct {
|
||||
Name string
|
||||
RangeStart string
|
||||
RangeEnd string
|
||||
Gateway string
|
||||
GatewayIP string
|
||||
PrefixLen int
|
||||
Region string
|
||||
AZ string
|
||||
}
|
||||
|
||||
// ExternalIPAM manages external IP allocation from admin-defined pools using NATS KV with CAS.
|
||||
type ExternalIPAM struct {
|
||||
kv nats.KeyValue
|
||||
pools []ExternalPoolConfig
|
||||
}
|
||||
|
||||
// NewExternalIPAM creates a new ExternalIPAM backed by NATS JetStream KV.
|
||||
func NewExternalIPAM(js nats.JetStreamContext, pools []ExternalPoolConfig) (*ExternalIPAM, error) {
|
||||
kv, err := utils.GetOrCreateKVBucket(js, KVBucketExternalIPAM, 5)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create external IPAM KV bucket: %w", err)
|
||||
}
|
||||
if err := utils.WriteVersion(kv, KVBucketExternalIPAMVersion); err != nil {
|
||||
return nil, fmt.Errorf("write version to %s: %w", KVBucketExternalIPAM, err)
|
||||
}
|
||||
ipam := &ExternalIPAM{kv: kv, pools: pools}
|
||||
if err := ipam.initPools(); err != nil {
|
||||
return nil, fmt.Errorf("init external IPAM pools: %w", err)
|
||||
}
|
||||
return ipam, nil
|
||||
}
|
||||
|
||||
// NewExternalIPAMWithKV creates an ExternalIPAM with an existing KV bucket (for testing).
|
||||
func NewExternalIPAMWithKV(kv nats.KeyValue, pools []ExternalPoolConfig) *ExternalIPAM {
|
||||
return &ExternalIPAM{kv: kv, pools: pools}
|
||||
}
|
||||
|
||||
// initPools ensures each configured pool has a KV record. Idempotent — safe to call
|
||||
// on every vpcd startup. Reserves the gateway IP in each pool.
|
||||
func (m *ExternalIPAM) initPools() error {
|
||||
for _, pool := range m.pools {
|
||||
if err := m.initPool(pool); err != nil {
|
||||
return fmt.Errorf("init pool %q: %w", pool.Name, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ExternalIPAM) initPool(pool ExternalPoolConfig) error {
|
||||
_, _, err := m.getRecord(pool.Name)
|
||||
if err == nil {
|
||||
slog.Debug("external IPAM pool already initialized", "pool", pool.Name)
|
||||
return nil // Already exists
|
||||
}
|
||||
if !errors.Is(err, nats.ErrKeyNotFound) {
|
||||
return err
|
||||
}
|
||||
|
||||
gwIP := pool.GatewayIP
|
||||
if gwIP == "" {
|
||||
gwIP = pool.RangeStart
|
||||
}
|
||||
|
||||
record := &ExternalIPAMRecord{
|
||||
PoolName: pool.Name,
|
||||
RangeStart: pool.RangeStart,
|
||||
RangeEnd: pool.RangeEnd,
|
||||
Gateway: pool.Gateway,
|
||||
GatewayIP: gwIP,
|
||||
PrefixLen: pool.PrefixLen,
|
||||
Region: pool.Region,
|
||||
AZ: pool.AZ,
|
||||
Allocated: map[string]ExternalIPAllocation{
|
||||
gwIP: {Type: "gateway", Note: "OVN router SNAT address"},
|
||||
},
|
||||
}
|
||||
|
||||
data, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal pool record: %w", err)
|
||||
}
|
||||
|
||||
if _, err := m.kv.Create(pool.Name, data); err != nil {
|
||||
// Another instance may have initialized concurrently — that's fine.
|
||||
if errors.Is(err, nats.ErrKeyExists) {
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("create pool KV entry: %w", err)
|
||||
}
|
||||
|
||||
slog.Info("external IPAM pool initialized", "pool", pool.Name, "gateway_ip", gwIP)
|
||||
return nil
|
||||
}
|
||||
|
||||
// AllocateIP allocates the next available external IP from the best pool
|
||||
// matching the given region/AZ. Returns the allocated IP and pool name.
|
||||
func (m *ExternalIPAM) AllocateIP(region, az, allocType, eniID, instanceID string) (string, string, error) {
|
||||
pool := m.findPool(region, az)
|
||||
if pool == nil {
|
||||
return "", "", fmt.Errorf("InsufficientAddressCapacity: no external pool available for region=%q az=%q", region, az)
|
||||
}
|
||||
ip, err := m.allocateFromPool(pool.Name, allocType, eniID, instanceID)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
return ip, pool.Name, nil
|
||||
}
|
||||
|
||||
// AllocateFromPool allocates an IP from a specific named pool.
|
||||
func (m *ExternalIPAM) AllocateFromPool(poolName, allocType, eniID, instanceID string) (string, error) {
|
||||
return m.allocateFromPool(poolName, allocType, eniID, instanceID)
|
||||
}
|
||||
|
||||
func (m *ExternalIPAM) allocateFromPool(poolName, allocType, eniID, instanceID string) (string, error) {
|
||||
for attempt := range 5 {
|
||||
record, revision, err := m.getRecord(poolName)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("get external IPAM record: %w", err)
|
||||
}
|
||||
|
||||
ip, err := nextAvailableExternalIP(record)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
record.Allocated[ip] = ExternalIPAllocation{
|
||||
Type: allocType,
|
||||
ENIId: eniID,
|
||||
InstanceId: instanceID,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("marshal external IPAM record: %w", err)
|
||||
}
|
||||
|
||||
if _, err := m.kv.Update(poolName, data, revision); err != nil {
|
||||
slog.Debug("external IPAM CAS conflict, retrying", "pool", poolName, "attempt", attempt)
|
||||
continue
|
||||
}
|
||||
|
||||
slog.Info("external IPAM allocated IP", "pool", poolName, "ip", ip, "type", allocType)
|
||||
return ip, nil
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("external IPAM allocation failed after CAS retries for pool %s", poolName)
|
||||
}
|
||||
|
||||
// ReleaseIP releases a previously allocated external IP back to its pool.
|
||||
func (m *ExternalIPAM) ReleaseIP(poolName, ip string) error {
|
||||
for attempt := range 5 {
|
||||
record, revision, err := m.getRecord(poolName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("get external IPAM record for release: %w", err)
|
||||
}
|
||||
|
||||
alloc, ok := record.Allocated[ip]
|
||||
if !ok {
|
||||
return fmt.Errorf("IP %s not allocated in pool %s", ip, poolName)
|
||||
}
|
||||
if alloc.Type == "gateway" {
|
||||
return fmt.Errorf("cannot release gateway IP %s in pool %s", ip, poolName)
|
||||
}
|
||||
|
||||
delete(record.Allocated, ip)
|
||||
|
||||
data, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
return fmt.Errorf("marshal external IPAM record: %w", err)
|
||||
}
|
||||
|
||||
if _, err := m.kv.Update(poolName, data, revision); err != nil {
|
||||
slog.Debug("external IPAM release CAS conflict, retrying", "pool", poolName, "attempt", attempt)
|
||||
continue
|
||||
}
|
||||
|
||||
slog.Info("external IPAM released IP", "pool", poolName, "ip", ip)
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("external IPAM release failed after CAS retries for pool %s", poolName)
|
||||
}
|
||||
|
||||
// GetPoolRecord returns the current IPAM record for a pool.
|
||||
func (m *ExternalIPAM) GetPoolRecord(poolName string) (*ExternalIPAMRecord, error) {
|
||||
record, _, err := m.getRecord(poolName)
|
||||
return record, err
|
||||
}
|
||||
|
||||
// findPool returns the best pool for the given region/AZ using the same
|
||||
// fallback order as topology.go: AZ-scoped → region-scoped → unscoped.
|
||||
func (m *ExternalIPAM) findPool(region, az string) *ExternalPoolConfig {
|
||||
// 1. AZ-scoped match
|
||||
for i := range m.pools {
|
||||
p := &m.pools[i]
|
||||
if p.AZ != "" && p.AZ == az && p.Region == region {
|
||||
return p
|
||||
}
|
||||
}
|
||||
// 2. Region-scoped (no AZ)
|
||||
for i := range m.pools {
|
||||
p := &m.pools[i]
|
||||
if p.AZ == "" && p.Region != "" && p.Region == region {
|
||||
return p
|
||||
}
|
||||
}
|
||||
// 3. Unscoped (global pool)
|
||||
for i := range m.pools {
|
||||
p := &m.pools[i]
|
||||
if p.Region == "" && p.AZ == "" {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *ExternalIPAM) getRecord(poolName string) (*ExternalIPAMRecord, uint64, error) {
|
||||
entry, err := m.kv.Get(poolName)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
|
||||
var record ExternalIPAMRecord
|
||||
if err := json.Unmarshal(entry.Value(), &record); err != nil {
|
||||
return nil, 0, fmt.Errorf("unmarshal external IPAM record: %w", err)
|
||||
}
|
||||
|
||||
return &record, entry.Revision(), nil
|
||||
}
|
||||
|
||||
// nextAvailableExternalIP finds the next unallocated IP in the pool's range.
|
||||
func nextAvailableExternalIP(record *ExternalIPAMRecord) (string, error) {
|
||||
startIP := net.ParseIP(record.RangeStart).To4()
|
||||
endIP := net.ParseIP(record.RangeEnd).To4()
|
||||
if startIP == nil || endIP == nil {
|
||||
return "", fmt.Errorf("invalid IP range: %s - %s", record.RangeStart, record.RangeEnd)
|
||||
}
|
||||
|
||||
startInt := ipToInt(startIP)
|
||||
endInt := ipToInt(endIP)
|
||||
|
||||
for i := startInt.Int64(); i <= endInt.Int64(); i++ {
|
||||
candidate := intToIP(intFromInt64(i)).String()
|
||||
if _, taken := record.Allocated[candidate]; !taken {
|
||||
return candidate, nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", fmt.Errorf("InsufficientAddressCapacity: pool %s exhausted", record.PoolName)
|
||||
}
|
||||
|
||||
// ValidatePoolConfig checks that a pool config is valid.
|
||||
func ValidatePoolConfig(pool ExternalPoolConfig) error {
|
||||
if pool.Name == "" {
|
||||
return fmt.Errorf("pool name is required")
|
||||
}
|
||||
startIP := net.ParseIP(pool.RangeStart)
|
||||
if startIP == nil {
|
||||
return fmt.Errorf("invalid range_start: %q", pool.RangeStart)
|
||||
}
|
||||
endIP := net.ParseIP(pool.RangeEnd)
|
||||
if endIP == nil {
|
||||
return fmt.Errorf("invalid range_end: %q", pool.RangeEnd)
|
||||
}
|
||||
if ipToInt(startIP.To4()).Cmp(ipToInt(endIP.To4())) > 0 {
|
||||
return fmt.Errorf("range_start %s is greater than range_end %s", pool.RangeStart, pool.RangeEnd)
|
||||
}
|
||||
if pool.Gateway == "" {
|
||||
return fmt.Errorf("gateway is required for pool %q", pool.Name)
|
||||
}
|
||||
if net.ParseIP(pool.Gateway) == nil {
|
||||
return fmt.Errorf("invalid gateway IP: %q", pool.Gateway)
|
||||
}
|
||||
if pool.GatewayIP != "" && net.ParseIP(pool.GatewayIP) == nil {
|
||||
return fmt.Errorf("invalid gateway_ip: %q", pool.GatewayIP)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// intFromInt64 wraps a plain int64 into a *big.Int-compatible IP conversion.
|
||||
func intFromInt64(v int64) *big.Int {
|
||||
return big.NewInt(v)
|
||||
}
|
||||
@@ -0,0 +1,421 @@
|
||||
package handlers_ec2_vpc
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/nats-io/nats-server/v2/server"
|
||||
"github.com/nats-io/nats.go"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func setupTestExternalIPAM(t *testing.T, pools []ExternalPoolConfig) *ExternalIPAM {
|
||||
t.Helper()
|
||||
opts := &server.Options{
|
||||
Host: "127.0.0.1",
|
||||
Port: -1,
|
||||
JetStream: true,
|
||||
StoreDir: t.TempDir(),
|
||||
NoLog: true,
|
||||
NoSigs: true,
|
||||
}
|
||||
ns, err := server.NewServer(opts)
|
||||
require.NoError(t, err)
|
||||
go ns.Start()
|
||||
require.True(t, ns.ReadyForConnections(5*time.Second))
|
||||
t.Cleanup(func() { ns.Shutdown() })
|
||||
|
||||
nc, err := nats.Connect(ns.ClientURL())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { nc.Close() })
|
||||
|
||||
js, err := nc.JetStream()
|
||||
require.NoError(t, err)
|
||||
|
||||
ipam, err := NewExternalIPAM(js, pools)
|
||||
require.NoError(t, err)
|
||||
return ipam
|
||||
}
|
||||
|
||||
func testPool() ExternalPoolConfig {
|
||||
return ExternalPoolConfig{
|
||||
Name: "wan",
|
||||
RangeStart: "192.168.1.150",
|
||||
RangeEnd: "192.168.1.160",
|
||||
Gateway: "192.168.1.1",
|
||||
PrefixLen: 24,
|
||||
}
|
||||
}
|
||||
|
||||
func TestExternalIPAM_AllocateSequential(t *testing.T) {
|
||||
pool := testPool()
|
||||
ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool})
|
||||
|
||||
// .150 is reserved for gateway, so first allocable is .151
|
||||
ip1, poolName, err := ipam.AllocateIP("", "", "auto_assign", "eni-1", "i-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "192.168.1.151", ip1)
|
||||
assert.Equal(t, "wan", poolName)
|
||||
|
||||
ip2, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-2", "i-2")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "192.168.1.152", ip2)
|
||||
|
||||
ip3, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-3", "i-3")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "192.168.1.153", ip3)
|
||||
}
|
||||
|
||||
func TestExternalIPAM_GatewayIPReserved(t *testing.T) {
|
||||
pool := testPool()
|
||||
ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool})
|
||||
|
||||
record, err := ipam.GetPoolRecord("wan")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Gateway IP (.150) should be reserved
|
||||
alloc, ok := record.Allocated["192.168.1.150"]
|
||||
assert.True(t, ok, "gateway IP should be in allocated map")
|
||||
assert.Equal(t, "gateway", alloc.Type)
|
||||
|
||||
// Cannot release gateway IP
|
||||
err = ipam.ReleaseIP("wan", "192.168.1.150")
|
||||
assert.ErrorContains(t, err, "cannot release gateway IP")
|
||||
}
|
||||
|
||||
func TestExternalIPAM_ExplicitGatewayIP(t *testing.T) {
|
||||
pool := testPool()
|
||||
pool.GatewayIP = "192.168.1.155"
|
||||
ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool})
|
||||
|
||||
record, err := ipam.GetPoolRecord("wan")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Explicit gateway IP (.155) reserved, not .150
|
||||
_, ok := record.Allocated["192.168.1.155"]
|
||||
assert.True(t, ok)
|
||||
_, ok = record.Allocated["192.168.1.150"]
|
||||
assert.False(t, ok)
|
||||
|
||||
// First allocable is .150 since .155 is the gateway
|
||||
ip1, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-1", "i-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "192.168.1.150", ip1)
|
||||
}
|
||||
|
||||
func TestExternalIPAM_Release(t *testing.T) {
|
||||
pool := testPool()
|
||||
ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool})
|
||||
|
||||
ip1, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-1", "i-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "192.168.1.151", ip1)
|
||||
|
||||
ip2, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-2", "i-2")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "192.168.1.152", ip2)
|
||||
|
||||
// Release first
|
||||
err = ipam.ReleaseIP("wan", "192.168.1.151")
|
||||
require.NoError(t, err)
|
||||
|
||||
// Next allocation reuses released IP
|
||||
ip3, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-3", "i-3")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "192.168.1.151", ip3)
|
||||
}
|
||||
|
||||
func TestExternalIPAM_Exhaustion(t *testing.T) {
|
||||
pool := ExternalPoolConfig{
|
||||
Name: "tiny",
|
||||
RangeStart: "10.0.0.1",
|
||||
RangeEnd: "10.0.0.3",
|
||||
Gateway: "10.0.0.254",
|
||||
PrefixLen: 24,
|
||||
}
|
||||
ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool})
|
||||
|
||||
// .1 reserved for gateway, .2 and .3 allocable
|
||||
ip1, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-1", "i-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "10.0.0.2", ip1)
|
||||
|
||||
ip2, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-2", "i-2")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "10.0.0.3", ip2)
|
||||
|
||||
// Pool exhausted
|
||||
_, _, err = ipam.AllocateIP("", "", "auto_assign", "eni-3", "i-3")
|
||||
assert.ErrorContains(t, err, "InsufficientAddressCapacity")
|
||||
}
|
||||
|
||||
func TestExternalIPAM_CASConflict(t *testing.T) {
|
||||
pool := testPool()
|
||||
ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool})
|
||||
|
||||
// Concurrent allocations should all succeed (CAS retry handles conflicts)
|
||||
var wg sync.WaitGroup
|
||||
results := make([]string, 5)
|
||||
errs := make([]error, 5)
|
||||
|
||||
for i := range 5 {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
ip, _, err := ipam.AllocateIP("", "", "auto_assign", "eni-"+itoa(idx), "i-"+itoa(idx))
|
||||
results[idx] = ip
|
||||
errs[idx] = err
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// All should succeed
|
||||
for i, err := range errs {
|
||||
assert.NoError(t, err, "concurrent allocation %d should succeed", i)
|
||||
}
|
||||
|
||||
// All IPs should be unique
|
||||
seen := make(map[string]bool)
|
||||
for _, ip := range results {
|
||||
assert.False(t, seen[ip], "duplicate IP: %s", ip)
|
||||
seen[ip] = true
|
||||
}
|
||||
}
|
||||
|
||||
func TestExternalIPAM_MultiPool(t *testing.T) {
|
||||
pools := []ExternalPoolConfig{
|
||||
{
|
||||
Name: "us-east",
|
||||
RangeStart: "203.0.113.2",
|
||||
RangeEnd: "203.0.113.10",
|
||||
Gateway: "203.0.113.1",
|
||||
PrefixLen: 24,
|
||||
Region: "us-east-1",
|
||||
AZ: "us-east-1a",
|
||||
},
|
||||
{
|
||||
Name: "eu-west",
|
||||
RangeStart: "198.51.100.2",
|
||||
RangeEnd: "198.51.100.10",
|
||||
Gateway: "198.51.100.1",
|
||||
PrefixLen: 24,
|
||||
Region: "eu-west-1",
|
||||
AZ: "eu-west-1a",
|
||||
},
|
||||
}
|
||||
ipam := setupTestExternalIPAM(t, pools)
|
||||
|
||||
// Allocate from US pool
|
||||
ip1, poolName1, err := ipam.AllocateIP("us-east-1", "us-east-1a", "auto_assign", "eni-1", "i-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "203.0.113.3", ip1) // .2 reserved for gateway
|
||||
assert.Equal(t, "us-east", poolName1)
|
||||
|
||||
// Allocate from EU pool
|
||||
ip2, poolName2, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", "auto_assign", "eni-2", "i-2")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "198.51.100.3", ip2)
|
||||
assert.Equal(t, "eu-west", poolName2)
|
||||
}
|
||||
|
||||
func TestExternalIPAM_PoolFallback(t *testing.T) {
|
||||
pools := []ExternalPoolConfig{
|
||||
{
|
||||
Name: "az-pool",
|
||||
RangeStart: "10.0.0.1",
|
||||
RangeEnd: "10.0.0.2",
|
||||
Gateway: "10.0.0.254",
|
||||
PrefixLen: 24,
|
||||
Region: "us-east-1",
|
||||
AZ: "us-east-1a",
|
||||
},
|
||||
{
|
||||
Name: "region-pool",
|
||||
RangeStart: "10.0.1.1",
|
||||
RangeEnd: "10.0.1.10",
|
||||
Gateway: "10.0.1.254",
|
||||
PrefixLen: 24,
|
||||
Region: "us-east-1",
|
||||
},
|
||||
{
|
||||
Name: "global-pool",
|
||||
RangeStart: "10.0.2.1",
|
||||
RangeEnd: "10.0.2.10",
|
||||
Gateway: "10.0.2.254",
|
||||
PrefixLen: 24,
|
||||
},
|
||||
}
|
||||
ipam := setupTestExternalIPAM(t, pools)
|
||||
|
||||
// AZ match: us-east-1a → az-pool
|
||||
ip1, pool1, err := ipam.AllocateIP("us-east-1", "us-east-1a", "auto_assign", "eni-1", "i-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "az-pool", pool1)
|
||||
assert.Equal(t, "10.0.0.2", ip1) // .1 reserved for gw
|
||||
|
||||
// Region match (different AZ): us-east-1b → region-pool
|
||||
ip2, pool2, err := ipam.AllocateIP("us-east-1", "us-east-1b", "auto_assign", "eni-2", "i-2")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "region-pool", pool2)
|
||||
assert.Equal(t, "10.0.1.2", ip2)
|
||||
|
||||
// No match: eu-west-1 → global-pool
|
||||
ip3, pool3, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", "auto_assign", "eni-3", "i-3")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "global-pool", pool3)
|
||||
assert.Equal(t, "10.0.2.2", ip3)
|
||||
}
|
||||
|
||||
func TestExternalIPAM_SpecificPool(t *testing.T) {
|
||||
pools := []ExternalPoolConfig{
|
||||
{
|
||||
Name: "pool-a",
|
||||
RangeStart: "10.0.0.1",
|
||||
RangeEnd: "10.0.0.10",
|
||||
Gateway: "10.0.0.254",
|
||||
PrefixLen: 24,
|
||||
},
|
||||
{
|
||||
Name: "pool-b",
|
||||
RangeStart: "10.0.1.1",
|
||||
RangeEnd: "10.0.1.10",
|
||||
Gateway: "10.0.1.254",
|
||||
PrefixLen: 24,
|
||||
},
|
||||
}
|
||||
ipam := setupTestExternalIPAM(t, pools)
|
||||
|
||||
// Allocate specifically from pool-b
|
||||
ip, err := ipam.AllocateFromPool("pool-b", "elastic_ip", "", "")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "10.0.1.2", ip) // .1 reserved for gw
|
||||
}
|
||||
|
||||
func TestExternalIPAM_RangeValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
pool ExternalPoolConfig
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "empty name",
|
||||
pool: ExternalPoolConfig{RangeStart: "10.0.0.1", RangeEnd: "10.0.0.10", Gateway: "10.0.0.254"},
|
||||
wantErr: "pool name is required",
|
||||
},
|
||||
{
|
||||
name: "bad range_start",
|
||||
pool: ExternalPoolConfig{Name: "bad", RangeStart: "not-an-ip", RangeEnd: "10.0.0.10", Gateway: "10.0.0.254"},
|
||||
wantErr: "invalid range_start",
|
||||
},
|
||||
{
|
||||
name: "bad range_end",
|
||||
pool: ExternalPoolConfig{Name: "bad", RangeStart: "10.0.0.1", RangeEnd: "not-an-ip", Gateway: "10.0.0.254"},
|
||||
wantErr: "invalid range_end",
|
||||
},
|
||||
{
|
||||
name: "start > end",
|
||||
pool: ExternalPoolConfig{Name: "bad", RangeStart: "10.0.0.10", RangeEnd: "10.0.0.1", Gateway: "10.0.0.254"},
|
||||
wantErr: "greater than range_end",
|
||||
},
|
||||
{
|
||||
name: "missing gateway",
|
||||
pool: ExternalPoolConfig{Name: "bad", RangeStart: "10.0.0.1", RangeEnd: "10.0.0.10"},
|
||||
wantErr: "gateway is required",
|
||||
},
|
||||
{
|
||||
name: "bad gateway",
|
||||
pool: ExternalPoolConfig{Name: "bad", RangeStart: "10.0.0.1", RangeEnd: "10.0.0.10", Gateway: "nope"},
|
||||
wantErr: "invalid gateway IP",
|
||||
},
|
||||
{
|
||||
name: "bad gateway_ip",
|
||||
pool: ExternalPoolConfig{Name: "ok", RangeStart: "10.0.0.1", RangeEnd: "10.0.0.10", Gateway: "10.0.0.254", GatewayIP: "nope"},
|
||||
wantErr: "invalid gateway_ip",
|
||||
},
|
||||
{
|
||||
name: "valid",
|
||||
pool: ExternalPoolConfig{Name: "ok", RangeStart: "10.0.0.1", RangeEnd: "10.0.0.10", Gateway: "10.0.0.254"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := ValidatePoolConfig(tt.pool)
|
||||
if tt.wantErr != "" {
|
||||
assert.ErrorContains(t, err, tt.wantErr)
|
||||
} else {
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestExternalIPAM_InitFromConfig(t *testing.T) {
|
||||
pool := testPool()
|
||||
// Create IPAM twice — second init should be idempotent
|
||||
opts := &server.Options{
|
||||
Host: "127.0.0.1",
|
||||
Port: -1,
|
||||
JetStream: true,
|
||||
StoreDir: t.TempDir(),
|
||||
NoLog: true,
|
||||
NoSigs: true,
|
||||
}
|
||||
ns, err := server.NewServer(opts)
|
||||
require.NoError(t, err)
|
||||
go ns.Start()
|
||||
require.True(t, ns.ReadyForConnections(5*time.Second))
|
||||
t.Cleanup(func() { ns.Shutdown() })
|
||||
|
||||
nc, err := nats.Connect(ns.ClientURL())
|
||||
require.NoError(t, err)
|
||||
t.Cleanup(func() { nc.Close() })
|
||||
|
||||
js, err := nc.JetStream()
|
||||
require.NoError(t, err)
|
||||
|
||||
// First init
|
||||
ipam1, err := NewExternalIPAM(js, []ExternalPoolConfig{pool})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Allocate an IP
|
||||
ip, _, err := ipam1.AllocateIP("", "", "auto_assign", "eni-1", "i-1")
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "192.168.1.151", ip)
|
||||
|
||||
// Second init (simulating restart) — should not lose allocation
|
||||
ipam2, err := NewExternalIPAM(js, []ExternalPoolConfig{pool})
|
||||
require.NoError(t, err)
|
||||
|
||||
record, err := ipam2.GetPoolRecord("wan")
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, record.Allocated, "192.168.1.151")
|
||||
assert.Contains(t, record.Allocated, "192.168.1.150") // gateway still reserved
|
||||
}
|
||||
|
||||
func TestExternalIPAM_ReleaseNotAllocated(t *testing.T) {
|
||||
pool := testPool()
|
||||
ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool})
|
||||
|
||||
err := ipam.ReleaseIP("wan", "192.168.1.200")
|
||||
assert.ErrorContains(t, err, "not allocated")
|
||||
}
|
||||
|
||||
func TestExternalIPAM_NoPoolAvailable(t *testing.T) {
|
||||
pool := ExternalPoolConfig{
|
||||
Name: "us-only",
|
||||
RangeStart: "10.0.0.1",
|
||||
RangeEnd: "10.0.0.10",
|
||||
Gateway: "10.0.0.254",
|
||||
PrefixLen: 24,
|
||||
Region: "us-east-1",
|
||||
AZ: "us-east-1a",
|
||||
}
|
||||
ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool})
|
||||
|
||||
// No pool matches eu-west
|
||||
_, _, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", "auto_assign", "eni-1", "i-1")
|
||||
assert.ErrorContains(t, err, "InsufficientAddressCapacity")
|
||||
}
|
||||
Reference in New Issue
Block a user