From 93aaf0f9918071fff44860166d1f11a246b97daa Mon Sep 17 00:00:00 2001 From: Ben Duncan <5178243+benduncan@users.noreply.github.com> Date: Mon, 23 Mar 2026 10:59:08 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20single-node=20E2E=20=E2=80=94=20OVN=20SB?= =?UTF-8?q?=20cleanup,=20chassis=20naming,=20macvlan=20NAT,=20DNS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three blockers fixed for VPC public subnet E2E on single-node: 1. OVN SB stale data: reset-dev-env.sh now deletes both NB+SB DB files instead of individual NB cleanup. Eliminates ovn-controller commit failure loop from stale chassis/port binding entries. 2. Chassis name mismatch: reset-dev-env.sh passes --chassis-id=chassis-node1 to setup-ovn.sh, aligning OVS system-id with vpcd's chassis-{nodeName}. 3. Macvlan unicast filtering: switched to centralized NAT (no external_mac on DNAT rules), set nat-addresses=router on localnet ports, and vpcd aligns macvlan MAC with OVN gateway router MAC on startup. Macvlan only delivers unicast matching its own MAC — this ensures inbound traffic from the LAN reaches OVS/OVN. Also fixed: DNS for VMs. admin init auto-detects host DNS servers via resolvectl, stores them in spinifex.toml dns_servers field, and vpcd propagates them to OVN DHCP options. Fallback: 8.8.8.8 + 1.1.1.1. Validated E2E: inbound ping, SSH, outbound ping, DNS, curl, apt update. --- cmd/spinifex/cmd/admin.go | 88 +++++- cmd/spinifex/cmd/service.go | 22 +- cmd/spinifex/cmd/templates/spinifex.toml | 3 + docs/vpc-networking/README.md | 344 +++++++++++++++++++++-- scripts/reset-dev-env.sh | 55 ++-- spinifex/admin/admin.go | 5 +- spinifex/config/config.go | 17 +- spinifex/services/vpcd/topology.go | 51 +++- spinifex/services/vpcd/vpcd.go | 34 +++ 9 files changed, 530 insertions(+), 89 deletions(-) diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 7071b3f29..e342fcc8a 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -26,6 +26,7 @@ import ( "net" "net/http" "os" + "os/exec" "path" "path/filepath" "sort" @@ -625,6 +626,15 @@ func runAdminInit(cmd *cobra.Command, args []string) { os.Exit(1) } + // Detect DNS servers from the host for VM DHCP + var dnsServers []string + if externalMode != "" { + dnsServers = detectDNSServers(externalIface) + if len(dnsServers) > 0 { + fmt.Printf(" DNS servers: %s\n", strings.Join(dnsServers, ", ")) + } + } + // For nat mode, we need the gateway IP. The DHCP flag tells setup-ovn.sh // to obtain one via DHCP on the macvlan/bridge interface. useExternalDHCP := externalMode == "nat" && poolStart == "" @@ -833,8 +843,9 @@ func runAdminInit(cmd *cobra.Command, args []string) { PoolName: "wan", PoolStart: poolStart, PoolEnd: poolEnd, - PoolGateway: externalGateway, - PoolPrefixLen: externalPrefixLen, + PoolGateway: externalGateway, + PoolPrefixLen: externalPrefixLen, + PoolDNSServers: dnsServers, BootstrapAccountId: admin.DefaultAccountID(), BootstrapVpcId: bootstrapVpcId, @@ -1681,3 +1692,76 @@ func writeBootstrapFilesWithAdmin(configDir string, masterKey []byte, accessKey, return handlers_iam.SaveBootstrapData(filepath.Join(configDir, "bootstrap.json"), bd) } + +// detectDNSServers reads the DNS servers configured on the host for the given +// interface. Uses resolvectl (systemd-resolved) first, then falls back to +// /etc/resolv.conf. Returns up to 3 servers. Falls back to public DNS if none found. +func detectDNSServers(iface string) []string { + // Try resolvectl for the specific link first (most reliable on modern systems) + if iface != "" { + out, err := exec.Command("resolvectl", "dns", iface).CombinedOutput() + if err == nil { + servers := parseDNSFromResolvectl(string(out)) + if len(servers) > 0 { + return servers + } + } + } + + // Try resolvectl global + out, err := exec.Command("resolvectl", "dns").CombinedOutput() + if err == nil { + servers := parseDNSFromResolvectl(string(out)) + if len(servers) > 0 { + return servers + } + } + + // Fall back to /etc/resolv.conf + data, err := os.ReadFile("/etc/resolv.conf") + if err == nil { + var servers []string + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "nameserver ") { + ip := strings.TrimSpace(strings.TrimPrefix(line, "nameserver")) + // Skip localhost (systemd-resolved stub) + if ip != "127.0.0.53" && ip != "127.0.0.1" && net.ParseIP(ip) != nil { + servers = append(servers, ip) + } + } + } + if len(servers) > 0 { + if len(servers) > 3 { + servers = servers[:3] + } + return servers + } + } + + // Fallback to well-known public DNS + return []string{"8.8.8.8", "1.1.1.1"} +} + +// parseDNSFromResolvectl extracts IP addresses from resolvectl dns output. +// Format: "Link 2 (enp0s3): 192.168.1.1 8.8.8.8 1.1.1.1" +func parseDNSFromResolvectl(output string) []string { + var servers []string + for _, line := range strings.Split(output, "\n") { + // Find the colon separator, IPs come after it + idx := strings.Index(line, ":") + if idx < 0 { + continue + } + fields := strings.Fields(line[idx+1:]) + for _, f := range fields { + if net.ParseIP(f) != nil && f != "127.0.0.53" && f != "127.0.0.1" { + servers = append(servers, f) + } + } + } + if len(servers) > 3 { + servers = servers[:3] + } + return servers +} diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index 8f96abe45..b546e72c8 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -704,6 +704,7 @@ var vpcdStartCmd = &cobra.Command{ Gateway: p.Gateway, GatewayIP: p.GatewayIP, PrefixLen: p.PrefixLen, + DNSServers: p.DNSServers, Region: p.Region, AZ: p.AZ, }) @@ -728,16 +729,17 @@ var vpcdStartCmd = &cobra.Command{ } svc, err := service.New("vpcd", &vpcd.Config{ - NatsHost: nodeConfig.NATS.Host, - NatsToken: nodeConfig.NATS.ACL.Token, - OVNNBAddr: nodeConfig.VPCD.OVNNBAddr, - OVNSBAddr: nodeConfig.VPCD.OVNSBAddr, - BaseDir: nodeConfig.BaseDir, - Debug: false, - ExternalMode: clusterConfig.Network.ExternalMode, - ExternalPools: extPools, - ChassisNames: chassisNames, - Bootstrap: bootstrap, + NatsHost: nodeConfig.NATS.Host, + NatsToken: nodeConfig.NATS.ACL.Token, + OVNNBAddr: nodeConfig.VPCD.OVNNBAddr, + OVNSBAddr: nodeConfig.VPCD.OVNSBAddr, + BaseDir: nodeConfig.BaseDir, + Debug: false, + ExternalMode: clusterConfig.Network.ExternalMode, + ExternalPools: extPools, + ChassisNames: chassisNames, + Bootstrap: bootstrap, + ExternalInterface: nodeConfig.VPCD.ExternalInterface, }) if err != nil { fmt.Println("Error starting vpcd service:", err) diff --git a/cmd/spinifex/cmd/templates/spinifex.toml b/cmd/spinifex/cmd/templates/spinifex.toml index c09187853..a89db6824 100644 --- a/cmd/spinifex/cmd/templates/spinifex.toml +++ b/cmd/spinifex/cmd/templates/spinifex.toml @@ -21,6 +21,9 @@ gateway = "{{.PoolGateway}}" gateway_ip = "{{.PoolGatewayIP}}" {{- end}} prefix_len = {{if .PoolPrefixLen}}{{.PoolPrefixLen}}{{else}}24{{end}} +{{- if .PoolDNSServers}} +dns_servers = [{{range $i, $s := .PoolDNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}] +{{- end}} {{- end}} {{- if .BootstrapVpcId}} diff --git a/docs/vpc-networking/README.md b/docs/vpc-networking/README.md index 726288bba..653be4470 100644 --- a/docs/vpc-networking/README.md +++ b/docs/vpc-networking/README.md @@ -718,11 +718,15 @@ ovn-nbctl lr-route-add vpc-{vpcId} 0.0.0.0/0 192.168.1.1 ## Per-Instance Public IP ```bash -# 1:1 NAT (distributed — NAT happens on VM's chassis, not gateway) -ovn-nbctl lr-nat-add vpc-{vpcId} dnat_and_snat {public_ip} {private_ip} \ - port-{eniId} {eni_mac} +# 1:1 NAT (centralized — processed on gateway chassis, uses router MAC) +ovn-nbctl lr-nat-add vpc-{vpcId} dnat_and_snat {public_ip} {private_ip} ``` +> **Note:** Spinifex uses centralized NAT (no `external_mac` / `logical_port`) +> because macvlan-based external bridges can only receive unicast traffic matching +> the macvlan's own MAC. The router port MAC is set on the macvlan, and OVN uses +> it for all ARP replies. For single-node, there is no performance difference. + ## Security Group ```bash @@ -822,6 +826,121 @@ aws ec2 describe-instances --instance-ids $INSTANCE \ # Troubleshooting +## Debugging Toolkit + +These commands are used throughout the troubleshooting sections below. Learn +them — they cover 90% of VPC networking issues. + +### OVN Northbound (logical topology) + +```bash +# Full topology overview (routers, switches, ports) +sudo ovn-nbctl show + +# List all VPC routers +sudo ovn-nbctl lr-list + +# List all switches (subnets + external) +sudo ovn-nbctl ls-list + +# NAT rules for a VPC +sudo ovn-nbctl lr-nat-list vpc-{vpcId} + +# Routes for a VPC +sudo ovn-nbctl lr-route-list vpc-{vpcId} + +# Port details (check "up" field for DHCP status) +sudo ovn-nbctl list Logical_Switch_Port port-eni-{eniId} + +# Gateway chassis assignment +sudo ovn-nbctl list Logical_Router_Port gw-vpc-{vpcId} + +# Localnet port options (check nat-addresses=router) +sudo ovn-nbctl get Logical_Switch_Port ext-port-vpc-{vpcId} options +``` + +### OVN Southbound (runtime state) + +```bash +# Chassis list + port bindings (which VM is on which host) +sudo ovn-sbctl show + +# Detailed chassis info (check name matches expectations) +sudo ovn-sbctl list Chassis + +# MAC binding table (shows ARP resolution for external traffic) +sudo ovn-sbctl list MAC_Binding + +# Trace a packet through the OVN pipeline (invaluable for debugging) +sudo ovn-trace ext-vpc-{vpcId} \ + 'inport=="ext-port-vpc-{vpcId}" && eth.dst==ff:ff:ff:ff:ff:ff && \ + arp.op==1 && arp.spa==192.168.1.13 && arp.tpa==192.168.1.201' +``` + +### OVS (datapath / physical wiring) + +```bash +# Full bridge + port topology +sudo ovs-vsctl show + +# Kernel datapath ports and stats +sudo ovs-dpctl show + +# Kernel datapath flow cache (actual forwarding rules) +sudo ovs-dpctl dump-flows + +# OpenFlow rules installed by ovn-controller +sudo ovs-ofctl dump-flows br-int | grep {pattern} + +# Conntrack entries (shows active NAT sessions) +sudo ovs-appctl dpctl/dump-conntrack | grep {ip} + +# FDB (MAC address table) for a bridge +sudo ovs-appctl fdb/show br-external + +# OVS external_ids (system-id, bridge-mappings, encap-ip) +sudo ovs-vsctl get Open_vSwitch . external_ids +``` + +### Network Interfaces + +```bash +# Check macvlan exists and has correct MAC +ip -d link show spx-ext-{nic} + +# Check macvlan is attached to OVS (look for "openvswitch_slave") +ip -d link show spx-ext-{nic} | grep openvswitch + +# Interface traffic stats (RX/TX counts, drops) +ip -s link show spx-ext-{nic} +``` + +### Packet Capture + +```bash +# Capture on the OVS bridge (sees traffic after OVS processing) +sudo tcpdump -i br-external -n -e arp + +# Capture on the macvlan (sees what leaves/enters the wire) +sudo tcpdump -i spx-ext-{nic} -n -e "host {public_ip}" + +# Capture on the physical NIC (sees everything on the wire) +sudo tcpdump -i {nic} -n "host {public_ip}" +``` + +### Service Logs + +```bash +# vpcd log (reconcile, NAT, topology operations) +cat ~/spinifex/logs/vpcd.log + +# ovn-controller log (port binding, commit failures) +sudo cat /var/log/ovn/ovn-controller.log | tail -50 + +# Daemon log (instance launch, network setup) +cat ~/spinifex/logs/spinifex.log +``` + ## VPC creation fails Check OVN services and vpcd daemon: @@ -873,25 +992,212 @@ ip route show ## Public IP not reachable from WAN -1. Verify br-external: - ```bash - sudo ovs-vsctl show | grep -A5 br-external - ``` +Work through these checks in order — each eliminates a class of issues. -2. Verify bridge mappings: - ```bash - sudo ovs-vsctl get Open_vSwitch . external-ids:ovn-bridge-mappings - ``` +### 1. Verify OVS wiring -3. Check gateway chassis: - ```bash - sudo ovn-nbctl lrp-get-gateway-chassis gw-$VPC - ``` +```bash +# br-external must exist with the macvlan port +sudo ovs-vsctl show | grep -A5 br-external -4. Verify `dnat_and_snat` rule: - ```bash - sudo ovn-nbctl lr-nat-list vpc-$VPC | grep dnat_and_snat - ``` +# Bridge mappings must be set +sudo ovs-vsctl get Open_vSwitch . external-ids:ovn-bridge-mappings +# Expected: "external:br-external" +``` + +### 2. Verify chassis and gateway scheduling + +```bash +# What OVS thinks the chassis name is +sudo ovs-vsctl get Open_vSwitch . external-ids:system-id + +# What OVN SB registered (must match the system-id above) +sudo ovn-sbctl show +# Look for: Chassis {name} + +# What vpcd scheduled as gateway chassis (must match SB chassis name) +sudo ovn-nbctl list Logical_Router_Port gw-vpc-{vpcId} | grep gateway_chassis +``` + +If these don't match, see "Chassis name mismatch" below. + +### 3. Verify NAT rule + +```bash +sudo ovn-nbctl lr-nat-list vpc-$VPC | grep dnat_and_snat +# Must show the public IP → private IP mapping +# Must NOT have external_mac or logical_port set (centralized NAT) +``` + +### 4. Verify ARP resolution + +From another host on the same LAN, check if OVN responds to ARP: + +```bash +# On the remote host: +ping -c 1 {public_ip} +ip neigh show {public_ip} +# Should show the OVN router MAC (02:00:00:xx:xx:xx) +``` + +If ARP fails, check the macvlan MAC alignment: + +```bash +# On the Spinifex host: +# Gateway router MAC +sudo ovn-nbctl get Logical_Router_Port gw-vpc-{vpcId} mac +# Macvlan MAC +ip link show spx-ext-{nic} | grep ether + +# These MUST match. If they don't, vpcd failed to set the macvlan MAC. +# Check vpcd.log for "macvlan MAC aligned" or errors. +``` + +### 5. Verify packet flow with tcpdump + +Capture at each layer to find where packets stop: + +```bash +# Layer 1: Does the ARP/ICMP arrive on the physical NIC? +sudo tcpdump -i {nic} -n "host {public_ip}" + +# Layer 2: Does it reach br-external? +sudo tcpdump -i br-external -n "host {public_ip}" + +# Layer 3: Does it reach the macvlan output? +sudo tcpdump -i spx-ext-{nic} -n -e "host {public_ip}" +``` + +If traffic arrives on the physical NIC but not br-external, the macvlan is +filtering it. Check MAC alignment (step 4). + +### 6. Use ovn-trace for pipeline debugging + +Simulate a packet through the entire OVN pipeline: + +```bash +sudo ovn-trace --ct=new ext-vpc-{vpcId} \ + 'inport=="ext-port-vpc-{vpcId}" && eth.dst==ff:ff:ff:ff:ff:ff && \ + arp.op==1 && arp.sha=={remote_mac} && arp.spa=={remote_ip} && \ + arp.tpa=={public_ip}' +``` + +The output shows every table the packet passes through and what action is +taken. Look for `drop` actions or unexpected paths. + +## OVN SB commit failure loop + +**Symptom:** ovn-controller log shows: + +``` +OVNSB commit failed, force recompute next time. +``` + +Repeated millions of times. Port binding never happens (`up: false`). + +**Cause:** Stale entries in the OVN Southbound DB (old chassis records, port +bindings, datapath bindings) conflict with ovn-controller's expected state. +Happens when `reset-dev-env.sh` cleans NB but not SB, or after ungraceful +shutdowns. + +**Fix:** Delete both OVN DB files and restart. `reset-dev-env.sh` does this +automatically: + +```bash +sudo systemctl stop ovn-central ovn-controller +sudo rm -f /var/lib/ovn/ovnnb_db.db /var/lib/ovn/ovnsb_db.db +sudo systemctl start ovn-central ovn-controller +# vpcd reconcile will recreate the NB topology on next startup +``` + +## Chassis name mismatch + +**Symptom:** Gateway chassis is set in NB but the SB has no matching chassis. +`ovn-nbctl show` reports a gateway chassis name that doesn't appear in +`ovn-sbctl show`. Gateway traffic (SNAT/DNAT) doesn't work. + +**Cause:** Three names must align: +1. OVS `external_ids:system-id` → what ovn-controller registers as +2. OVN SB `Chassis.name` → what actually exists in the SB DB +3. vpcd's chassis name → `chassis-{nodeName}` from config + +If `setup-ovn.sh` used `chassis-$(hostname -s)` but vpcd uses +`chassis-{nodeName}`, they'll differ on any host where hostname ≠ node name. + +**Diagnosis:** + +```bash +# What OVS has +sudo ovs-vsctl get Open_vSwitch . external-ids:system-id +# What SB registered +sudo ovn-sbctl list Chassis | grep name +# What vpcd expects +grep node ~/spinifex/config/spinifex.toml | head -1 +``` + +**Fix:** `reset-dev-env.sh` passes `--chassis-id=chassis-node1` to +`setup-ovn.sh`. For manual fix: + +```bash +sudo ovs-vsctl set Open_vSwitch . external_ids:system-id=chassis-node1 +sudo systemctl restart ovn-controller +``` + +## Macvlan unicast filtering (inbound traffic lost) + +**Symptom:** VM gets DHCP, outbound works (ping 8.8.8.8 from VM), but inbound +from the LAN to the VM's public IP fails. ARP replies go out but ICMP never +arrives at br-external. + +**Cause:** macvlan in bridge mode only delivers unicast frames matching its own +MAC. OVN announces the router MAC for external IPs, but the macvlan has a +different (random) MAC. Broadcasts (ARP requests) pass through; unicast return +traffic is filtered by the kernel macvlan subsystem. + +**Diagnosis:** + +```bash +# Compare these two — they must match +sudo ovn-nbctl get Logical_Router_Port gw-vpc-{vpcId} mac +ip link show spx-ext-{nic} | grep ether + +# If they differ, inbound unicast is being dropped +``` + +**Fix:** vpcd automatically aligns the macvlan MAC on startup. If it failed: + +```bash +# Get the gateway MAC from OVN +GW_MAC=$(sudo ovn-nbctl get Logical_Router_Port gw-vpc-{vpcId} mac | tr -d '"') + +# Set it on the macvlan +sudo ip link set spx-ext-{nic} down +sudo ip link set spx-ext-{nic} address $GW_MAC +sudo ip link set spx-ext-{nic} up +``` + +Also ensure the DNAT rule uses centralized NAT (no `external_mac`): + +```bash +sudo ovn-nbctl lr-nat-list vpc-{vpcId} +# The dnat_and_snat row should have EMPTY external_mac and logical_port columns +``` + +## Stale ARP on remote hosts + +**Symptom:** Ping from a LAN host to a VM public IP fails after a reset, but +worked before. The remote host has a stale ARP entry with the old MAC. + +**Fix:** Flush the ARP entry on the remote host: + +```bash +# On the remote host: +sudo ip neigh flush dev {nic} {public_ip} +ping {public_ip} # should work now +``` + +OVN's `nat-addresses=router` option sends periodic gratuitous ARPs that will +eventually update remote ARP caches, but flushing is faster for testing. ## Security group rules not taking effect diff --git a/scripts/reset-dev-env.sh b/scripts/reset-dev-env.sh index 5208e2b0f..483052bfc 100755 --- a/scripts/reset-dev-env.sh +++ b/scripts/reset-dev-env.sh @@ -86,37 +86,19 @@ if command -v ovs-vsctl >/dev/null 2>&1; then sudo systemctl stop openvswitch-switch 2>/dev/null || true fi -# Clean OVN Northbound database (remove all logical routers, switches, NAT rules, etc.) -echo "Cleaning OVN Northbound database" -if command -v ovn-nbctl >/dev/null 2>&1; then - sudo systemctl start ovn-central 2>/dev/null || true - # Wait for NB DB - for i in $(seq 1 10); do - if sudo ovn-nbctl --timeout=2 show >/dev/null 2>&1; then break; fi - sleep 1 - done - - # Delete all logical routers (and their ports, NAT rules, routes) - for lr in $(sudo ovn-nbctl --no-headings --columns=name find Logical_Router 2>/dev/null | awk -F'"' '{print $2}'); do - [ -n "$lr" ] && sudo ovn-nbctl lr-del "$lr" 2>/dev/null && echo " Deleted router: $lr" - done - # Delete all logical switches (and their ports) - for ls in $(sudo ovn-nbctl --no-headings --columns=name find Logical_Switch 2>/dev/null | awk -F'"' '{print $2}'); do - [ -n "$ls" ] && sudo ovn-nbctl ls-del "$ls" 2>/dev/null && echo " Deleted switch: $ls" - done - # Delete all port groups - for pg in $(sudo ovn-nbctl --no-headings --columns=name find Port_Group 2>/dev/null | awk -F'"' '{print $2}'); do - [ -n "$pg" ] && sudo ovn-nbctl pg-del "$pg" 2>/dev/null && echo " Deleted port group: $pg" - done - # Delete all DHCP options - for uuid in $(sudo ovn-nbctl --no-headings --columns=_uuid find DHCP_Options 2>/dev/null | awk '{print $NF}'); do - [ -n "$uuid" ] && sudo ovn-nbctl dhcp-options-del "$uuid" 2>/dev/null - done - - sudo systemctl stop ovn-central 2>/dev/null || true - echo " OVN NB database cleaned" +# Clean OVN databases (both Northbound and Southbound) +# Delete the DB files outright — setup-ovn.sh will restart ovn-central with fresh +# empty databases. This eliminates stale SB state (chassis entries, port bindings, +# datapath bindings) that accumulates across resets and causes ovn-controller to +# enter a commit failure loop ("OVNSB commit failed, force recompute"). +echo "Removing OVN database files" +sudo systemctl stop ovn-central 2>/dev/null || true +sudo systemctl stop ovn-controller 2>/dev/null || true +if [ -d /var/lib/ovn ]; then + sudo rm -f /var/lib/ovn/ovnnb_db.db /var/lib/ovn/ovnsb_db.db + echo " Deleted /var/lib/ovn/ovn{nb,sb}_db.db" else - echo " ovn-nbctl not found, skipping OVN cleanup" + echo " /var/lib/ovn not found, skipping OVN DB cleanup" fi # Remove macvlan interfaces created by setup-ovn.sh @@ -138,13 +120,20 @@ WAN_GW=$(ip -4 route show default | awk '{print $3}' | head -1) echo "Detected WAN interface: ${WAN_IFACE:-none}, gateway: ${WAN_GW:-none}" -echo "Re-initializing OVN" +# Chassis ID must match what vpcd derives from the node name in spinifex.toml. +# vpcd generates "chassis-{nodeName}" (e.g., chassis-node1) and uses it to schedule +# gateway chassis in the NB DB. ovn-controller registers in the SB DB using the OVS +# system-id. If they don't match, gateway traffic has no host to schedule to. +NODE_NAME="node1" +CHASSIS_ID="chassis-${NODE_NAME}" + +echo "Re-initializing OVN (chassis-id: $CHASSIS_ID)" if [ -n "$WAN_IFACE" ]; then echo " Using macvlan on $WAN_IFACE for br-external" - ./scripts/setup-ovn.sh --management --external-bridge --external-iface="$WAN_IFACE" + ./scripts/setup-ovn.sh --management --external-bridge --external-iface="$WAN_IFACE" --chassis-id="$CHASSIS_ID" else echo " No WAN interface detected, management-only" - ./scripts/setup-ovn.sh --management + ./scripts/setup-ovn.sh --management --chassis-id="$CHASSIS_ID" fi echo "Initializing platform" diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index 684ef7545..c51199d4f 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -72,8 +72,9 @@ type ConfigSettings struct { PoolStart string // First IP in external pool range PoolEnd string // Last IP in external pool range PoolGateway string // WAN gateway IP - PoolGatewayIP string // Explicit SNAT IP (for nat mode without DHCP) - PoolPrefixLen int // Subnet prefix length (default 24) + PoolGatewayIP string // Explicit SNAT IP (for nat mode without DHCP) + PoolPrefixLen int // Subnet prefix length (default 24) + PoolDNSServers []string // DNS servers for VM DHCP (auto-detected from host) // Other nodes in the cluster (for config source of truth) RemoteNodes []RemoteNode diff --git a/spinifex/config/config.go b/spinifex/config/config.go index 1e498734c..22fbf4d34 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -20,14 +20,15 @@ type ClusterConfig struct { // ExternalPool defines a range of routable IPs that Spinifex manages for public subnets. type ExternalPool struct { - Name string `mapstructure:"name"` // Pool identifier (e.g., "wan", "dc1-primary") - RangeStart string `mapstructure:"range_start"` // First IP in range - RangeEnd string `mapstructure:"range_end"` // Last IP in range - Gateway string `mapstructure:"gateway"` // WAN default gateway (next hop for 0.0.0.0/0) - GatewayIP string `mapstructure:"gateway_ip"` // OVN router external IP (override; defaults to first IP in range) - PrefixLen int `mapstructure:"prefix_len"` // Subnet mask (default 24) - Region string `mapstructure:"region"` // Scope to region (optional — empty means any region) - AZ string `mapstructure:"az"` // Scope to AZ (optional — empty means any AZ in region) + Name string `mapstructure:"name"` // Pool identifier (e.g., "wan", "dc1-primary") + RangeStart string `mapstructure:"range_start"` // First IP in range + RangeEnd string `mapstructure:"range_end"` // Last IP in range + Gateway string `mapstructure:"gateway"` // WAN default gateway (next hop for 0.0.0.0/0) + GatewayIP string `mapstructure:"gateway_ip"` // OVN router external IP (override; defaults to first IP in range) + PrefixLen int `mapstructure:"prefix_len"` // Subnet mask (default 24) + DNSServers []string `mapstructure:"dns_servers"` // DNS servers for VM DHCP (auto-detected from host; fallback: 8.8.8.8, 1.1.1.1) + Region string `mapstructure:"region"` // Scope to region (optional — empty means any region) + AZ string `mapstructure:"az"` // Scope to AZ (optional — empty means any AZ in region) } // NetworkConfig holds cluster-wide external network settings. diff --git a/spinifex/services/vpcd/topology.go b/spinifex/services/vpcd/topology.go index 9d14be3cc..5dff86f3c 100644 --- a/spinifex/services/vpcd/topology.go +++ b/spinifex/services/vpcd/topology.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "net" + "strings" "github.com/mulgadc/spinifex/spinifex/services/vpcd/nbdb" "github.com/mulgadc/spinifex/spinifex/types" @@ -94,6 +95,19 @@ func WithChassisNames(names []string) TopologyOption { } } +// dnsServer returns the DNS server string for OVN DHCP options. +// Uses dns_servers from the external pool config (auto-detected by admin init). +// Falls back to 8.8.8.8 and 1.1.1.1 if none configured. +// OVN DHCP expects the format "{ip1, ip2}" for multiple servers. +func (h *TopologyHandler) dnsServer() string { + pool := h.findExternalPool("", "") + if pool != nil && len(pool.DNSServers) > 0 { + return "{" + strings.Join(pool.DNSServers, ", ") + "}" + } + // Fallback: public DNS + return "{8.8.8.8, 1.1.1.1}" +} + // findExternalPool returns the first pool matching the given region/AZ, // using the fallback order: AZ-scoped → region-scoped → unscoped. func (h *TopologyHandler) findExternalPool(region, az string) *ExternalPoolConfig { @@ -373,7 +387,7 @@ func (h *TopologyHandler) handleSubnetCreate(msg *nats.Msg) { "server_mac": routerMAC, "lease_time": "3600", "router": gwIP, - "dns_server": "169.254.169.253", + "dns_server": h.dnsServer(), "mtu": "1442", // Geneve overhead }, ExternalIDs: map[string]string{ @@ -581,12 +595,16 @@ func (h *TopologyHandler) handleIGWAttach(msg *nats.Msg) { } // 2. Create localnet port on external switch (maps to physical network) + // nat-addresses=router: OVN sends gratuitous ARPs for all NAT external IPs + // using the router port MAC. Required for macvlan-based external bridges + // where the macvlan MAC matches the router MAC. localnetPort := &nbdb.LogicalSwitchPort{ Name: extPortName, Type: "localnet", Addresses: []string{"unknown"}, Options: map[string]string{ - "network_name": "external", + "network_name": "external", + "nat-addresses": "router", }, ExternalIDs: map[string]string{ "spinifex:vpc_id": evt.VpcId, @@ -825,15 +843,15 @@ func (h *TopologyHandler) handleAddNAT(msg *nats.Msg) { ctx := context.Background() routerName := "vpc-" + evt.VpcId - logicalPort := evt.PortName - externalMAC := evt.MAC - + // Use centralized NAT (no ExternalMAC/LogicalPort). With macvlan-based + // external bridges, OVN must use the router port MAC for all ARP replies + // so the macvlan can receive inbound unicast traffic. Distributed NAT + // (ExternalMAC/LogicalPort) announces the VM's MAC which the macvlan + // filters out. For single-node, centralized NAT has zero overhead. natRule := &nbdb.NAT{ - Type: "dnat_and_snat", - ExternalIP: evt.ExternalIP, - LogicalIP: evt.LogicalIP, - LogicalPort: &logicalPort, - ExternalMAC: &externalMAC, + Type: "dnat_and_snat", + ExternalIP: evt.ExternalIP, + LogicalIP: evt.LogicalIP, ExternalIDs: map[string]string{ "spinifex:vpc_id": evt.VpcId, "spinifex:public_ip": evt.ExternalIP, @@ -971,7 +989,7 @@ func (h *TopologyHandler) reconcileSubnet(ctx context.Context, subnetId, vpcId, "server_mac": routerMAC, "lease_time": "3600", "router": gwIP, - "dns_server": "169.254.169.253", + "dns_server": h.dnsServer(), "mtu": "1442", }, ExternalIDs: map[string]string{ @@ -1042,10 +1060,13 @@ func (h *TopologyHandler) reconcileIGW(ctx context.Context, vpcId, igwId string) portExtIDs["spinifex:igw_id"] = igwId } localnetPort := &nbdb.LogicalSwitchPort{ - Name: extPortName, - Type: "localnet", - Addresses: []string{"unknown"}, - Options: map[string]string{"network_name": "external"}, + Name: extPortName, + Type: "localnet", + Addresses: []string{"unknown"}, + Options: map[string]string{ + "network_name": "external", + "nat-addresses": "router", + }, ExternalIDs: portExtIDs, } if err := h.ovn.CreateLogicalSwitchPort(ctx, extSwitchName, localnetPort); err != nil { diff --git a/spinifex/services/vpcd/vpcd.go b/spinifex/services/vpcd/vpcd.go index 02159d604..362b0f35f 100644 --- a/spinifex/services/vpcd/vpcd.go +++ b/spinifex/services/vpcd/vpcd.go @@ -60,6 +60,9 @@ type Config struct { ChassisNames []string // Bootstrap holds the default VPC config from spinifex.toml for first-boot reconciliation. Bootstrap *BootstrapVPC + // ExternalInterface is the WAN NIC name (e.g., "enp0s3"). Used to align + // the macvlan MAC with the OVN gateway MAC for inbound traffic. + ExternalInterface string } // ExternalPoolConfig mirrors config.ExternalPool for vpcd's internal use. @@ -70,6 +73,7 @@ type ExternalPoolConfig struct { Gateway string GatewayIP string PrefixLen int + DNSServers []string Region string AZ string } @@ -230,6 +234,20 @@ func launchService(cfg *Config) error { // Run reconciliation before subscribing Reconcile(ctx, topo, cfg.Bootstrap) + // Align macvlan MAC with the OVN gateway router MAC. The macvlan only + // delivers inbound unicast matching its own MAC. With centralized NAT, + // OVN uses the router MAC for all external ARP replies. Setting the + // macvlan MAC to the router MAC ensures inbound traffic reaches OVS. + if cfg.Bootstrap != nil && cfg.Bootstrap.VpcId != "" && cfg.ExternalInterface != "" { + gwMAC := generateMAC("gw-" + cfg.Bootstrap.VpcId) + macvlanName := "spx-ext-" + cfg.ExternalInterface + if err := setMacvlanMAC(macvlanName, gwMAC); err != nil { + slog.Warn("vpcd: failed to set macvlan MAC (inbound traffic may not work)", "iface", macvlanName, "mac", gwMAC, "err", err) + } else { + slog.Info("vpcd: macvlan MAC aligned with gateway", "iface", macvlanName, "mac", gwMAC) + } + } + subs, err := topo.Subscribe(nc) if err != nil { slog.Error("Failed to subscribe to VPC topics", "err", err) @@ -251,3 +269,19 @@ func launchService(cfg *Config) error { slog.Info("vpcd service shutting down") return nil } + +// setMacvlanMAC sets the MAC address on a macvlan interface. The interface is +// brought down, MAC changed, and brought back up. Requires sudo/NET_ADMIN. +func setMacvlanMAC(iface, mac string) error { + cmds := [][]string{ + {"sudo", "ip", "link", "set", iface, "down"}, + {"sudo", "ip", "link", "set", iface, "address", mac}, + {"sudo", "ip", "link", "set", iface, "up"}, + } + for _, args := range cmds { + if out, err := exec.Command(args[0], args[1:]...).CombinedOutput(); err != nil { + return fmt.Errorf("%s: %w (%s)", args[3], err, string(out)) + } + } + return nil +}