mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
fix: single-node E2E — OVN SB cleanup, chassis naming, macvlan NAT, DNS
Three blockers fixed for VPC public subnet E2E on single-node:
1. OVN SB stale data: reset-dev-env.sh now deletes both NB+SB DB files
instead of individual NB cleanup. Eliminates ovn-controller commit
failure loop from stale chassis/port binding entries.
2. Chassis name mismatch: reset-dev-env.sh passes --chassis-id=chassis-node1
to setup-ovn.sh, aligning OVS system-id with vpcd's chassis-{nodeName}.
3. Macvlan unicast filtering: switched to centralized NAT (no external_mac
on DNAT rules), set nat-addresses=router on localnet ports, and vpcd
aligns macvlan MAC with OVN gateway router MAC on startup. Macvlan only
delivers unicast matching its own MAC — this ensures inbound traffic
from the LAN reaches OVS/OVN.
Also fixed: DNS for VMs. admin init auto-detects host DNS servers via
resolvectl, stores them in spinifex.toml dns_servers field, and vpcd
propagates them to OVN DHCP options. Fallback: 8.8.8.8 + 1.1.1.1.
Validated E2E: inbound ping, SSH, outbound ping, DNS, curl, apt update.
This commit is contained in:
Ben Duncan
@@ -26,6 +26,7 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
@@ -625,6 +626,15 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Detect DNS servers from the host for VM DHCP
|
||||
var dnsServers []string
|
||||
if externalMode != "" {
|
||||
dnsServers = detectDNSServers(externalIface)
|
||||
if len(dnsServers) > 0 {
|
||||
fmt.Printf(" DNS servers: %s\n", strings.Join(dnsServers, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
// For nat mode, we need the gateway IP. The DHCP flag tells setup-ovn.sh
|
||||
// to obtain one via DHCP on the macvlan/bridge interface.
|
||||
useExternalDHCP := externalMode == "nat" && poolStart == ""
|
||||
@@ -833,8 +843,9 @@ func runAdminInit(cmd *cobra.Command, args []string) {
|
||||
PoolName: "wan",
|
||||
PoolStart: poolStart,
|
||||
PoolEnd: poolEnd,
|
||||
PoolGateway: externalGateway,
|
||||
PoolPrefixLen: externalPrefixLen,
|
||||
PoolGateway: externalGateway,
|
||||
PoolPrefixLen: externalPrefixLen,
|
||||
PoolDNSServers: dnsServers,
|
||||
|
||||
BootstrapAccountId: admin.DefaultAccountID(),
|
||||
BootstrapVpcId: bootstrapVpcId,
|
||||
@@ -1681,3 +1692,76 @@ func writeBootstrapFilesWithAdmin(configDir string, masterKey []byte, accessKey,
|
||||
|
||||
return handlers_iam.SaveBootstrapData(filepath.Join(configDir, "bootstrap.json"), bd)
|
||||
}
|
||||
|
||||
// detectDNSServers reads the DNS servers configured on the host for the given
|
||||
// interface. Uses resolvectl (systemd-resolved) first, then falls back to
|
||||
// /etc/resolv.conf. Returns up to 3 servers. Falls back to public DNS if none found.
|
||||
func detectDNSServers(iface string) []string {
|
||||
// Try resolvectl for the specific link first (most reliable on modern systems)
|
||||
if iface != "" {
|
||||
out, err := exec.Command("resolvectl", "dns", iface).CombinedOutput()
|
||||
if err == nil {
|
||||
servers := parseDNSFromResolvectl(string(out))
|
||||
if len(servers) > 0 {
|
||||
return servers
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try resolvectl global
|
||||
out, err := exec.Command("resolvectl", "dns").CombinedOutput()
|
||||
if err == nil {
|
||||
servers := parseDNSFromResolvectl(string(out))
|
||||
if len(servers) > 0 {
|
||||
return servers
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to /etc/resolv.conf
|
||||
data, err := os.ReadFile("/etc/resolv.conf")
|
||||
if err == nil {
|
||||
var servers []string
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "nameserver ") {
|
||||
ip := strings.TrimSpace(strings.TrimPrefix(line, "nameserver"))
|
||||
// Skip localhost (systemd-resolved stub)
|
||||
if ip != "127.0.0.53" && ip != "127.0.0.1" && net.ParseIP(ip) != nil {
|
||||
servers = append(servers, ip)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(servers) > 0 {
|
||||
if len(servers) > 3 {
|
||||
servers = servers[:3]
|
||||
}
|
||||
return servers
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to well-known public DNS
|
||||
return []string{"8.8.8.8", "1.1.1.1"}
|
||||
}
|
||||
|
||||
// parseDNSFromResolvectl extracts IP addresses from resolvectl dns output.
|
||||
// Format: "Link 2 (enp0s3): 192.168.1.1 8.8.8.8 1.1.1.1"
|
||||
func parseDNSFromResolvectl(output string) []string {
|
||||
var servers []string
|
||||
for _, line := range strings.Split(output, "\n") {
|
||||
// Find the colon separator, IPs come after it
|
||||
idx := strings.Index(line, ":")
|
||||
if idx < 0 {
|
||||
continue
|
||||
}
|
||||
fields := strings.Fields(line[idx+1:])
|
||||
for _, f := range fields {
|
||||
if net.ParseIP(f) != nil && f != "127.0.0.53" && f != "127.0.0.1" {
|
||||
servers = append(servers, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(servers) > 3 {
|
||||
servers = servers[:3]
|
||||
}
|
||||
return servers
|
||||
}
|
||||
|
||||
+12
-10
@@ -704,6 +704,7 @@ var vpcdStartCmd = &cobra.Command{
|
||||
Gateway: p.Gateway,
|
||||
GatewayIP: p.GatewayIP,
|
||||
PrefixLen: p.PrefixLen,
|
||||
DNSServers: p.DNSServers,
|
||||
Region: p.Region,
|
||||
AZ: p.AZ,
|
||||
})
|
||||
@@ -728,16 +729,17 @@ var vpcdStartCmd = &cobra.Command{
|
||||
}
|
||||
|
||||
svc, err := service.New("vpcd", &vpcd.Config{
|
||||
NatsHost: nodeConfig.NATS.Host,
|
||||
NatsToken: nodeConfig.NATS.ACL.Token,
|
||||
OVNNBAddr: nodeConfig.VPCD.OVNNBAddr,
|
||||
OVNSBAddr: nodeConfig.VPCD.OVNSBAddr,
|
||||
BaseDir: nodeConfig.BaseDir,
|
||||
Debug: false,
|
||||
ExternalMode: clusterConfig.Network.ExternalMode,
|
||||
ExternalPools: extPools,
|
||||
ChassisNames: chassisNames,
|
||||
Bootstrap: bootstrap,
|
||||
NatsHost: nodeConfig.NATS.Host,
|
||||
NatsToken: nodeConfig.NATS.ACL.Token,
|
||||
OVNNBAddr: nodeConfig.VPCD.OVNNBAddr,
|
||||
OVNSBAddr: nodeConfig.VPCD.OVNSBAddr,
|
||||
BaseDir: nodeConfig.BaseDir,
|
||||
Debug: false,
|
||||
ExternalMode: clusterConfig.Network.ExternalMode,
|
||||
ExternalPools: extPools,
|
||||
ChassisNames: chassisNames,
|
||||
Bootstrap: bootstrap,
|
||||
ExternalInterface: nodeConfig.VPCD.ExternalInterface,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Println("Error starting vpcd service:", err)
|
||||
|
||||
@@ -21,6 +21,9 @@ gateway = "{{.PoolGateway}}"
|
||||
gateway_ip = "{{.PoolGatewayIP}}"
|
||||
{{- end}}
|
||||
prefix_len = {{if .PoolPrefixLen}}{{.PoolPrefixLen}}{{else}}24{{end}}
|
||||
{{- if .PoolDNSServers}}
|
||||
dns_servers = [{{range $i, $s := .PoolDNSServers}}{{if $i}}, {{end}}"{{$s}}"{{end}}]
|
||||
{{- end}}
|
||||
{{- end}}
|
||||
{{- if .BootstrapVpcId}}
|
||||
|
||||
|
||||
+325
-19
@@ -718,11 +718,15 @@ ovn-nbctl lr-route-add vpc-{vpcId} 0.0.0.0/0 192.168.1.1
|
||||
## Per-Instance Public IP
|
||||
|
||||
```bash
|
||||
# 1:1 NAT (distributed — NAT happens on VM's chassis, not gateway)
|
||||
ovn-nbctl lr-nat-add vpc-{vpcId} dnat_and_snat {public_ip} {private_ip} \
|
||||
port-{eniId} {eni_mac}
|
||||
# 1:1 NAT (centralized — processed on gateway chassis, uses router MAC)
|
||||
ovn-nbctl lr-nat-add vpc-{vpcId} dnat_and_snat {public_ip} {private_ip}
|
||||
```
|
||||
|
||||
> **Note:** Spinifex uses centralized NAT (no `external_mac` / `logical_port`)
|
||||
> because macvlan-based external bridges can only receive unicast traffic matching
|
||||
> the macvlan's own MAC. The router port MAC is set on the macvlan, and OVN uses
|
||||
> it for all ARP replies. For single-node, there is no performance difference.
|
||||
|
||||
## Security Group
|
||||
|
||||
```bash
|
||||
@@ -822,6 +826,121 @@ aws ec2 describe-instances --instance-ids $INSTANCE \
|
||||
|
||||
# Troubleshooting
|
||||
|
||||
## Debugging Toolkit
|
||||
|
||||
These commands are used throughout the troubleshooting sections below. Learn
|
||||
them — they cover 90% of VPC networking issues.
|
||||
|
||||
### OVN Northbound (logical topology)
|
||||
|
||||
```bash
|
||||
# Full topology overview (routers, switches, ports)
|
||||
sudo ovn-nbctl show
|
||||
|
||||
# List all VPC routers
|
||||
sudo ovn-nbctl lr-list
|
||||
|
||||
# List all switches (subnets + external)
|
||||
sudo ovn-nbctl ls-list
|
||||
|
||||
# NAT rules for a VPC
|
||||
sudo ovn-nbctl lr-nat-list vpc-{vpcId}
|
||||
|
||||
# Routes for a VPC
|
||||
sudo ovn-nbctl lr-route-list vpc-{vpcId}
|
||||
|
||||
# Port details (check "up" field for DHCP status)
|
||||
sudo ovn-nbctl list Logical_Switch_Port port-eni-{eniId}
|
||||
|
||||
# Gateway chassis assignment
|
||||
sudo ovn-nbctl list Logical_Router_Port gw-vpc-{vpcId}
|
||||
|
||||
# Localnet port options (check nat-addresses=router)
|
||||
sudo ovn-nbctl get Logical_Switch_Port ext-port-vpc-{vpcId} options
|
||||
```
|
||||
|
||||
### OVN Southbound (runtime state)
|
||||
|
||||
```bash
|
||||
# Chassis list + port bindings (which VM is on which host)
|
||||
sudo ovn-sbctl show
|
||||
|
||||
# Detailed chassis info (check name matches expectations)
|
||||
sudo ovn-sbctl list Chassis
|
||||
|
||||
# MAC binding table (shows ARP resolution for external traffic)
|
||||
sudo ovn-sbctl list MAC_Binding
|
||||
|
||||
# Trace a packet through the OVN pipeline (invaluable for debugging)
|
||||
sudo ovn-trace ext-vpc-{vpcId} \
|
||||
'inport=="ext-port-vpc-{vpcId}" && eth.dst==ff:ff:ff:ff:ff:ff && \
|
||||
arp.op==1 && arp.spa==192.168.1.13 && arp.tpa==192.168.1.201'
|
||||
```
|
||||
|
||||
### OVS (datapath / physical wiring)
|
||||
|
||||
```bash
|
||||
# Full bridge + port topology
|
||||
sudo ovs-vsctl show
|
||||
|
||||
# Kernel datapath ports and stats
|
||||
sudo ovs-dpctl show
|
||||
|
||||
# Kernel datapath flow cache (actual forwarding rules)
|
||||
sudo ovs-dpctl dump-flows
|
||||
|
||||
# OpenFlow rules installed by ovn-controller
|
||||
sudo ovs-ofctl dump-flows br-int | grep {pattern}
|
||||
|
||||
# Conntrack entries (shows active NAT sessions)
|
||||
sudo ovs-appctl dpctl/dump-conntrack | grep {ip}
|
||||
|
||||
# FDB (MAC address table) for a bridge
|
||||
sudo ovs-appctl fdb/show br-external
|
||||
|
||||
# OVS external_ids (system-id, bridge-mappings, encap-ip)
|
||||
sudo ovs-vsctl get Open_vSwitch . external_ids
|
||||
```
|
||||
|
||||
### Network Interfaces
|
||||
|
||||
```bash
|
||||
# Check macvlan exists and has correct MAC
|
||||
ip -d link show spx-ext-{nic}
|
||||
|
||||
# Check macvlan is attached to OVS (look for "openvswitch_slave")
|
||||
ip -d link show spx-ext-{nic} | grep openvswitch
|
||||
|
||||
# Interface traffic stats (RX/TX counts, drops)
|
||||
ip -s link show spx-ext-{nic}
|
||||
```
|
||||
|
||||
### Packet Capture
|
||||
|
||||
```bash
|
||||
# Capture on the OVS bridge (sees traffic after OVS processing)
|
||||
sudo tcpdump -i br-external -n -e arp
|
||||
|
||||
# Capture on the macvlan (sees what leaves/enters the wire)
|
||||
sudo tcpdump -i spx-ext-{nic} -n -e "host {public_ip}"
|
||||
|
||||
# Capture on the physical NIC (sees everything on the wire)
|
||||
sudo tcpdump -i {nic} -n "host {public_ip}"
|
||||
```
|
||||
|
||||
### Service Logs
|
||||
|
||||
```bash
|
||||
# vpcd log (reconcile, NAT, topology operations)
|
||||
cat ~/spinifex/logs/vpcd.log
|
||||
|
||||
# ovn-controller log (port binding, commit failures)
|
||||
sudo cat /var/log/ovn/ovn-controller.log | tail -50
|
||||
|
||||
# Daemon log (instance launch, network setup)
|
||||
cat ~/spinifex/logs/spinifex.log
|
||||
```
|
||||
|
||||
## VPC creation fails
|
||||
|
||||
Check OVN services and vpcd daemon:
|
||||
@@ -873,25 +992,212 @@ ip route show
|
||||
|
||||
## Public IP not reachable from WAN
|
||||
|
||||
1. Verify br-external:
|
||||
```bash
|
||||
sudo ovs-vsctl show | grep -A5 br-external
|
||||
```
|
||||
Work through these checks in order — each eliminates a class of issues.
|
||||
|
||||
2. Verify bridge mappings:
|
||||
```bash
|
||||
sudo ovs-vsctl get Open_vSwitch . external-ids:ovn-bridge-mappings
|
||||
```
|
||||
### 1. Verify OVS wiring
|
||||
|
||||
3. Check gateway chassis:
|
||||
```bash
|
||||
sudo ovn-nbctl lrp-get-gateway-chassis gw-$VPC
|
||||
```
|
||||
```bash
|
||||
# br-external must exist with the macvlan port
|
||||
sudo ovs-vsctl show | grep -A5 br-external
|
||||
|
||||
4. Verify `dnat_and_snat` rule:
|
||||
```bash
|
||||
sudo ovn-nbctl lr-nat-list vpc-$VPC | grep dnat_and_snat
|
||||
```
|
||||
# Bridge mappings must be set
|
||||
sudo ovs-vsctl get Open_vSwitch . external-ids:ovn-bridge-mappings
|
||||
# Expected: "external:br-external"
|
||||
```
|
||||
|
||||
### 2. Verify chassis and gateway scheduling
|
||||
|
||||
```bash
|
||||
# What OVS thinks the chassis name is
|
||||
sudo ovs-vsctl get Open_vSwitch . external-ids:system-id
|
||||
|
||||
# What OVN SB registered (must match the system-id above)
|
||||
sudo ovn-sbctl show
|
||||
# Look for: Chassis {name}
|
||||
|
||||
# What vpcd scheduled as gateway chassis (must match SB chassis name)
|
||||
sudo ovn-nbctl list Logical_Router_Port gw-vpc-{vpcId} | grep gateway_chassis
|
||||
```
|
||||
|
||||
If these don't match, see "Chassis name mismatch" below.
|
||||
|
||||
### 3. Verify NAT rule
|
||||
|
||||
```bash
|
||||
sudo ovn-nbctl lr-nat-list vpc-$VPC | grep dnat_and_snat
|
||||
# Must show the public IP → private IP mapping
|
||||
# Must NOT have external_mac or logical_port set (centralized NAT)
|
||||
```
|
||||
|
||||
### 4. Verify ARP resolution
|
||||
|
||||
From another host on the same LAN, check if OVN responds to ARP:
|
||||
|
||||
```bash
|
||||
# On the remote host:
|
||||
ping -c 1 {public_ip}
|
||||
ip neigh show {public_ip}
|
||||
# Should show the OVN router MAC (02:00:00:xx:xx:xx)
|
||||
```
|
||||
|
||||
If ARP fails, check the macvlan MAC alignment:
|
||||
|
||||
```bash
|
||||
# On the Spinifex host:
|
||||
# Gateway router MAC
|
||||
sudo ovn-nbctl get Logical_Router_Port gw-vpc-{vpcId} mac
|
||||
# Macvlan MAC
|
||||
ip link show spx-ext-{nic} | grep ether
|
||||
|
||||
# These MUST match. If they don't, vpcd failed to set the macvlan MAC.
|
||||
# Check vpcd.log for "macvlan MAC aligned" or errors.
|
||||
```
|
||||
|
||||
### 5. Verify packet flow with tcpdump
|
||||
|
||||
Capture at each layer to find where packets stop:
|
||||
|
||||
```bash
|
||||
# Layer 1: Does the ARP/ICMP arrive on the physical NIC?
|
||||
sudo tcpdump -i {nic} -n "host {public_ip}"
|
||||
|
||||
# Layer 2: Does it reach br-external?
|
||||
sudo tcpdump -i br-external -n "host {public_ip}"
|
||||
|
||||
# Layer 3: Does it reach the macvlan output?
|
||||
sudo tcpdump -i spx-ext-{nic} -n -e "host {public_ip}"
|
||||
```
|
||||
|
||||
If traffic arrives on the physical NIC but not br-external, the macvlan is
|
||||
filtering it. Check MAC alignment (step 4).
|
||||
|
||||
### 6. Use ovn-trace for pipeline debugging
|
||||
|
||||
Simulate a packet through the entire OVN pipeline:
|
||||
|
||||
```bash
|
||||
sudo ovn-trace --ct=new ext-vpc-{vpcId} \
|
||||
'inport=="ext-port-vpc-{vpcId}" && eth.dst==ff:ff:ff:ff:ff:ff && \
|
||||
arp.op==1 && arp.sha=={remote_mac} && arp.spa=={remote_ip} && \
|
||||
arp.tpa=={public_ip}'
|
||||
```
|
||||
|
||||
The output shows every table the packet passes through and what action is
|
||||
taken. Look for `drop` actions or unexpected paths.
|
||||
|
||||
## OVN SB commit failure loop
|
||||
|
||||
**Symptom:** ovn-controller log shows:
|
||||
|
||||
```
|
||||
OVNSB commit failed, force recompute next time.
|
||||
```
|
||||
|
||||
Repeated millions of times. Port binding never happens (`up: false`).
|
||||
|
||||
**Cause:** Stale entries in the OVN Southbound DB (old chassis records, port
|
||||
bindings, datapath bindings) conflict with ovn-controller's expected state.
|
||||
Happens when `reset-dev-env.sh` cleans NB but not SB, or after ungraceful
|
||||
shutdowns.
|
||||
|
||||
**Fix:** Delete both OVN DB files and restart. `reset-dev-env.sh` does this
|
||||
automatically:
|
||||
|
||||
```bash
|
||||
sudo systemctl stop ovn-central ovn-controller
|
||||
sudo rm -f /var/lib/ovn/ovnnb_db.db /var/lib/ovn/ovnsb_db.db
|
||||
sudo systemctl start ovn-central ovn-controller
|
||||
# vpcd reconcile will recreate the NB topology on next startup
|
||||
```
|
||||
|
||||
## Chassis name mismatch
|
||||
|
||||
**Symptom:** Gateway chassis is set in NB but the SB has no matching chassis.
|
||||
`ovn-nbctl show` reports a gateway chassis name that doesn't appear in
|
||||
`ovn-sbctl show`. Gateway traffic (SNAT/DNAT) doesn't work.
|
||||
|
||||
**Cause:** Three names must align:
|
||||
1. OVS `external_ids:system-id` → what ovn-controller registers as
|
||||
2. OVN SB `Chassis.name` → what actually exists in the SB DB
|
||||
3. vpcd's chassis name → `chassis-{nodeName}` from config
|
||||
|
||||
If `setup-ovn.sh` used `chassis-$(hostname -s)` but vpcd uses
|
||||
`chassis-{nodeName}`, they'll differ on any host where hostname ≠ node name.
|
||||
|
||||
**Diagnosis:**
|
||||
|
||||
```bash
|
||||
# What OVS has
|
||||
sudo ovs-vsctl get Open_vSwitch . external-ids:system-id
|
||||
# What SB registered
|
||||
sudo ovn-sbctl list Chassis | grep name
|
||||
# What vpcd expects
|
||||
grep node ~/spinifex/config/spinifex.toml | head -1
|
||||
```
|
||||
|
||||
**Fix:** `reset-dev-env.sh` passes `--chassis-id=chassis-node1` to
|
||||
`setup-ovn.sh`. For manual fix:
|
||||
|
||||
```bash
|
||||
sudo ovs-vsctl set Open_vSwitch . external_ids:system-id=chassis-node1
|
||||
sudo systemctl restart ovn-controller
|
||||
```
|
||||
|
||||
## Macvlan unicast filtering (inbound traffic lost)
|
||||
|
||||
**Symptom:** VM gets DHCP, outbound works (ping 8.8.8.8 from VM), but inbound
|
||||
from the LAN to the VM's public IP fails. ARP replies go out but ICMP never
|
||||
arrives at br-external.
|
||||
|
||||
**Cause:** macvlan in bridge mode only delivers unicast frames matching its own
|
||||
MAC. OVN announces the router MAC for external IPs, but the macvlan has a
|
||||
different (random) MAC. Broadcasts (ARP requests) pass through; unicast return
|
||||
traffic is filtered by the kernel macvlan subsystem.
|
||||
|
||||
**Diagnosis:**
|
||||
|
||||
```bash
|
||||
# Compare these two — they must match
|
||||
sudo ovn-nbctl get Logical_Router_Port gw-vpc-{vpcId} mac
|
||||
ip link show spx-ext-{nic} | grep ether
|
||||
|
||||
# If they differ, inbound unicast is being dropped
|
||||
```
|
||||
|
||||
**Fix:** vpcd automatically aligns the macvlan MAC on startup. If it failed:
|
||||
|
||||
```bash
|
||||
# Get the gateway MAC from OVN
|
||||
GW_MAC=$(sudo ovn-nbctl get Logical_Router_Port gw-vpc-{vpcId} mac | tr -d '"')
|
||||
|
||||
# Set it on the macvlan
|
||||
sudo ip link set spx-ext-{nic} down
|
||||
sudo ip link set spx-ext-{nic} address $GW_MAC
|
||||
sudo ip link set spx-ext-{nic} up
|
||||
```
|
||||
|
||||
Also ensure the DNAT rule uses centralized NAT (no `external_mac`):
|
||||
|
||||
```bash
|
||||
sudo ovn-nbctl lr-nat-list vpc-{vpcId}
|
||||
# The dnat_and_snat row should have EMPTY external_mac and logical_port columns
|
||||
```
|
||||
|
||||
## Stale ARP on remote hosts
|
||||
|
||||
**Symptom:** Ping from a LAN host to a VM public IP fails after a reset, but
|
||||
worked before. The remote host has a stale ARP entry with the old MAC.
|
||||
|
||||
**Fix:** Flush the ARP entry on the remote host:
|
||||
|
||||
```bash
|
||||
# On the remote host:
|
||||
sudo ip neigh flush dev {nic} {public_ip}
|
||||
ping {public_ip} # should work now
|
||||
```
|
||||
|
||||
OVN's `nat-addresses=router` option sends periodic gratuitous ARPs that will
|
||||
eventually update remote ARP caches, but flushing is faster for testing.
|
||||
|
||||
## Security group rules not taking effect
|
||||
|
||||
|
||||
+22
-33
@@ -86,37 +86,19 @@ if command -v ovs-vsctl >/dev/null 2>&1; then
|
||||
sudo systemctl stop openvswitch-switch 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# Clean OVN Northbound database (remove all logical routers, switches, NAT rules, etc.)
|
||||
echo "Cleaning OVN Northbound database"
|
||||
if command -v ovn-nbctl >/dev/null 2>&1; then
|
||||
sudo systemctl start ovn-central 2>/dev/null || true
|
||||
# Wait for NB DB
|
||||
for i in $(seq 1 10); do
|
||||
if sudo ovn-nbctl --timeout=2 show >/dev/null 2>&1; then break; fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Delete all logical routers (and their ports, NAT rules, routes)
|
||||
for lr in $(sudo ovn-nbctl --no-headings --columns=name find Logical_Router 2>/dev/null | awk -F'"' '{print $2}'); do
|
||||
[ -n "$lr" ] && sudo ovn-nbctl lr-del "$lr" 2>/dev/null && echo " Deleted router: $lr"
|
||||
done
|
||||
# Delete all logical switches (and their ports)
|
||||
for ls in $(sudo ovn-nbctl --no-headings --columns=name find Logical_Switch 2>/dev/null | awk -F'"' '{print $2}'); do
|
||||
[ -n "$ls" ] && sudo ovn-nbctl ls-del "$ls" 2>/dev/null && echo " Deleted switch: $ls"
|
||||
done
|
||||
# Delete all port groups
|
||||
for pg in $(sudo ovn-nbctl --no-headings --columns=name find Port_Group 2>/dev/null | awk -F'"' '{print $2}'); do
|
||||
[ -n "$pg" ] && sudo ovn-nbctl pg-del "$pg" 2>/dev/null && echo " Deleted port group: $pg"
|
||||
done
|
||||
# Delete all DHCP options
|
||||
for uuid in $(sudo ovn-nbctl --no-headings --columns=_uuid find DHCP_Options 2>/dev/null | awk '{print $NF}'); do
|
||||
[ -n "$uuid" ] && sudo ovn-nbctl dhcp-options-del "$uuid" 2>/dev/null
|
||||
done
|
||||
|
||||
sudo systemctl stop ovn-central 2>/dev/null || true
|
||||
echo " OVN NB database cleaned"
|
||||
# Clean OVN databases (both Northbound and Southbound)
|
||||
# Delete the DB files outright — setup-ovn.sh will restart ovn-central with fresh
|
||||
# empty databases. This eliminates stale SB state (chassis entries, port bindings,
|
||||
# datapath bindings) that accumulates across resets and causes ovn-controller to
|
||||
# enter a commit failure loop ("OVNSB commit failed, force recompute").
|
||||
echo "Removing OVN database files"
|
||||
sudo systemctl stop ovn-central 2>/dev/null || true
|
||||
sudo systemctl stop ovn-controller 2>/dev/null || true
|
||||
if [ -d /var/lib/ovn ]; then
|
||||
sudo rm -f /var/lib/ovn/ovnnb_db.db /var/lib/ovn/ovnsb_db.db
|
||||
echo " Deleted /var/lib/ovn/ovn{nb,sb}_db.db"
|
||||
else
|
||||
echo " ovn-nbctl not found, skipping OVN cleanup"
|
||||
echo " /var/lib/ovn not found, skipping OVN DB cleanup"
|
||||
fi
|
||||
|
||||
# Remove macvlan interfaces created by setup-ovn.sh
|
||||
@@ -138,13 +120,20 @@ WAN_GW=$(ip -4 route show default | awk '{print $3}' | head -1)
|
||||
|
||||
echo "Detected WAN interface: ${WAN_IFACE:-none}, gateway: ${WAN_GW:-none}"
|
||||
|
||||
echo "Re-initializing OVN"
|
||||
# Chassis ID must match what vpcd derives from the node name in spinifex.toml.
|
||||
# vpcd generates "chassis-{nodeName}" (e.g., chassis-node1) and uses it to schedule
|
||||
# gateway chassis in the NB DB. ovn-controller registers in the SB DB using the OVS
|
||||
# system-id. If they don't match, gateway traffic has no host to schedule to.
|
||||
NODE_NAME="node1"
|
||||
CHASSIS_ID="chassis-${NODE_NAME}"
|
||||
|
||||
echo "Re-initializing OVN (chassis-id: $CHASSIS_ID)"
|
||||
if [ -n "$WAN_IFACE" ]; then
|
||||
echo " Using macvlan on $WAN_IFACE for br-external"
|
||||
./scripts/setup-ovn.sh --management --external-bridge --external-iface="$WAN_IFACE"
|
||||
./scripts/setup-ovn.sh --management --external-bridge --external-iface="$WAN_IFACE" --chassis-id="$CHASSIS_ID"
|
||||
else
|
||||
echo " No WAN interface detected, management-only"
|
||||
./scripts/setup-ovn.sh --management
|
||||
./scripts/setup-ovn.sh --management --chassis-id="$CHASSIS_ID"
|
||||
fi
|
||||
|
||||
echo "Initializing platform"
|
||||
|
||||
@@ -72,8 +72,9 @@ type ConfigSettings struct {
|
||||
PoolStart string // First IP in external pool range
|
||||
PoolEnd string // Last IP in external pool range
|
||||
PoolGateway string // WAN gateway IP
|
||||
PoolGatewayIP string // Explicit SNAT IP (for nat mode without DHCP)
|
||||
PoolPrefixLen int // Subnet prefix length (default 24)
|
||||
PoolGatewayIP string // Explicit SNAT IP (for nat mode without DHCP)
|
||||
PoolPrefixLen int // Subnet prefix length (default 24)
|
||||
PoolDNSServers []string // DNS servers for VM DHCP (auto-detected from host)
|
||||
|
||||
// Other nodes in the cluster (for config source of truth)
|
||||
RemoteNodes []RemoteNode
|
||||
|
||||
@@ -20,14 +20,15 @@ type ClusterConfig struct {
|
||||
|
||||
// ExternalPool defines a range of routable IPs that Spinifex manages for public subnets.
|
||||
type ExternalPool struct {
|
||||
Name string `mapstructure:"name"` // Pool identifier (e.g., "wan", "dc1-primary")
|
||||
RangeStart string `mapstructure:"range_start"` // First IP in range
|
||||
RangeEnd string `mapstructure:"range_end"` // Last IP in range
|
||||
Gateway string `mapstructure:"gateway"` // WAN default gateway (next hop for 0.0.0.0/0)
|
||||
GatewayIP string `mapstructure:"gateway_ip"` // OVN router external IP (override; defaults to first IP in range)
|
||||
PrefixLen int `mapstructure:"prefix_len"` // Subnet mask (default 24)
|
||||
Region string `mapstructure:"region"` // Scope to region (optional — empty means any region)
|
||||
AZ string `mapstructure:"az"` // Scope to AZ (optional — empty means any AZ in region)
|
||||
Name string `mapstructure:"name"` // Pool identifier (e.g., "wan", "dc1-primary")
|
||||
RangeStart string `mapstructure:"range_start"` // First IP in range
|
||||
RangeEnd string `mapstructure:"range_end"` // Last IP in range
|
||||
Gateway string `mapstructure:"gateway"` // WAN default gateway (next hop for 0.0.0.0/0)
|
||||
GatewayIP string `mapstructure:"gateway_ip"` // OVN router external IP (override; defaults to first IP in range)
|
||||
PrefixLen int `mapstructure:"prefix_len"` // Subnet mask (default 24)
|
||||
DNSServers []string `mapstructure:"dns_servers"` // DNS servers for VM DHCP (auto-detected from host; fallback: 8.8.8.8, 1.1.1.1)
|
||||
Region string `mapstructure:"region"` // Scope to region (optional — empty means any region)
|
||||
AZ string `mapstructure:"az"` // Scope to AZ (optional — empty means any AZ in region)
|
||||
}
|
||||
|
||||
// NetworkConfig holds cluster-wide external network settings.
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"github.com/mulgadc/spinifex/spinifex/services/vpcd/nbdb"
|
||||
"github.com/mulgadc/spinifex/spinifex/types"
|
||||
@@ -94,6 +95,19 @@ func WithChassisNames(names []string) TopologyOption {
|
||||
}
|
||||
}
|
||||
|
||||
// dnsServer returns the DNS server string for OVN DHCP options.
|
||||
// Uses dns_servers from the external pool config (auto-detected by admin init).
|
||||
// Falls back to 8.8.8.8 and 1.1.1.1 if none configured.
|
||||
// OVN DHCP expects the format "{ip1, ip2}" for multiple servers.
|
||||
func (h *TopologyHandler) dnsServer() string {
|
||||
pool := h.findExternalPool("", "")
|
||||
if pool != nil && len(pool.DNSServers) > 0 {
|
||||
return "{" + strings.Join(pool.DNSServers, ", ") + "}"
|
||||
}
|
||||
// Fallback: public DNS
|
||||
return "{8.8.8.8, 1.1.1.1}"
|
||||
}
|
||||
|
||||
// findExternalPool returns the first pool matching the given region/AZ,
|
||||
// using the fallback order: AZ-scoped → region-scoped → unscoped.
|
||||
func (h *TopologyHandler) findExternalPool(region, az string) *ExternalPoolConfig {
|
||||
@@ -373,7 +387,7 @@ func (h *TopologyHandler) handleSubnetCreate(msg *nats.Msg) {
|
||||
"server_mac": routerMAC,
|
||||
"lease_time": "3600",
|
||||
"router": gwIP,
|
||||
"dns_server": "169.254.169.253",
|
||||
"dns_server": h.dnsServer(),
|
||||
"mtu": "1442", // Geneve overhead
|
||||
},
|
||||
ExternalIDs: map[string]string{
|
||||
@@ -581,12 +595,16 @@ func (h *TopologyHandler) handleIGWAttach(msg *nats.Msg) {
|
||||
}
|
||||
|
||||
// 2. Create localnet port on external switch (maps to physical network)
|
||||
// nat-addresses=router: OVN sends gratuitous ARPs for all NAT external IPs
|
||||
// using the router port MAC. Required for macvlan-based external bridges
|
||||
// where the macvlan MAC matches the router MAC.
|
||||
localnetPort := &nbdb.LogicalSwitchPort{
|
||||
Name: extPortName,
|
||||
Type: "localnet",
|
||||
Addresses: []string{"unknown"},
|
||||
Options: map[string]string{
|
||||
"network_name": "external",
|
||||
"network_name": "external",
|
||||
"nat-addresses": "router",
|
||||
},
|
||||
ExternalIDs: map[string]string{
|
||||
"spinifex:vpc_id": evt.VpcId,
|
||||
@@ -825,15 +843,15 @@ func (h *TopologyHandler) handleAddNAT(msg *nats.Msg) {
|
||||
ctx := context.Background()
|
||||
routerName := "vpc-" + evt.VpcId
|
||||
|
||||
logicalPort := evt.PortName
|
||||
externalMAC := evt.MAC
|
||||
|
||||
// Use centralized NAT (no ExternalMAC/LogicalPort). With macvlan-based
|
||||
// external bridges, OVN must use the router port MAC for all ARP replies
|
||||
// so the macvlan can receive inbound unicast traffic. Distributed NAT
|
||||
// (ExternalMAC/LogicalPort) announces the VM's MAC which the macvlan
|
||||
// filters out. For single-node, centralized NAT has zero overhead.
|
||||
natRule := &nbdb.NAT{
|
||||
Type: "dnat_and_snat",
|
||||
ExternalIP: evt.ExternalIP,
|
||||
LogicalIP: evt.LogicalIP,
|
||||
LogicalPort: &logicalPort,
|
||||
ExternalMAC: &externalMAC,
|
||||
Type: "dnat_and_snat",
|
||||
ExternalIP: evt.ExternalIP,
|
||||
LogicalIP: evt.LogicalIP,
|
||||
ExternalIDs: map[string]string{
|
||||
"spinifex:vpc_id": evt.VpcId,
|
||||
"spinifex:public_ip": evt.ExternalIP,
|
||||
@@ -971,7 +989,7 @@ func (h *TopologyHandler) reconcileSubnet(ctx context.Context, subnetId, vpcId,
|
||||
"server_mac": routerMAC,
|
||||
"lease_time": "3600",
|
||||
"router": gwIP,
|
||||
"dns_server": "169.254.169.253",
|
||||
"dns_server": h.dnsServer(),
|
||||
"mtu": "1442",
|
||||
},
|
||||
ExternalIDs: map[string]string{
|
||||
@@ -1042,10 +1060,13 @@ func (h *TopologyHandler) reconcileIGW(ctx context.Context, vpcId, igwId string)
|
||||
portExtIDs["spinifex:igw_id"] = igwId
|
||||
}
|
||||
localnetPort := &nbdb.LogicalSwitchPort{
|
||||
Name: extPortName,
|
||||
Type: "localnet",
|
||||
Addresses: []string{"unknown"},
|
||||
Options: map[string]string{"network_name": "external"},
|
||||
Name: extPortName,
|
||||
Type: "localnet",
|
||||
Addresses: []string{"unknown"},
|
||||
Options: map[string]string{
|
||||
"network_name": "external",
|
||||
"nat-addresses": "router",
|
||||
},
|
||||
ExternalIDs: portExtIDs,
|
||||
}
|
||||
if err := h.ovn.CreateLogicalSwitchPort(ctx, extSwitchName, localnetPort); err != nil {
|
||||
|
||||
@@ -60,6 +60,9 @@ type Config struct {
|
||||
ChassisNames []string
|
||||
// Bootstrap holds the default VPC config from spinifex.toml for first-boot reconciliation.
|
||||
Bootstrap *BootstrapVPC
|
||||
// ExternalInterface is the WAN NIC name (e.g., "enp0s3"). Used to align
|
||||
// the macvlan MAC with the OVN gateway MAC for inbound traffic.
|
||||
ExternalInterface string
|
||||
}
|
||||
|
||||
// ExternalPoolConfig mirrors config.ExternalPool for vpcd's internal use.
|
||||
@@ -70,6 +73,7 @@ type ExternalPoolConfig struct {
|
||||
Gateway string
|
||||
GatewayIP string
|
||||
PrefixLen int
|
||||
DNSServers []string
|
||||
Region string
|
||||
AZ string
|
||||
}
|
||||
@@ -230,6 +234,20 @@ func launchService(cfg *Config) error {
|
||||
// Run reconciliation before subscribing
|
||||
Reconcile(ctx, topo, cfg.Bootstrap)
|
||||
|
||||
// Align macvlan MAC with the OVN gateway router MAC. The macvlan only
|
||||
// delivers inbound unicast matching its own MAC. With centralized NAT,
|
||||
// OVN uses the router MAC for all external ARP replies. Setting the
|
||||
// macvlan MAC to the router MAC ensures inbound traffic reaches OVS.
|
||||
if cfg.Bootstrap != nil && cfg.Bootstrap.VpcId != "" && cfg.ExternalInterface != "" {
|
||||
gwMAC := generateMAC("gw-" + cfg.Bootstrap.VpcId)
|
||||
macvlanName := "spx-ext-" + cfg.ExternalInterface
|
||||
if err := setMacvlanMAC(macvlanName, gwMAC); err != nil {
|
||||
slog.Warn("vpcd: failed to set macvlan MAC (inbound traffic may not work)", "iface", macvlanName, "mac", gwMAC, "err", err)
|
||||
} else {
|
||||
slog.Info("vpcd: macvlan MAC aligned with gateway", "iface", macvlanName, "mac", gwMAC)
|
||||
}
|
||||
}
|
||||
|
||||
subs, err := topo.Subscribe(nc)
|
||||
if err != nil {
|
||||
slog.Error("Failed to subscribe to VPC topics", "err", err)
|
||||
@@ -251,3 +269,19 @@ func launchService(cfg *Config) error {
|
||||
slog.Info("vpcd service shutting down")
|
||||
return nil
|
||||
}
|
||||
|
||||
// setMacvlanMAC sets the MAC address on a macvlan interface. The interface is
|
||||
// brought down, MAC changed, and brought back up. Requires sudo/NET_ADMIN.
|
||||
func setMacvlanMAC(iface, mac string) error {
|
||||
cmds := [][]string{
|
||||
{"sudo", "ip", "link", "set", iface, "down"},
|
||||
{"sudo", "ip", "link", "set", iface, "address", mac},
|
||||
{"sudo", "ip", "link", "set", iface, "up"},
|
||||
}
|
||||
for _, args := range cmds {
|
||||
if out, err := exec.Command(args[0], args[1:]...).CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("%s: %w (%s)", args[3], err, string(out))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user