diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..945ae3a08 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,21 @@ +# CODEOWNERS — GitHub PR review enforcement +# +# Format: [ …] +# Order matters — last matching pattern wins. +# +# Set the matching branch protection rule on `dev` and `main` to +# "Require review from Code Owners" or this file is advisory only. + +# Fall-through: any change needs a team review. +* @mulgadc/mulgadc-engineers + +# Network layer is governed by ADR-0006 invariant tests. +# Changes here must respect those clauses — see +# spinifex/network/invariants/ and +# docs/development/proposals/az-local-architecture/0006-spinifex-network-layer-contract.md +/spinifex/network/ @joshsiv-mulga +/spinifex/vpcd/ @joshsiv-mulga + +# Invariants suite is the executable ADR-0006 fence — any widening or +# narrowing of the scan roots needs an explicit owner review. +/spinifex/network/invariants/ @joshsiv-mulga diff --git a/.github/scripts/capture-net-diag.sh b/.github/scripts/capture-net-diag.sh new file mode 100755 index 000000000..43e9123fe --- /dev/null +++ b/.github/scripts/capture-net-diag.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# capture-net-diag.sh — write a snapshot of network / OVN / OVS / IPsec +# state into $1 (a directory the caller already created on the local host). +# Designed to run on a spinifex node after an E2E suite, before tofu destroy +# pulls the rug out. Every command is independent — `|| true` so a missing +# binary on one path doesn't abort the rest of the capture. +# +# Output layout (under $1/net-diag/): +# openvswitch-ipsec.status systemctl status (output-only) +# openvswitch-ipsec.journal last 200 journal lines +# ipsec.openvswitch_conf ovs-vsctl get Open_vSwitch . other_config +# ipsec.xfrm_state ip xfrm state list +# ipsec.xfrm_policy ip xfrm policy list +# ovn.sb_show ovn-sbctl show +# ovn.nb_show ovn-nbctl show +# ovs.show ovs-vsctl show +# ovs.flows_br_int ovs-ofctl dump-flows br-int +# ovs.tunnels ovs-appctl ofproto/list-tunnels br-int +# net.links ip -s link show +# net.routes ip route show table all +# net.addrs ip -br addr show +# systemd.failed_units systemctl list-units --state=failed +set +e +OUT="${1:?usage: capture-net-diag.sh }" +mkdir -p "$OUT" + +run() { + local label="$1"; shift + { + echo "# $*" + "$@" 2>&1 + } > "$OUT/$label" || true +} + +run openvswitch-ipsec.status sudo systemctl status openvswitch-ipsec.service --no-pager --full --lines=50 +run openvswitch-ipsec.journal sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso -n 200 +run ipsec.openvswitch_conf sudo ovs-vsctl get Open_vSwitch . other_config +run ipsec.xfrm_state sudo ip xfrm state list +run ipsec.xfrm_policy sudo ip xfrm policy list + +run ovn.sb_show sudo ovn-sbctl show +run ovn.nb_show sudo ovn-nbctl show +run ovn.sb_chassis sudo ovn-sbctl list chassis + +run ovs.show sudo ovs-vsctl show +run ovs.flows_br_int sudo ovs-ofctl dump-flows br-int +run ovs.tunnels sudo ovs-appctl ofproto/list-tunnels br-int + +run net.links ip -s link show +run net.routes ip route show table all +run net.addrs ip -br addr show + +run systemd.failed_units sudo systemctl list-units --state=failed --no-pager + +exit 0 diff --git a/.github/workflows/e2e-go.yml b/.github/workflows/e2e-go.yml index 49141b187..1258a66cd 100644 --- a/.github/workflows/e2e-go.yml +++ b/.github/workflows/e2e-go.yml @@ -4,8 +4,8 @@ name: E2E (Go) # Mirrors e2e.yml's single-node provisioning chain but runs `go test` via # gotestsum so we get JUnit XML + per-test artifact directories surfaced to # GitHub Actions. Manual-dispatch only while the conversion lands — push -# trigger lifts once the single-node Go port (mulga-siv-89) is green and -# replaces run-e2e.sh; matches e2e-ddil.yml's Phase 1 trigger model. +# trigger lifts once the single-node Go port is green and replaces +# run-e2e.sh; matches e2e-ddil.yml's Phase 1 trigger model. on: workflow_dispatch: @@ -17,6 +17,10 @@ on: description: "Space-separated Make targets to run (e.g. 'e2e-single e2e-multinode e2e-cert e2e-lb')" required: false default: "e2e-single e2e-multinode e2e-cert e2e-lb" + test_run: + description: "Optional regex for -test.run (e.g. '^TestMultinodeIPSec$'). Empty = run all tests in suite." + required: false + default: "" permissions: contents: read @@ -32,6 +36,7 @@ concurrency: env: GIT_BRANCH: ${{ github.ref_name }} SUITES: ${{ inputs.suites || 'e2e-single e2e-multinode e2e-cert e2e-lb' }} + TEST_RUN: ${{ inputs.test_run || '' }} jobs: e2e_go_single: @@ -226,6 +231,13 @@ jobs: echo "rc=0" >> "$GITHUB_OUTPUT" exit 0 fi + # Optional regex passed through to `go test -test.run`. Empty input + # means run all tests in the suite (default behaviour). + if [ -n "${TEST_RUN}" ]; then + TEST_RUN_FLAG="-test.run=${TEST_RUN}" + else + TEST_RUN_FLAG="" + fi VM_HOST="tf-user@${{ steps.vm.outputs.wan_ip }}" SSH_KEY="${{ steps.vm.outputs.ssh_key }}" # On cancel / runner SIGTERM / step timeout, force-kill any test @@ -258,7 +270,7 @@ jobs: date -u +%Y-%m-%dT%H:%M:%SZ > '${ARTIFACT_DIR}/test-'\$SUITE'.start'; \ GITHUB_ACTIONS=true FORCE_COLOR=1 SPINIFEX_E2E=1 SPINIFEX_MODE=single \ ARTIFACT_DIR='${ARTIFACT_DIR}' \ - ./\$SUITE.test -test.v -test.timeout=30m \ + ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \ 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \ | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \ rc=\${PIPESTATUS[0]}; \ @@ -277,6 +289,23 @@ jobs: echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT" exit 0 # don't fail the step — artifact upload + final gate happen below + - name: Capture network/IPsec diagnostics on VM + if: always() + run: | + set +e + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + # Ship the diag script, run it into the VM's ARTIFACT_DIR, leave the + # output for the next step's tar to pull down. Captures OVN/OVS/IPsec + # state at failure time — before teardown destroys the VM. + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ + "$VM":/tmp/capture-net-diag.sh + ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'" + ok "net diagnostics captured" + - name: Collect artifacts from VM if: always() run: | @@ -605,6 +634,11 @@ jobs: echo "rc=0" >> "$GITHUB_OUTPUT" exit 0 fi + if [ -n "${TEST_RUN}" ]; then + TEST_RUN_FLAG="-test.run=${TEST_RUN}" + else + TEST_RUN_FLAG="" + fi VM_HOST="tf-user@${{ steps.cluster.outputs.node1_ip }}" SSH_KEY="${{ steps.cluster.outputs.ssh_key }}" NODES_CSV="${{ steps.cluster.outputs.nodes_csv }}" @@ -634,7 +668,7 @@ jobs: SPINIFEX_SSH_USER=tf-user \ SPINIFEX_SSH_KEY=\$HOME/.ssh/tf-user-ap-southeast-2 \ ARTIFACT_DIR='${ARTIFACT_DIR}' \ - ./\$SUITE.test -test.v -test.timeout=30m \ + ./\$SUITE.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \ 2>&1 | tee '${ARTIFACT_DIR}/test-'\$SUITE'.log' \ | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//'; \ rc=\${PIPESTATUS[0]}; \ @@ -653,6 +687,42 @@ jobs: echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT" exit 0 + - name: Capture network/IPsec diagnostics on all nodes + if: always() + run: | + set +e + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") + # Capture per-node into the node1 ARTIFACT_DIR (where Collect pulls + # from). One subdir per node so xfrm SA state is comparable across + # peers — critical for diagnosing the IPsec wiring. + IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}" + NODE1_IP="${NODE_LIST[0]}" + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ + tf-user@"$NODE1_IP":/tmp/capture-net-diag.sh + for i in "${!NODE_LIST[@]}"; do + idx=$((i + 1)) + ip="${NODE_LIST[$i]}" + if [ "$idx" -ne 1 ]; then + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ + tf-user@"$ip":/tmp/capture-net-diag.sh + fi + ssh "${SSH_OPTS[@]}" tf-user@"$ip" \ + "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag-node${idx}'" + # node1 keeps its diag locally; pull node2..N back into node1's + # ARTIFACT_DIR so the existing tar-pull below catches everything. + if [ "$idx" -ne 1 ]; then + ssh "${SSH_OPTS[@]}" tf-user@"$ip" \ + "tar -C '${ARTIFACT_DIR}' -cf - 'net-diag-node${idx}' 2>/dev/null || true" \ + | ssh "${SSH_OPTS[@]}" tf-user@"$NODE1_IP" \ + "tar -C '${ARTIFACT_DIR}' -xf - 2>/dev/null" + fi + ok "net diagnostics node${idx} captured" + done + - name: Collect artifacts from node1 if: always() run: | @@ -675,6 +745,15 @@ jobs: > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null ok "journal node${idx}: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null || echo 0) lines" done + # openvswitch-ipsec journal per node (separate file). + # Debian unit name; the binary inside is /usr/share/openvswitch/scripts/ovs-ctl + # which spawns ovs-monitor-ipsec as a child. + for i in "${!NODE_LIST[@]}"; do + idx=$((i + 1)) + ip="${NODE_LIST[$i]}" + ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \ + > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null + done - name: Convert test logs to JUnit XML if: always() @@ -919,6 +998,11 @@ jobs: R=$'\033[0m'; B=$'\033[1m'; C=$'\033[36m' printf '\n%s%s━━ suite reboot ━━%s\n' "$B" "$C" "$R" + if [ -n "${TEST_RUN}" ]; then + TEST_RUN_FLAG="-test.run=${TEST_RUN}" + else + TEST_RUN_FLAG="" + fi GITHUB_ACTIONS=true FORCE_COLOR=1 \ SPINIFEX_E2E=1 SPINIFEX_MODE=single \ SPINIFEX_WAN_IP="${{ steps.vm.outputs.wan_ip }}" \ @@ -927,7 +1011,7 @@ jobs: SPINIFEX_AWS_ACCESS_KEY_ID="${{ steps.creds.outputs.ak }}" \ SPINIFEX_AWS_SECRET_ACCESS_KEY="${{ steps.creds.outputs.sk }}" \ ARTIFACT_DIR="${ARTIFACT_DIR}" \ - ./spinifex/tests/e2e/_bin/reboot.test -test.v -test.timeout=30m \ + ./spinifex/tests/e2e/_bin/reboot.test -test.v -test.timeout=30m ${TEST_RUN_FLAG} \ 2>&1 | tee "${ARTIFACT_DIR}/test-reboot.log" \ | sed -uE 's/^[[:space:]]+([a-zA-Z0-9_]+_test\.go:[0-9]+: )?//' rc=${PIPESTATUS[0]} @@ -1029,7 +1113,7 @@ jobs: # dispatched ref, always present. The previous `&& inputs.sha` gate # plus `statuses/${{ inputs.sha }}` produced an 'invalid hash' error # whenever the workflow was dispatched without the optional input - # (tests passed, status step silently failed). Refs mulga-siv-112. + # (tests passed, status step silently failed). if: always() needs: [ e2e_go_single, e2e_go_multinode, e2e_go_reboot ] runs-on: [ self-hosted, ci-single ] diff --git a/Makefile b/Makefile index b1873a661..618126ea6 100644 --- a/Makefile +++ b/Makefile @@ -132,7 +132,7 @@ test: COVERPROFILE ?= coverage.out test-cover: @echo -e "\n....Running tests with coverage for $(GO_PROJECT_NAME)...." - $(_Q)LOG_IGNORE=1 go test -timeout 120s -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... $(_COVQ) + $(_Q)LOG_IGNORE=1 go test -timeout 120s -coverpkg=./spinifex/... -coverprofile=$(COVERPROFILE) -covermode=atomic ./spinifex/... $(_COVQ) @scripts/check-coverage.sh $(COVERPROFILE) $(QUIET) # Run unit tests with race detector @@ -281,20 +281,28 @@ distro-clean: # Ansible dev lifecycle (experimental, parallel to dev-*.sh / reset-dev-env.sh). # See scripts/ansible/README.md and docs/development/improvements/ansible-dev-lifecycle.md. +# +# Variable overrides: pass EXTRA_VARS="key=val key2=val2" to forward as +# `--extra-vars` to ansible-playbook. Plain `-e` on the make command line is +# make's own flag and does NOT reach ansible. +# make ansible-dev-install EXTRA_VARS="spinifex_external_pool_start=192.168.1.90 spinifex_external_pool_end=192.168.1.99" +EXTRA_VARS ?= +_ANSIBLE_EXTRA = $(if $(strip $(EXTRA_VARS)),--extra-vars "$(EXTRA_VARS)",) + ansible-dev-preflight: - cd scripts/ansible && ansible-playbook playbooks/dev-preflight.yml + cd scripts/ansible && ansible-playbook playbooks/dev-preflight.yml $(_ANSIBLE_EXTRA) ansible-dev-teardown: - cd scripts/ansible && ansible-playbook playbooks/dev-teardown.yml + cd scripts/ansible && ansible-playbook playbooks/dev-teardown.yml $(_ANSIBLE_EXTRA) ansible-dev-install: - cd scripts/ansible && ansible-playbook playbooks/dev-install.yml + cd scripts/ansible && ansible-playbook playbooks/dev-install.yml $(_ANSIBLE_EXTRA) ansible-dev-reset: - cd scripts/ansible && ansible-playbook playbooks/dev-reset.yml + cd scripts/ansible && ansible-playbook playbooks/dev-reset.yml $(_ANSIBLE_EXTRA) ansible-dev-deploy: - cd scripts/ansible && ansible-playbook playbooks/dev-deploy.yml + cd scripts/ansible && ansible-playbook playbooks/dev-deploy.yml $(_ANSIBLE_EXTRA) ansible-dev-status: cd scripts/ansible && ansible-playbook playbooks/dev-status.yml diff --git a/build/systemd/spinifex-daemon.service b/build/systemd/spinifex-daemon.service index 6d9edb95c..b7106fec1 100644 --- a/build/systemd/spinifex-daemon.service +++ b/build/systemd/spinifex-daemon.service @@ -1,7 +1,14 @@ [Unit] Description=Spinifex Daemon (VM Orchestration) PartOf=spinifex.target -After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service +After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service openvswitch-ipsec.service +# Wants=openvswitch-ipsec.service so systemd brings the IPsec userspace +# daemon up before spinifex flips other_config:ipsec_encapsulation=true. +# Without that daemon, OVS holds the cert pointers unused and strongSwan +# never gets IKE config — tunnel traffic is encrypted with no SAs and +# silently dropped. Wants= (not Requires=) keeps daemon bootable on +# single-node / lab hosts where the package is intentionally absent. +Wants=openvswitch-ipsec.service # No Wants=spinifex-nats.service: Wants= reactivates a stopped unit during # `systemctl restart spinifex-daemon`, which would mask the standalone-startup # path the Tier 1 DDIL Scenario B exercises. spinifex.target Wants= covers @@ -23,7 +30,7 @@ Group=spinifex ExecStart=/usr/local/bin/spx service spinifex start Restart=on-failure RestartSec=5 -# DDIL Tier 1 (mulga-siv-58): KillMode=process so systemd only signals the +# DDIL Tier 1: KillMode=process so systemd only signals the # daemon's main PID, not the cgroup. QEMU children stay alive across # `systemctl restart spinifex-daemon` and the new daemon reattaches to them # via the persisted local state file + pidfile + QMP socket. With the @@ -38,7 +45,7 @@ OOMScoreAdjust=-500 Environment=SPINIFEX_CONFIG_PATH=/etc/spinifex/spinifex.toml Environment=SPINIFEX_BASE_DIR=/var/lib/spinifex/spinifex/ Environment=SPINIFEX_WAL_DIR=/var/lib/spinifex/spinifex/ -# DDIL Tier 1 (mulga-siv-58): QEMU pidfiles, QMP sockets, and the +# DDIL Tier 1: QEMU pidfiles, QMP sockets, and the # shared qemu.log all resolve via utils.pidPath() — which falls back # to os.TempDir() (=/tmp) when $HOME/spinifex doesn't exist. With # PrivateTmp=yes, each daemon start gets a fresh /tmp namespace, so diff --git a/cmd/dhcptest/main.go b/cmd/dhcptest/main.go index b774a18ce..6ac423700 100644 --- a/cmd/dhcptest/main.go +++ b/cmd/dhcptest/main.go @@ -89,7 +89,7 @@ func main() { } } - // Build nclient4 options — identical to spinifex/services/vpcd/dhcp/nclient4.go. + // Build nclient4 options — standalone DHCP probe for upstream-router debugging. opts := []nclient4.ClientOpt{ nclient4.WithHWAddr(hwAddr), nclient4.WithTimeout(*timeout), @@ -105,7 +105,7 @@ func main() { } defer client.Close() - // Identity modifiers — identical to identityModifiers() in vpcd dhcp package. + // Identity modifiers — client-id / hostname / vendor-class for upstream lease tagging. var mods []dhcpv4.Modifier if cid != "" { mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(cid)))) diff --git a/cmd/installer/install/install.go b/cmd/installer/install/install.go index d62174af0..3f01c86a1 100644 --- a/cmd/installer/install/install.go +++ b/cmd/installer/install/install.go @@ -304,9 +304,10 @@ func installSpinifex(cfg *Config) error { return err } - // dhcpcd-base is present on the installed system (used by setup-ovn.sh for - // macvlan mode). Mask the standalone dhcpcd.service so it never auto-starts - // and races with systemd-networkd's built-in DHCP client on br-wan. + // dhcpcd-base is present on the installed system (used by setup-ovn.sh + // for WAN DHCP acquisition). Mask the standalone dhcpcd.service so it + // never auto-starts and races with systemd-networkd's built-in DHCP + // client on br-wan. if err := maskSystemdUnit(mountRoot, "dhcpcd.service"); err != nil { slog.Warn("installSpinifex: failed to mask dhcpcd.service", "err", err) } diff --git a/cmd/spinifex/cmd/admin.go b/cmd/spinifex/cmd/admin.go index 7c4208480..4db61fa3c 100644 --- a/cmd/spinifex/cmd/admin.go +++ b/cmd/spinifex/cmd/admin.go @@ -300,6 +300,7 @@ func init() { adminInitCmd.Flags().Int("external-prefix-len", 24, "External pool subnet prefix length (auto-detected)") adminInitCmd.Flags().Bool("no-external", false, "Disable external networking (overlay-only, no internet for VMs)") adminInitCmd.Flags().Bool("gpu-passthrough", false, "Enable VFIO GPU passthrough (sets gpu_passthrough = true in daemon config)") + adminInitCmd.Flags().Bool("ipsec", true, "Encrypt intra-AZ Geneve via OVN native IPsec (cluster-wide); disable only for trusted single-rack lab") // Flags for admin join adminJoinCmd.Flags().String("region", "ap-southeast-2", "Region for this node") @@ -844,6 +845,7 @@ func runAdminInit(cmd *cobra.Command, args []string) { gatewayIP, _ := cmd.Flags().GetString("gateway-ip") noExternal, _ := cmd.Flags().GetBool("no-external") gpuPassthrough, _ := cmd.Flags().GetBool("gpu-passthrough") + ipsecEnabled, _ := cmd.Flags().GetBool("ipsec") // Fire telemetry in background (completes during init work, waited at end) noTelemetry, _ := cmd.Flags().GetBool("no-telemetry") @@ -893,9 +895,9 @@ func runAdminInit(cmd *cobra.Command, args []string) { fmt.Printf(" %-14s %-18s %-20s %-16s %s\n", iface.Name, iface.IP, iface.Subnet, gw, strings.ToUpper(iface.Role)) } if detected.LANCount == 0 { - fmt.Println("\n Mode: single-NIC (macvlan for external bridge)") + fmt.Println("\n Mode: single-NIC (veth-bridged external)") } else { - fmt.Printf("\n Mode: %d LAN + 1 WAN (macvlan for external bridge)\n", detected.LANCount) + fmt.Printf("\n Mode: %d LAN + 1 WAN (veth-bridged external)\n", detected.LANCount) } // Apply auto-detected values when flags not explicitly set @@ -910,13 +912,11 @@ func runAdminInit(cmd *cobra.Command, args []string) { externalPrefixLen = detected.WAN.PrefixLen } - // Default mode: always "pool" — with static range if --external-pool - // is given, otherwise with source=dhcp (gateway IP from router DHCP) + // Default mode: always "pool". Source defaults to "static"; if + // --external-pool is omitted the validator below will error with + // a SuggestPoolRange hint. if externalMode == "" && !cmd.Flags().Changed("external-mode") { externalMode = "pool" - if externalPool == "" && externalSource == "" { - externalSource = "dhcp" - } } } } @@ -928,42 +928,29 @@ func runAdminInit(cmd *cobra.Command, args []string) { os.Exit(1) } if externalMode == "pool" { - // Resolve source: if not explicitly set, infer from whether a pool range was given - if externalSource == "" { - if externalPool != "" { - externalSource = "static" - } else { - externalSource = "dhcp" - } + if externalSource != "" && externalSource != "static" { + fmt.Fprintf(os.Stderr, "❌ Error: --external-source must be 'static' (only supported value), got: %s\n", externalSource) + os.Exit(1) } - if externalSource == "dhcp" { - // DHCP source: no static range needed, just gateway - if externalGateway == "" { - fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required when --external-mode=pool\n") - os.Exit(1) + externalSource = "static" + if externalPool == "" { + fmt.Fprintf(os.Stderr, "❌ Error: --external-pool is required when --external-mode=pool (e.g., 192.168.1.150-192.168.1.250)\n") + if detectedNet != nil && detectedNet.WAN != nil { + sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN) + fmt.Fprintf(os.Stderr, " Suggested: --external-pool=%s-%s\n", sugStart, sugEnd) } - } else { - // Static source: need pool range - if externalPool == "" { - fmt.Fprintf(os.Stderr, "❌ Error: --external-pool is required when --external-mode=pool (e.g., 192.168.1.150-192.168.1.250)\n") - fmt.Fprintf(os.Stderr, " Or use --external-source=dhcp to get IPs from router DHCP\n") - if detectedNet != nil && detectedNet.WAN != nil { - sugStart, sugEnd := admin.SuggestPoolRange(detectedNet.WAN) - fmt.Fprintf(os.Stderr, " Suggested: --external-pool=%s-%s\n", sugStart, sugEnd) - } - os.Exit(1) - } - if externalGateway == "" { - fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required when --external-mode=pool\n") - os.Exit(1) - } - parts := strings.SplitN(externalPool, "-", 2) - if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil { - fmt.Fprintf(os.Stderr, "❌ Error: --external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s\n", externalPool) - os.Exit(1) - } - poolStart, poolEnd = parts[0], parts[1] + os.Exit(1) } + if externalGateway == "" { + fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is required when --external-mode=pool\n") + os.Exit(1) + } + parts := strings.SplitN(externalPool, "-", 2) + if len(parts) != 2 || net.ParseIP(parts[0]) == nil || net.ParseIP(parts[1]) == nil { + fmt.Fprintf(os.Stderr, "❌ Error: --external-pool must be start-end IPs (e.g., 192.168.1.150-192.168.1.250), got: %s\n", externalPool) + os.Exit(1) + } + poolStart, poolEnd = parts[0], parts[1] } if externalGateway != "" && net.ParseIP(externalGateway) == nil { fmt.Fprintf(os.Stderr, "❌ Error: --external-gateway is not a valid IP: %s\n", externalGateway) @@ -983,10 +970,6 @@ func runAdminInit(cmd *cobra.Command, args []string) { } } - // For pool/dhcp source, the gateway IP is obtained via DHCP on the - // macvlan/bridge interface (no static pool range or explicit gateway-ip). - useExternalDHCP := externalMode == "pool" && externalSource == "dhcp" && gatewayIP == "" - // Validate IP address format if net.ParseIP(bindIP) == nil { fmt.Fprintf(os.Stderr, "❌ Error: Invalid IP address for --bind: %s\n", bindIP) @@ -1104,6 +1087,18 @@ func runAdminInit(cmd *cobra.Command, args []string) { // Generate SSL certificates (with bind IP in SANs for multi-node support) certPath := admin.GenerateCertificatesIfNeeded(configDir, force, bindIP) + // Generate per-node IPsec peer cert when cluster-wide IPsec is enabled + // (default true). Reuses the cluster CA — no intermediate strongSwan PKI. + if ipsecEnabled { + caCertPath := filepath.Join(configDir, "ca.pem") + caKeyPath := filepath.Join(configDir, "ca.key") + if err := admin.GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, node, bindIP); err != nil { + fmt.Fprintf(os.Stderr, "Error generating IPsec peer certificate: %v\n", err) + os.Exit(1) + } + fmt.Println("🔐 IPsec peer certificate generated (intra-AZ Geneve encryption ON)") + } + // Install CA certificate into system trust store installCACertificate(filepath.Join(configDir, "ca.pem")) @@ -1125,30 +1120,31 @@ func runAdminInit(cmd *cobra.Command, args []string) { isMultiNode := nodes >= 2 if isMultiNode { - // Build cluster-wide network config for propagation to joining nodes - var networkConfig *formation.NetworkConfig + // Build cluster-wide network config for propagation to joining nodes. + // Always emit so the IPSecEnabled flag reaches joiners even when + // external networking is disabled. + networkConfig := &formation.NetworkConfig{ + IPSecEnabled: ipsecEnabled, + } if externalMode != "" { bootstrapVpcId := utils.GenerateResourceID("vpc") bootstrapSubnetId := utils.GenerateResourceID("subnet") bootstrapIgwId := utils.GenerateResourceID("igw") - networkConfig = &formation.NetworkConfig{ - ExternalMode: externalMode, - ExternalDHCP: useExternalDHCP, - PoolName: "wan", - PoolSource: externalSource, - PoolStart: poolStart, - PoolEnd: poolEnd, - PoolGateway: externalGateway, - PoolGatewayIP: gatewayIP, - PoolPrefixLen: externalPrefixLen, - PoolDNSServers: dnsServers, - BootstrapAccountId: admin.DefaultAccountID(), - BootstrapVpcId: bootstrapVpcId, - BootstrapSubnetId: bootstrapSubnetId, - BootstrapIgwId: bootstrapIgwId, - BootstrapCidr: handlers_ec2_vpc.DefaultVPCCidr, - BootstrapSubnetCidr: handlers_ec2_vpc.DefaultSubnetCidr, - } + networkConfig.ExternalMode = externalMode + networkConfig.PoolName = "wan" + networkConfig.PoolSource = externalSource + networkConfig.PoolStart = poolStart + networkConfig.PoolEnd = poolEnd + networkConfig.PoolGateway = externalGateway + networkConfig.PoolGatewayIP = gatewayIP + networkConfig.PoolPrefixLen = externalPrefixLen + networkConfig.PoolDNSServers = dnsServers + networkConfig.BootstrapAccountId = admin.DefaultAccountID() + networkConfig.BootstrapVpcId = bootstrapVpcId + networkConfig.BootstrapSubnetId = bootstrapSubnetId + networkConfig.BootstrapIgwId = bootstrapIgwId + networkConfig.BootstrapCidr = handlers_ec2_vpc.DefaultVPCCidr + networkConfig.BootstrapSubnetCidr = handlers_ec2_vpc.DefaultSubnetCidr } runAdminInitMultiNode(cmd, accessKey, secretKey, accountID, natsToken, clusterName, @@ -1248,8 +1244,6 @@ func runAdminInit(cmd *cobra.Command, args []string) { ExternalMode: externalMode, ExternalIface: externalIface, - DhcpBindBridge: detectedDhcpBindBridge(detectedNet), - ExternalDHCP: useExternalDHCP, PoolName: "wan", PoolSource: externalSource, PoolStart: poolStart, @@ -1268,6 +1262,7 @@ func runAdminInit(cmd *cobra.Command, args []string) { BootstrapSubnetCidr: handlers_ec2_vpc.DefaultSubnetCidr, GPUPassthrough: gpuPassthrough, + IPSecEnabled: ipsecEnabled, } // Print external networking summary @@ -1278,10 +1273,8 @@ func runAdminInit(cmd *cobra.Command, args []string) { fmt.Printf(" Source: static (IP range)\n") fmt.Printf(" IP pool: %s - %s\n", poolStart, poolEnd) fmt.Printf(" ⚠️ Ensure %s-%s is excluded from your router's DHCP range.\n", poolStart, poolEnd) - } else if useExternalDHCP { - fmt.Printf(" Source: dhcp (from router DHCP)\n") - fmt.Println(" Gateway IP: DHCP (obtained during OVN setup)") - } else if gatewayIP != "" { + } + if gatewayIP != "" { fmt.Printf(" Gateway IP: %s (static)\n", gatewayIP) } } @@ -1861,6 +1854,24 @@ func runAdminJoin(cmd *cobra.Command, args []string) { } fmt.Printf("✅ Server certificate generated with bind IP: %s\n\n", bindIP) + // Match the leader's intra-AZ IPsec posture: NetworkConfig.IPSecEnabled is + // authoritative. When the formation response omits NetworkConfig entirely + // (no current code path; defensive guard against future regressions) fall + // back to the AWS-parity default of ON. + ipsecEnabled := true + if statusResp.NetworkConfig != nil { + ipsecEnabled = statusResp.NetworkConfig.IPSecEnabled + } + if ipsecEnabled { + caCertPath := filepath.Join(configDir, "ca.pem") + caKeyPath := filepath.Join(configDir, "ca.key") + if err := admin.GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, node, bindIP); err != nil { + fmt.Fprintf(os.Stderr, "Error generating IPsec peer certificate: %v\n", err) + os.Exit(1) + } + fmt.Println("🔐 IPsec peer certificate generated (intra-AZ Geneve encryption ON)") + } + // Build cluster topology from formation data clusterRoutes := formation.BuildClusterRoutes(statusResp.Nodes) predastoreNodes := formation.BuildPredastoreNodes(statusResp.Nodes) @@ -2270,8 +2281,8 @@ func finalizeNodeSetup(dataDir, certPath, adminAccessKey, adminSecretKey, region // applyNetworkConfig copies cluster-wide network settings from a formation // NetworkConfig into ConfigSettings and auto-detects the local WAN interface. func applyNetworkConfig(settings *admin.ConfigSettings, nc *formation.NetworkConfig) { + settings.IPSecEnabled = nc.IPSecEnabled settings.ExternalMode = nc.ExternalMode - settings.ExternalDHCP = nc.ExternalDHCP settings.PoolName = nc.PoolName settings.PoolSource = nc.PoolSource settings.PoolStart = nc.PoolStart @@ -2292,7 +2303,6 @@ func applyNetworkConfig(settings *admin.ConfigSettings, nc *formation.NetworkCon detected, err := admin.DetectNetwork() if err == nil && detected.WAN != nil { settings.ExternalIface = detected.WAN.Name - settings.DhcpBindBridge = detectedDhcpBindBridge(detected) } } } @@ -2401,29 +2411,6 @@ func writeBootstrapFilesWithAdmin(configDir, bootstrapDir string, masterKey []by return handlers_iam.SaveBootstrapData(filepath.Join(bootstrapDir, "bootstrap.json"), bd) } -// detectedDhcpBindBridge returns the bridge name where the vpcd DHCP client -// should bind its AF_PACKET socket — i.e. the interface that physically sees -// LAN DHCP traffic. -// -// - Default route on a bridge (Linux or OVS, `br-*` prefix): return the -// bridge name verbatim. In veth mode this is the Linux bridge holding the -// WAN NIC; in direct mode this is the OVS bridge holding the WAN NIC. -// - Default route on a bare physical NIC: return "br-wan" as the name the -// installer will create. -// -// Never returns "br-ext". br-ext is the OVN-side bridge owned by -// setup-ovn.sh's ovn-bridge-mappings and never sees LAN DHCP frames. -func detectedDhcpBindBridge(detected *admin.DetectedNetwork) string { - if detected == nil || detected.WAN == nil { - return "" - } - name := detected.WAN.Name - if strings.HasPrefix(name, "br-") { - return name - } - return "br-wan" -} - // detectDNSServers auto-detects DNS servers from the host for the specified // interface. Uses resolvectl (systemd-resolved) first, then falls back to // /etc/resolv.conf. Returns up to 3 servers. Falls back to public DNS if none found. diff --git a/cmd/spinifex/cmd/admin_test.go b/cmd/spinifex/cmd/admin_test.go index 9a5e9f0da..b5b9dd80a 100644 --- a/cmd/spinifex/cmd/admin_test.go +++ b/cmd/spinifex/cmd/admin_test.go @@ -142,49 +142,6 @@ func TestSpinifexTomlTemplate_AdvertiseOmittedWhenEmpty(t *testing.T) { assert.NotContains(t, string(data), "advertise =") } -// detectedDhcpBindBridge must return the default-route interface name verbatim -// when it's a bridge (Linux or OVS, br-* prefix). The old detectedWanBridge() -// returned hardcoded "br-ext" for Linux bridges — that value broke DHCP on -// consumer-router LANs because br-ext never sees LAN DHCP traffic (mulga-998). -func TestDetectedDhcpBindBridge(t *testing.T) { - tests := []struct { - name string - detected *admin.DetectedNetwork - want string - }{ - { - name: "Linux bridge default route returns bridge name (not 'br-ext')", - detected: &admin.DetectedNetwork{WAN: &admin.DetectedInterface{Name: "br-wan"}}, - want: "br-wan", - }, - { - name: "OVS bridge default route returned verbatim", - detected: &admin.DetectedNetwork{WAN: &admin.DetectedInterface{Name: "br-ext"}}, - want: "br-ext", - }, - { - name: "physical NIC defaults to 'br-wan' (bridge the installer creates)", - detected: &admin.DetectedNetwork{WAN: &admin.DetectedInterface{Name: "enp0s3"}}, - want: "br-wan", - }, - { - name: "nil DetectedNetwork", - detected: nil, - want: "", - }, - { - name: "DetectedNetwork with nil WAN", - detected: &admin.DetectedNetwork{}, - want: "", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, detectedDhcpBindBridge(tt.detected)) - }) - } -} - // Legacy `wan_bridge` TOML key must fail-start vpcd with guidance, not silently // alias (per mulga-998 D3). Prevents the footgun where operators inherited the // old key pointing at 'br-ext' and got broken DHCP on veth-mode hosts. @@ -206,8 +163,8 @@ wan_bridge = "br-ext" err := checkLegacyWanBridgeKey("node1", cfgPath) require.Error(t, err) - assert.Contains(t, err.Error(), "dhcp_bind_bridge") assert.Contains(t, err.Error(), "wan_bridge") + assert.Contains(t, err.Error(), "Remove") } func TestCheckLegacyWanBridgeKey_EnvVarRejected(t *testing.T) { @@ -218,7 +175,7 @@ func TestCheckLegacyWanBridgeKey_EnvVarRejected(t *testing.T) { err := checkLegacyWanBridgeKey("node1", "/tmp/unused.toml") require.Error(t, err) assert.Contains(t, err.Error(), "SPINIFEX_VPCD_WAN_BRIDGE") - assert.Contains(t, err.Error(), "dhcp_bind_bridge") + assert.Contains(t, err.Error(), "Remove") } func TestCheckLegacyWanBridgeKey_CleanConfigPasses(t *testing.T) { @@ -228,7 +185,7 @@ func TestCheckLegacyWanBridgeKey_CleanConfigPasses(t *testing.T) { node = "node1" [nodes.node1.vpcd] ovn_nb_addr = "tcp:127.0.0.1:6641" -dhcp_bind_bridge = "br-wan" +external_interface = "enp0s3" `), 0o644)) viper.Reset() diff --git a/cmd/spinifex/cmd/service.go b/cmd/spinifex/cmd/service.go index 0ac6b9fbd..ecc91265c 100644 --- a/cmd/spinifex/cmd/service.go +++ b/cmd/spinifex/cmd/service.go @@ -29,7 +29,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/services/predastore" "github.com/mulgadc/spinifex/spinifex/services/spinifexui" "github.com/mulgadc/spinifex/spinifex/services/viperblockd" - "github.com/mulgadc/spinifex/spinifex/services/vpcd" + "github.com/mulgadc/spinifex/spinifex/vpcd" "github.com/spf13/cobra" "github.com/spf13/viper" ) @@ -678,11 +678,8 @@ var spinifexUIStatusCmd = &cobra.Command{ } // checkLegacyWanBridgeKey fails vpcd startup if the deprecated `wan_bridge` -// TOML key or `SPINIFEX_VPCD_WAN_BRIDGE` env-var is present. The key was -// renamed to `dhcp_bind_bridge` (see mulga-998.a) because the old name misled -// operators into pointing at the OVN-side bridge ("br-ext"), which never sees -// LAN DHCP traffic. Per D3: no silent alias, no auto-rewrite — operator must -// rename the key before vpcd will start. +// TOML key or `SPINIFEX_VPCD_WAN_BRIDGE` env-var is present. Per D3: no silent +// alias, no auto-rewrite — operator must remove the key before vpcd will start. func checkLegacyWanBridgeKey(node, cfgFile string) error { legacyInTOML := viper.IsSet("nodes." + node + ".vpcd.wan_bridge") legacyInEnv := os.Getenv("SPINIFEX_VPCD_WAN_BRIDGE") != "" @@ -695,9 +692,7 @@ func checkLegacyWanBridgeKey(node, cfgFile string) error { } return fmt.Errorf( "vpcd: deprecated 'wan_bridge' key found in %s. "+ - "Rename 'wan_bridge' to 'dhcp_bind_bridge'. The value may also be wrong — verify it is "+ - "the Linux bridge holding your WAN NIC (veth mode) or the OVS bridge holding your WAN NIC (direct mode); "+ - "never 'br-ext'. Typical value on a consumer-router LAN: 'br-wan'. "+ + "Remove the key entirely; vpcd auto-detects the WAN bridge (br-wan). "+ "Then: sudo systemctl restart spinifex-vpcd", source, ) @@ -733,7 +728,6 @@ var vpcdStartCmd = &cobra.Command{ for _, p := range clusterConfig.Network.ExternalPools { extPools = append(extPools, vpcd.ExternalPoolConfig{ Name: p.Name, - Source: p.Source, RangeStart: p.RangeStart, RangeEnd: p.RangeEnd, Gateway: p.Gateway, @@ -742,7 +736,6 @@ var vpcdStartCmd = &cobra.Command{ DNSServers: p.DNSServers, Region: p.Region, AZ: p.AZ, - DhcpBindBridge: nodeConfig.VPCD.DhcpBindBridge, GwLrpRangeStart: p.GwLrpRangeStart, GwLrpRangeEnd: p.GwLrpRangeEnd, }) @@ -778,8 +771,8 @@ var vpcdStartCmd = &cobra.Command{ ExternalPools: extPools, Bootstrap: bootstrap, ExternalInterface: nodeConfig.VPCD.ExternalInterface, - DhcpBindBridge: nodeConfig.VPCD.DhcpBindBridge, BridgeMode: nodeConfig.VPCD.BridgeMode, + AZ: nodeConfig.AZ, }) if err != nil { fmt.Println("Error starting vpcd service:", err) diff --git a/cmd/spinifex/cmd/templates/spinifex.toml b/cmd/spinifex/cmd/templates/spinifex.toml index f341002f1..983fcee91 100644 --- a/cmd/spinifex/cmd/templates/spinifex.toml +++ b/cmd/spinifex/cmd/templates/spinifex.toml @@ -1,5 +1,5 @@ # Domain configuration file in TOML format. -version = "2" +version = "3" epoch = 1 node = "{{.Node}}" {{- if .OperatorEmail}} @@ -7,13 +7,11 @@ node = "{{.Node}}" [operator] email = "{{.OperatorEmail}}" {{- end}} -{{- if .ExternalMode}} [network] +ipsec_enabled = {{.IPSecEnabled}} +{{- if .ExternalMode}} external_mode = "{{.ExternalMode}}" -{{- if .ExternalDHCP}} -external_dhcp = true -{{- end}} [[network.external_pools]] name = "{{.PoolName}}" @@ -97,15 +95,6 @@ ovn_sb_addr = "{{.OVNSBAddr}}" {{- if .ExternalIface}} external_interface = "{{.ExternalIface}}" {{- end}} -{{- if .DhcpBindBridge}} -# dhcp_bind_bridge: the bridge where vpcd's DHCP client binds its AF_PACKET -# socket. On Linux-bridge-WAN hosts (netplan default with the WAN NIC enslaved), -# this is the Linux bridge name (e.g. "br-wan"). On direct-OVS hosts, this is -# the OVS bridge holding the WAN NIC. -# NEVER set this to "br-ext" — that is the OVN-side bridge and never sees LAN -# DHCP traffic. -dhcp_bind_bridge = "{{.DhcpBindBridge}}" -{{- end}} [nodes.{{.Node}}.predastore] host = "{{.BindIP}}:8443" diff --git a/docs/service-interfaces.yaml b/docs/service-interfaces.yaml index e45e33089..f92c6f276 100644 --- a/docs/service-interfaces.yaml +++ b/docs/service-interfaces.yaml @@ -131,7 +131,7 @@ services: - "s3.*" vpcd: - path: spinifex/services/vpcd + path: spinifex/vpcd # OVN-backed VPC / subnet / SG / NAT / IGW data plane. subscribes: - "vpc.create" diff --git a/scripts/ansible/README.md b/scripts/ansible/README.md index 2b53c367f..2f1631224 100644 --- a/scripts/ansible/README.md +++ b/scripts/ansible/README.md @@ -5,7 +5,6 @@ Lives alongside the existing `scripts/dev-*.sh` and `scripts/reset-dev-env.sh`; neither set is removed. Decide which to keep after parity testing. Plan: `docs/development/improvements/ansible-dev-lifecycle.md` (mulga top-level). -Bead: `mulga-siv-9`. ## Prerequisites @@ -107,12 +106,39 @@ Useful overrides: | `teardown_process_wait_seconds` | `30` | Grace period for processes to exit | | `spinifex_region` | `ap-southeast-2` | Region passed to `spx admin init` | | `spinifex_az` | `{{ region }}a` | AZ passed to `spx admin init` | -| `spinifex_external_mode` | `pool` | `pool` \| `nat` external networking | +| `spinifex_external_mode` | `pool` | External networking mode — `pool` or empty (NAT removed) | +| `spinifex_external_no_external` | `false` | `true` → emit `--no-external` and skip external networking | +| `spinifex_external_pool_start` | `""` | **Required** for `pool` mode — first IP of the static WAN pool | +| `spinifex_external_pool_end` | `""` | **Required** for `pool` mode — last IP of the static WAN pool | +| `spinifex_external_gateway` | `""` | WAN gateway (auto-detected when blank) | +| `spinifex_external_prefix_len` | `""` | WAN prefix length (auto-detected when blank) | +| `spinifex_external_gateway_ip` | `""` | OVN router external-IP override (`--gateway-ip`) | + +`pool` mode is mandatory unless you set `spinifex_external_no_external=true`. +`spx admin init` rejects the run without a `--external-pool` range, so the +init role asserts `spinifex_external_pool_start/_end` upfront. Either persist +them in `vars/defaults.yml`, or pass them at invocation: + +Direct `ansible-playbook` (use `-e` here — this is ansible's flag): + +``` +ansible-playbook playbooks/dev-install.yml \ + -e spinifex_external_pool_start=192.168.1.150 \ + -e spinifex_external_pool_end=192.168.1.250 +``` + +Via `make` (use `EXTRA_VARS=` — make eats its own `-e`): + +``` +make ansible-dev-install \ + EXTRA_VARS="spinifex_external_pool_start=192.168.1.150 spinifex_external_pool_end=192.168.1.250" +``` `dev-reset.yml` automatically captures `region`, `az`, `external_mode`, -pool range, gateway, prefix-len, nat gateway_ip from the existing +pool range, gateway, prefix-len and `gateway_ip` from the existing `/etc/spinifex/spinifex.toml` before teardown and replays them into -`init`. Use `-e` to override. +`init`. Legacy `external_mode = "nat"` is coerced to `"pool"` with a warn. +Use `-e` to override. ## Helper scripts @@ -129,7 +155,7 @@ calling user (no `sudo -u` invoked → no `initgroups(3)` → supplementary groups inherited from the operator's shell). On a fresh login the operator's shell would have `spinifex` (added by `setup.sh`); on shells that predate the group add it does not. The helpers sidestep this -entirely. See bead `mulga-siv-92`. +entirely. ## Leak catalog diff --git a/scripts/ansible/playbooks/dev-reset.yml b/scripts/ansible/playbooks/dev-reset.yml index 6ae496bc8..82fa47d77 100644 --- a/scripts/ansible/playbooks/dev-reset.yml +++ b/scripts/ansible/playbooks/dev-reset.yml @@ -23,6 +23,7 @@ when: reset_config_stat.stat.exists # List-concat-and-take-first pattern: missing regex match falls back to current default. + # `gateway\s*=` deliberately does not match `gateway_ip` / `gateway_lrp_*`. - name: capture — extract scalars ansible.builtin.set_fact: spinifex_region: "{{ (reset_config_txt | regex_findall('region\\s*=\\s*\"([^\"]+)\"') + [spinifex_region])[0] }}" @@ -30,11 +31,25 @@ spinifex_external_mode: "{{ (reset_config_txt | regex_findall('external_mode\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_mode])[0] }}" spinifex_external_pool_start: "{{ (reset_config_txt | regex_findall('range_start\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_pool_start])[0] }}" spinifex_external_pool_end: "{{ (reset_config_txt | regex_findall('range_end\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_pool_end])[0] }}" - spinifex_external_gateway: "{{ (reset_config_txt | regex_findall('gateway\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_gateway])[0] }}" + spinifex_external_gateway: "{{ (reset_config_txt | regex_findall('\\bgateway\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_gateway])[0] }}" spinifex_external_prefix_len: "{{ (reset_config_txt | regex_findall('prefix_len\\s*=\\s*([0-9]+)') + [spinifex_external_prefix_len | string])[0] }}" - spinifex_nat_gateway_ip: "{{ (reset_config_txt | regex_findall('gateway_ip\\s*=\\s*\"([^\"]+)\"') + [spinifex_nat_gateway_ip])[0] }}" + spinifex_external_gateway_ip: "{{ (reset_config_txt | regex_findall('gateway_ip\\s*=\\s*\"([^\"]+)\"') + [spinifex_external_gateway_ip])[0] }}" when: reset_config_stat.stat.exists + # Legacy: external_mode = "nat" was removed from spx. Coerce to "pool" + # so old configs still reset cleanly; warn so the operator notices. + - name: capture — coerce legacy external_mode=nat to pool + ansible.builtin.set_fact: + spinifex_external_mode: "pool" + when: spinifex_external_mode == 'nat' + + - name: capture — warn on legacy nat coercion + ansible.builtin.debug: + msg: "⚠️ Captured external_mode=nat from legacy config; coerced to 'pool' (NAT mode removed from spx admin init)." + when: + - reset_config_stat.stat.exists + - (reset_config_txt | regex_findall('external_mode\\s*=\\s*\"nat\"')) | length > 0 + - name: capture — summary ansible.builtin.debug: msg: | @@ -44,7 +59,7 @@ external_mode={{ spinifex_external_mode }} pool={{ spinifex_external_pool_start }}-{{ spinifex_external_pool_end }} gateway={{ spinifex_external_gateway }}, prefix={{ spinifex_external_prefix_len }} - nat_gateway={{ spinifex_nat_gateway_ip }} + gateway_ip={{ spinifex_external_gateway_ip }} {% else %} No existing config; using defaults from vars/defaults.yml (or -e overrides). {% endif %} diff --git a/scripts/ansible/roles/init/tasks/main.yml b/scripts/ansible/roles/init/tasks/main.yml index ff3f48b52..62f49e5fb 100644 --- a/scripts/ansible/roles/init/tasks/main.yml +++ b/scripts/ansible/roles/init/tasks/main.yml @@ -8,28 +8,38 @@ --node {{ spinifex_node_name }} --nodes {{ spinifex_node_count }} +- name: init — no-external opt-out + ansible.builtin.set_fact: + init_external_flags: "--no-external" + when: spinifex_external_no_external | bool + +- name: init — assert pool range supplied when mode=pool + ansible.builtin.assert: + that: + - spinifex_external_pool_start | length > 0 + - spinifex_external_pool_end | length > 0 + fail_msg: | + spx admin init requires --external-pool when --external-mode=pool. + Set spinifex_external_pool_start / spinifex_external_pool_end in + scripts/ansible/vars/defaults.yml, or pass: + -e spinifex_external_pool_start=192.168.1.150 + -e spinifex_external_pool_end=192.168.1.250 + Pass -e spinifex_external_no_external=true to skip external networking. + when: + - not (spinifex_external_no_external | bool) + - spinifex_external_mode == 'pool' + - name: init — pool-mode external flags ansible.builtin.set_fact: init_external_flags: >- --external-mode=pool --external-pool={{ spinifex_external_pool_start }}-{{ spinifex_external_pool_end }} - --external-gateway={{ spinifex_external_gateway }} - --external-prefix-len={{ spinifex_external_prefix_len }} + {% if spinifex_external_gateway | length > 0 %}--external-gateway={{ spinifex_external_gateway }}{% endif %} + {% if spinifex_external_prefix_len | string | length > 0 %}--external-prefix-len={{ spinifex_external_prefix_len }}{% endif %} + {% if spinifex_external_gateway_ip | length > 0 %}--gateway-ip={{ spinifex_external_gateway_ip }}{% endif %} when: + - not (spinifex_external_no_external | bool) - spinifex_external_mode == 'pool' - - spinifex_external_pool_start | length > 0 - - spinifex_external_pool_end | length > 0 - -- name: init — NAT-mode external flags - ansible.builtin.set_fact: - init_external_flags: >- - --external-mode=nat - --gateway-ip={{ spinifex_nat_gateway_ip }} - --external-gateway={{ spinifex_external_gateway }} - --external-prefix-len={{ spinifex_external_prefix_len }} - when: - - spinifex_external_mode == 'nat' - - spinifex_nat_gateway_ip | length > 0 - name: init — assemble full flag string ansible.builtin.set_fact: @@ -75,6 +85,6 @@ msg: | Init complete: region={{ spinifex_region }}, az={{ spinifex_az }}, node={{ spinifex_node_name }} - external_mode={{ spinifex_external_mode }} + external_mode={{ 'disabled' if (spinifex_external_no_external | bool) else spinifex_external_mode }} spinifex.target: started CA: {{ spinifex_ca_trust_path }} diff --git a/scripts/ansible/roles/install/tasks/main.yml b/scripts/ansible/roles/install/tasks/main.yml index 60735e70e..cde96bef5 100644 --- a/scripts/ansible/roles/install/tasks/main.yml +++ b/scripts/ansible/roles/install/tasks/main.yml @@ -57,7 +57,7 @@ - name: install — run setup-ovn.sh --management become: true ansible.builtin.command: - cmd: /usr/local/share/spinifex/setup-ovn.sh --management + cmd: "/usr/local/share/spinifex/setup-ovn.sh --management --node-name={{ spinifex_node_name }}" changed_when: true - name: install — summary diff --git a/scripts/ansible/vars/defaults.yml b/scripts/ansible/vars/defaults.yml index ad2dad7c5..244a1b4cd 100644 --- a/scripts/ansible/vars/defaults.yml +++ b/scripts/ansible/vars/defaults.yml @@ -23,12 +23,20 @@ spinifex_node_name: node1 spinifex_node_count: 1 # External network. +# spx admin init contract (as of network-layer-redesign): +# external_mode : "pool" or "" (NAT path removed) +# external_pool : MANDATORY when mode=pool — set start/end below or pass -e. +# external_gw : auto-detected from default route when blank. +# external_prefix: auto-detected from WAN when blank. +# external_gw_ip : OVN router external-IP override (was nat_gateway_ip). +# no_external : true → emit --no-external (skip external networking). spinifex_external_mode: pool +spinifex_external_no_external: false spinifex_external_pool_start: "" spinifex_external_pool_end: "" spinifex_external_gateway: "" spinifex_external_prefix_len: "" -spinifex_nat_gateway_ip: "" +spinifex_external_gateway_ip: "" # Preflight checks. spinifex_ports: @@ -80,7 +88,7 @@ mulga_root: "{{ lookup('pipe', 'readlink -f ' ~ playbook_dir ~ '/../../../..') } # `spinifex` supplementary group active. Needed because ansible's # `become_user` is a no-op when target user equals current user, so # supplementary groups are never re-initialised and /etc/spinifex/ca.pem -# (drwxr-x--- root:spinifex) is unreadable. See mulga-siv-92. +# (drwxr-x--- root:spinifex) is unreadable. spinifex_aws_cli: "{{ playbook_dir }}/../scripts/aws-as-spinifex.sh" spinifex_spx_cli: "{{ playbook_dir }}/../scripts/spx-as-spinifex.sh" diff --git a/scripts/reset-dev-env.sh b/scripts/reset-dev-env.sh index a8b5a0ac3..d0dcc8661 100755 --- a/scripts/reset-dev-env.sh +++ b/scripts/reset-dev-env.sh @@ -170,9 +170,10 @@ if [ -e /etc/systemd/network/15-spinifex-veth-wan.netdev ] || \ sudo networkctl reload 2>/dev/null || true fi -# Remove macvlan interfaces created by setup-ovn.sh +# Legacy: clean up macvlan interfaces left over from pre-veth setup-ovn.sh +# runs. Safe to remove once no operator system retains these. for iface in $(ip -o link show type macvlan 2>/dev/null | awk -F': ' '{print $2}' | grep '^spx-ext-'); do - echo " Deleting macvlan: $iface" + echo " Deleting legacy macvlan: $iface" sudo ip link del "$iface" 2>/dev/null || true done diff --git a/scripts/setup-ovn.sh b/scripts/setup-ovn.sh index 223cf4036..9c15fd49f 100755 --- a/scripts/setup-ovn.sh +++ b/scripts/setup-ovn.sh @@ -16,9 +16,7 @@ # Options: # --management Also start OVN central services (NB DB, SB DB, ovn-northd) # --wan-bridge=NAME OVS bridge for WAN traffic (default: auto-detect from default route) -# --wan-iface=NAME Physical NIC to add to the WAN bridge (use with --wan-bridge or --macvlan) -# --macvlan Create macvlan off --wan-iface instead of moving NIC directly. -# SSH-safe for single-NIC hosts where WAN NIC carries SSH. +# --wan-iface=NAME Physical NIC to add to the WAN bridge (use with --wan-bridge) # --dhcp Obtain gateway IP via DHCP on the WAN bridge interface # --mgmt-bridge=NAME OVS bridge for system-instance control plane (default: br-mgmt) # --mgmt-cidr=CIDR IPv4 CIDR to assign on the mgmt bridge (default: 10.15.8.1/24) @@ -41,9 +39,6 @@ # # Dedicated WAN NIC (not your SSH NIC — you take responsibility): # ./scripts/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=eth1 # -# # Single-NIC host (SSH-safe macvlan): -# ./scripts/setup-ovn.sh --management --macvlan --wan-iface=eth0 -# # # Compute node joining an existing cluster: # ./scripts/setup-ovn.sh --ovn-remote=tcp:10.0.0.1:6642 --encap-ip=10.0.0.2 # @@ -56,7 +51,6 @@ set -e MANAGEMENT=false WAN_BRIDGE="" WAN_IFACE="" -MACVLAN_MODE=false EXTERNAL_DHCP=false MGMT_BRIDGE_ENABLED=true MGMT_BRIDGE="br-mgmt" @@ -64,12 +58,19 @@ MGMT_CIDR="10.15.8.1/24" MGMT_IFACE="" OVN_REMOTE="tcp:127.0.0.1:6642" ENCAP_IP="" +# NODE_NAME is left empty by default. The chassis-id pin block at Step 4 +# only runs when --node-name=NAME is explicitly given. Passing nothing +# preserves whatever system-id already lives in OVS (gold-image UUID, +# bootstrap-install.sh's pre-written nodeN, manual ansible value, etc.). +# Bootstrap callers that want IPsec-cert-identity matching pin the +# system-id themselves before / after invoking setup-ovn.sh — they +# don't need this script to do it for them. +NODE_NAME="" # Parse arguments for arg in "$@"; do case "$arg" in --management) MANAGEMENT=true ;; - --macvlan) MACVLAN_MODE=true ;; --dhcp) EXTERNAL_DHCP=true ;; --wan-bridge=*) WAN_BRIDGE="${arg#*=}" ;; --wan-iface=*) WAN_IFACE="${arg#*=}" ;; @@ -79,6 +80,7 @@ for arg in "$@"; do --no-mgmt-bridge) MGMT_BRIDGE_ENABLED=false ;; --ovn-remote=*) OVN_REMOTE="${arg#*=}" ;; --encap-ip=*) ENCAP_IP="${arg#*=}" ;; + --node-name=*) NODE_NAME="${arg#*=}" ;; --help|-h) head -50 "$0" | tail -48 exit 0 @@ -92,15 +94,13 @@ done # --- WAN bridge auto-detection --- # Determine the WAN bridge name and how to set it up. -WAN_BRIDGE_MODE="" # "existing", "veth", "direct", "macvlan", or "" +WAN_BRIDGE_MODE="" # "existing", "veth", "direct", or "" LINUX_BRIDGE="" # Set when WAN_BRIDGE_MODE="veth" — the Linux bridge behind the veth pair detect_wan_bridge() { # If --wan-bridge was explicitly given, use it if [ -n "$WAN_BRIDGE" ]; then - if [ "$MACVLAN_MODE" = true ] && [ -n "$WAN_IFACE" ]; then - WAN_BRIDGE_MODE="macvlan" - elif [ -n "$WAN_IFACE" ]; then + if [ -n "$WAN_IFACE" ]; then WAN_BRIDGE_MODE="direct" elif sudo ovs-vsctl br-exists "$WAN_BRIDGE" 2>/dev/null; then WAN_BRIDGE_MODE="existing" @@ -111,17 +111,6 @@ detect_wan_bridge() { return fi - # If --macvlan was given without --wan-bridge, we need --wan-iface - if [ "$MACVLAN_MODE" = true ]; then - if [ -z "$WAN_IFACE" ]; then - echo "ERROR: --macvlan requires --wan-iface=" - exit 1 - fi - WAN_BRIDGE="br-wan" - WAN_BRIDGE_MODE="macvlan" - return - fi - # Auto-detect: find the default route interface local default_dev default_dev=$(ip -4 route show default 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="dev") print $(i+1)}' | head -1) @@ -176,10 +165,7 @@ detect_wan_bridge() { echo " 2. Dedicated WAN NIC (NOT your SSH connection):" echo " ./scripts/setup-ovn.sh --management --wan-bridge=br-wan --wan-iface=$default_dev" echo "" - echo " 3. Single-NIC host (SSH-safe macvlan):" - echo " ./scripts/setup-ovn.sh --management --macvlan --wan-iface=$default_dev" - echo "" - echo " 4. No external networking (overlay-only):" + echo " 3. No external networking (overlay-only):" echo " ./scripts/setup-ovn.sh --management --encap-ip=$wan_ip" echo "============================================================" echo "" @@ -224,32 +210,31 @@ echo " OVN Remote (SB): $OVN_REMOTE" echo " Encap IP: $ENCAP_IP" echo "" -# --- Step 1: Install packages --- -echo "Step 1: Checking OVN/OVS packages..." +# Packages (openvswitch-switch, ovn-host, openvswitch-ipsec, strongswan-charon, +# and ovn-central on management nodes) are baked into the gold image via +# scripts/tofu-cluster/image-builder/scripts/provision.sh. Runtime apt-get +# was removed to keep CI off the (flaky) apt-cacher-ng path. If a package is +# missing, the downstream ovs/ovn commands will fail loudly — the fix is to +# rebuild the gold image, not to re-add apt-get here. -install_packages() { - local missing=() - for pkg in openvswitch-switch ovn-host; do - if ! dpkg -s "$pkg" >/dev/null 2>&1; then - missing+=("$pkg") - fi - done - if [ "$MANAGEMENT" = true ]; then - if ! dpkg -s ovn-central >/dev/null 2>&1; then - missing+=("ovn-central") - fi - fi - - if [ ${#missing[@]} -gt 0 ]; then - echo " Installing: ${missing[*]}" - sudo apt-get update -qq - sudo apt-get install -y -qq "${missing[@]}" - else - echo " All packages installed" - fi -} - -install_packages +# strongswan-charon ships an AppArmor profile for /usr/lib/ipsec/charon that +# only allows reading from /etc/ipsec.*, /etc/strongswan.*, and a few other +# fixed paths. ovs-monitor-ipsec writes the per-tunnel strongSwan config with +# absolute paths to our peer cert + key under /etc/spinifex/ipsec/ AND to the +# cluster CA at /etc/spinifex/ca.pem, so charon hits "Permission denied" on +# both load paths and surfaces as `no trusted RSA public key found for +# ''` (CA can't be loaded → can't validate peer cert chain → no SAs). +# The profile includes a 'local' override file expressly for site additions +# — grant reads on /etc/spinifex/** (covers ca.pem AND ipsec/peer.{pem,key}) +# and reload. +if [ -d /etc/apparmor.d/local ]; then + LOCAL_OVERRIDE=/etc/apparmor.d/local/usr.lib.ipsec.charon + # Always rewrite — re-runs may have an older, narrower rule (e.g. the + # initial /etc/spinifex/ipsec/** only) that misses /etc/spinifex/ca.pem. + echo " Adding AppArmor read grant for /etc/spinifex (cert + CA) to charon profile" + echo "/etc/spinifex/** r," | sudo tee "$LOCAL_OVERRIDE" >/dev/null + sudo apparmor_parser -r /etc/apparmor.d/usr.lib.ipsec.charon 2>/dev/null || true +fi # --- Step 2: Enable services --- echo "" @@ -296,7 +281,6 @@ echo " br-int: created, fail-mode=secure, up" # rule => default management => no carrier => link down). OVS dataplane # still forwards, but ovn-controller flow programming and any tooling that # probes link state misbehave. Unmanaged=yes keeps OVS in sole control. -# (mulga-siv-37) OVS_INTERNAL_NET=/etc/systemd/network/05-spinifex-ovs-internal.network if [ ! -f "$OVS_INTERNAL_NET" ]; then sudo tee "$OVS_INTERNAL_NET" >/dev/null <<'NETWORK' @@ -477,36 +461,6 @@ NETWORK echo " $WAN_BRIDGE: direct bridge on $WAN_IFACE" echo " NOTE: $WAN_IFACE is now an OVS port — no host IP on this NIC" ;; - - macvlan) - # Create a macvlan sub-interface in bridge mode off the WAN NIC. - # The host keeps its IP on the parent NIC — SSH-safe. OVN localnet - # traffic flows through the macvlan to the physical wire. - if ! ip link show "$WAN_IFACE" >/dev/null 2>&1; then - echo " ERROR: interface $WAN_IFACE does not exist" - echo " Available interfaces:" - ip -o link show | awk -F': ' '{print " " $2}' - exit 1 - fi - - MACVLAN_NAME="spx-ext-${WAN_IFACE}" - - sudo ovs-vsctl --may-exist add-br "$WAN_BRIDGE" - sudo ip link set "$WAN_BRIDGE" up - - if ip link show "$MACVLAN_NAME" >/dev/null 2>&1; then - echo " macvlan $MACVLAN_NAME already exists" - else - sudo ip link add "$MACVLAN_NAME" link "$WAN_IFACE" type macvlan mode bridge - echo " created macvlan: $MACVLAN_NAME (bridge mode) on $WAN_IFACE" - fi - - sudo ip link set "$MACVLAN_NAME" up - sudo ovs-vsctl --may-exist add-port "$WAN_BRIDGE" "$MACVLAN_NAME" - echo " $WAN_BRIDGE: macvlan port $MACVLAN_NAME on $WAN_IFACE" - echo " NOTE: host keeps its IP on $WAN_IFACE (SSH-safe)" - echo " QUIRK: host cannot reach VMs at their public IPs (macvlan isolation)" - ;; esac # --- DHCP: obtain gateway IP for OVN SNAT --- @@ -514,13 +468,8 @@ NETWORK echo "" echo "Step 3c: Obtaining external gateway IP via DHCP..." - # For macvlan mode, DHCP on the macvlan interface (it has L2 access to WAN). - # For direct/existing bridge, DHCP on the bridge itself. - if [ "$WAN_BRIDGE_MODE" = "macvlan" ]; then - DHCP_IFACE="spx-ext-${WAN_IFACE}" - else - DHCP_IFACE="$WAN_BRIDGE" - fi + # DHCP on the WAN bridge itself (direct/existing/veth). + DHCP_IFACE="$WAN_BRIDGE" # Run DHCP client to get a lease if command -v dhcpcd >/dev/null 2>&1; then @@ -629,11 +578,33 @@ else external_ids:ovn-encap-type="geneve" fi -# system-id is owned by the openvswitch-switch package (persisted in -# /etc/openvswitch/system-id.conf and re-applied on every boot). Read it back -# rather than overriding — overriding here would drift from the on-disk value -# and the next reboot would silently flip the chassis identity (mulga-999). -echo " system-id: $(sudo ovs-vsctl get open . external_ids:system-id)" +# Pin OVS system-id (= OVN chassis-id) to NODE_NAME when --node-name was +# explicitly given. Reasons to pin: +# 1. IPsec identity: ovs-monitor-ipsec uses chassis-id as the IKEv2 +# `@` peer identity. Our per-node IPsec peer cert +# (admin.GenerateIPSecPeerCert) carries the cluster node name as CN +# + dnsName SAN — see spx admin init/join, which take --node NAME and +# bake NAME into the cert. Leaving chassis-id as the package-generated +# UUID would cause `received AUTHENTICATION_FAILED` because charon +# validates `@` against a cert dnsName=. +# 2. Ops legibility: `ovn-sbctl show` lists chassis by name, not UUID. +# +# When --node-name is NOT given (CI bootstrap-install.sh today, dev +# single-node bring-up, ansible roles that don't yet pass it), leave the +# system-id alone. A hostname fallback used to live here, but on CI +# single-node it rewrote system-id from the gold-image UUID to a +# different hostname while the existing SBDB chassis row kept name=UUID; +# vpcd's discoverChassis then logged "skipping stale local chassis" and +# refused to start. Preserving the on-disk value is always safe — any +# caller that needs IPsec cert identity matching pins it themselves. +if [ -n "$NODE_NAME" ]; then + echo "$NODE_NAME" | sudo tee /etc/openvswitch/system-id.conf >/dev/null + sudo ovs-vsctl set Open_vSwitch . external_ids:system-id="$NODE_NAME" + echo " system-id: $NODE_NAME (pinned via --node-name)" +else + CURRENT_ID=$(sudo ovs-vsctl get Open_vSwitch . external_ids:system-id 2>/dev/null | tr -d '"') + echo " system-id: ${CURRENT_ID:-} (preserved; no --node-name given)" +fi echo " ovn-remote: $OVN_REMOTE" echo " ovn-encap-ip: $ENCAP_IP" echo " ovn-encap-type: geneve" @@ -779,8 +750,20 @@ echo "" echo "Step 10: Enabling OVN auto-start on boot..." sudo systemctl enable openvswitch-switch 2>/dev/null || true sudo systemctl enable ovn-controller 2>/dev/null || true -echo " openvswitch-switch: enabled on boot" -echo " ovn-controller: enabled on boot" +# ovs-monitor-ipsec drives strongSwan from OVS DB cert pointers. The daemon's +# enableOVNIPSec() flips ipsec_encapsulation=true at runtime and silently drops +# tunnel traffic if this unit isn't already up — enable at provision time so +# daemon never needs systemd-write capability (only is-active read). +sudo systemctl enable openvswitch-ipsec.service 2>/dev/null || true +# When openvswitch-ipsec is installed before strongswan-starter, the deb +# post-install starts the service against a missing /usr/sbin/ipsec and +# leaves it in 'failed' state. Clear the failure and restart unconditionally +# now that strongswan-starter is present. +sudo systemctl reset-failed openvswitch-ipsec.service 2>/dev/null || true +sudo systemctl restart openvswitch-ipsec.service 2>/dev/null || true +echo " openvswitch-switch: enabled on boot" +echo " ovn-controller: enabled on boot" +echo " openvswitch-ipsec: enabled on boot" # --- Step 11: Health check --- echo "" @@ -815,14 +798,6 @@ if [ -n "$WAN_BRIDGE" ]; then echo " direct bridge: FAILED ($WAN_IFACE not on $WAN_BRIDGE)" OK=false fi - elif [ "$WAN_BRIDGE_MODE" = "macvlan" ]; then - MACVLAN_NAME="spx-ext-${WAN_IFACE}" - if ip link show "$MACVLAN_NAME" >/dev/null 2>&1; then - echo " macvlan: OK ($MACVLAN_NAME)" - else - echo " macvlan: FAILED ($MACVLAN_NAME not found)" - OK=false - fi fi else echo " $WAN_BRIDGE:$(printf '%*s' $((15 - ${#WAN_BRIDGE})) '') FAILED" diff --git a/scripts/setup.sh b/scripts/setup.sh index 9abeef2bd..612ba007b 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -186,6 +186,8 @@ install_sudoers() { spinifex-daemon ALL=(root) NOPASSWD: /sbin/ip, /usr/sbin/ip spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/ovs-vsctl, /usr/bin/ovs-appctl spinifex-daemon ALL=(root) NOPASSWD: /usr/sbin/dhcpcd +spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/systemctl is-active openvswitch-ipsec.service +spinifex-daemon ALL=(root) NOPASSWD: /usr/bin/ovn-nbctl set NB_Global . ipsec=true # Spinifex VPC daemon: OVN and OVS read/write, OVN controller status check and DHCP spinifex-vpcd ALL=(root) NOPASSWD: /usr/sbin/dhcpcd @@ -218,7 +220,7 @@ install_apt_deps() { libvirt-daemon-system libvirt-clients \ pciutils \ jq curl iproute2 netcat-openbsd wget unzip xz-utils file \ - ovn-central ovn-host openvswitch-switch dhcpcd-base \ + ovn-central ovn-host openvswitch-switch openvswitch-ipsec strongswan-charon dhcpcd-base \ > /dev/null info "System dependencies installed" diff --git a/spinifex/admin/admin.go b/spinifex/admin/admin.go index 130c9f9c9..f803f8a64 100644 --- a/spinifex/admin/admin.go +++ b/spinifex/admin/admin.go @@ -72,14 +72,12 @@ type ConfigSettings struct { // External networking for public subnets ExternalMode string // "pool" or "" (disabled) ExternalIface string // WAN NIC name (e.g., "eth0", "eth1") - DhcpBindBridge string // Bridge where the DHCP AF_PACKET socket binds (Linux bridge in veth mode, OVS bridge in direct mode; never "br-ext") - ExternalDHCP bool // Obtain gateway IP via DHCP on macvlan/bridge PoolName string // External pool name (e.g., "wan") - PoolSource string // IP source: "static" (default) or "dhcp" (from router DHCP) - PoolStart string // First IP in external pool range (static source only) - PoolEnd string // Last IP in external pool range (static source only) + PoolSource string // IP source: "static" (default; only valid value) + PoolStart string // First IP in external pool range + PoolEnd string // Last IP in external pool range PoolGateway string // WAN gateway IP - PoolGatewayIP string // Explicit SNAT IP (for nat mode without DHCP) + PoolGatewayIP string // Explicit SNAT IP (overrides default of first IP in range) PoolPrefixLen int // Subnet prefix length (default 24) PoolDNSServers []string // DNS servers for VM DHCP (auto-detected from host) @@ -104,6 +102,11 @@ type ConfigSettings struct { // GPUPassthrough enables VFIO GPU passthrough in the daemon config. // Sets gpu_passthrough = true under [nodes..daemon]. GPUPassthrough bool + + // IPSecEnabled toggles cluster-wide OVN native IPsec on intra-AZ Geneve + // (default true). Surfaced under [network] in spinifex.toml so the value + // is visible/auditable; the daemon reads it via cluster config. + IPSecEnabled bool } // PredastoreNodeConfig describes a single Predastore node for multi-node config generation. diff --git a/spinifex/admin/ipsec.go b/spinifex/admin/ipsec.go new file mode 100644 index 000000000..c08acf63e --- /dev/null +++ b/spinifex/admin/ipsec.go @@ -0,0 +1,235 @@ +package admin + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" + "encoding/pem" + "fmt" + "log/slog" + "math/big" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// IPSec peer-cert layout under configDir. ovs-monitor-ipsec consumes these +// paths via Open_vSwitch.other_config (certificate, private_key, ca_cert) +// set by the daemon at startup when network.ipsec_enabled is true. +const ( + ipsecDirName = "ipsec" + ipsecCertFileName = "peer.pem" + ipsecKeyFileName = "peer.key" + ipsecCACertSymlink = "ca.pem" +) + +// id-kp-ipsecIKE per RFC 4945 §5.1.3.12. strongSwan accepts certs carrying +// this EKU as IKEv2 peer credentials; ovs-monitor-ipsec's auto-generated +// strongSwan config consumes them as-is. +var oidExtKeyUsageIPSecIKE = asn1.ObjectIdentifier{1, 3, 6, 1, 5, 5, 7, 3, 17} + +// IPSecCertPaths returns the canonical paths for the per-node IPsec peer +// credential pair under configDir. Both paths live in +// {configDir}/ipsec/. The CA trust anchor is the existing cluster CA in +// configDir/ca.pem; ovs-monitor-ipsec is pointed at that path directly. +func IPSecCertPaths(configDir string) (certPath, keyPath string) { + dir := filepath.Join(configDir, ipsecDirName) + return filepath.Join(dir, ipsecCertFileName), filepath.Join(dir, ipsecKeyFileName) +} + +// GenerateIPSecPeerCert issues an IKEv2 peer certificate for this node, +// signed by the existing cluster CA. The cert carries the IPSec-IKE +// extended key usage so strongSwan accepts it as a peer credential. +// +// hostname is the node's primary hostname (used as Common Name + DNS SAN). +// nodeIP is the node's primary cluster IP (added as an IP SAN). Additional +// SANs discovered from the local machine's interfaces are not added here — +// IPSec peer identity should be the explicit cluster identity only, not +// every interface address, to keep peer-identity matching strict. +// +// Cross-cutting invariant: this dnsName SAN must equal the OVS chassis-id +// (Open_vSwitch.external_ids:system-id) on every node, because +// ovs-monitor-ipsec generates strongSwan configs with leftid=@ +// / rightid=@ and charon validates those names against the +// peer cert's dnsName SAN. setup-ovn.sh pins system-id to `hostname -s` +// for exactly this reason — see the comment in setup-ovn.sh near +// `system-id.conf`. Changing the cert identity here without updating +// setup-ovn.sh (or vice versa) breaks IKE with AUTHENTICATION_FAILED. +func GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, hostname, nodeIP string) error { + if hostname == "" { + return fmt.Errorf("ipsec peer cert: hostname required") + } + if nodeIP == "" { + return fmt.Errorf("ipsec peer cert: nodeIP required") + } + parsedIP := net.ParseIP(nodeIP) + if parsedIP == nil { + return fmt.Errorf("ipsec peer cert: invalid nodeIP %q", nodeIP) + } + + caCert, caKey, err := loadCAKeyPair(caCertPath, caKeyPath) + if err != nil { + return err + } + + peerKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return fmt.Errorf("ipsec peer cert: generate key: %w", err) + } + + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return fmt.Errorf("ipsec peer cert: serial: %w", err) + } + + notBefore := time.Now() + template := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{ + CommonName: hostname, + Organization: []string{"Spinifex Platform"}, + }, + NotBefore: notBefore, + NotAfter: notBefore.Add(365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + UnknownExtKeyUsage: []asn1.ObjectIdentifier{oidExtKeyUsageIPSecIKE}, + BasicConstraintsValid: true, + DNSNames: []string{hostname}, + IPAddresses: []net.IP{parsedIP}, + } + + der, err := x509.CreateCertificate(rand.Reader, &template, caCert, &peerKey.PublicKey, caKey) + if err != nil { + return fmt.Errorf("ipsec peer cert: create certificate: %w", err) + } + + dir := filepath.Join(configDir, ipsecDirName) + if err := os.MkdirAll(dir, 0750); err != nil { + return fmt.Errorf("ipsec peer cert: mkdir %s: %w", dir, err) + } + + certPath, keyPath := IPSecCertPaths(configDir) + if err := writePEMFile(certPath, "CERTIFICATE", der, 0644); err != nil { + return fmt.Errorf("ipsec peer cert: write cert: %w", err) + } + + keyBytes, err := x509.MarshalPKCS8PrivateKey(peerKey) + if err != nil { + return fmt.Errorf("ipsec peer cert: marshal key: %w", err) + } + if err := writePEMFile(keyPath, "PRIVATE KEY", keyBytes, 0600); err != nil { + return fmt.Errorf("ipsec peer cert: write key: %w", err) + } + + // Symlink the cluster CA into charon's default cacerts scan dir. + // ovs-monitor-ipsec writes per-conn `cacert=` references, but + // strongSwan 6.0 only loads CAs declared in standalone `ca` sections + // or files placed in /etc/ipsec.d/cacerts at startup. Without this, + // charon authenticates the local side OK but rejects every inbound + // peer cert as `no trusted RSA public key found for ''` and + // no SAs negotiate. The dir is on charon's default AppArmor allowlist + // so no profile change is needed (unlike /etc/spinifex/). + if err := installCAIntoCharonTrustStore(caCertPath); err != nil { + return fmt.Errorf("ipsec peer cert: install CA into charon trust store: %w", err) + } + + return nil +} + +// charonCATrustDir is the strongSwan default cacerts scan dir. Overridden +// in tests so they don't pollute the real /etc/ipsec.d/cacerts. +var charonCATrustDir = "/etc/ipsec.d/cacerts" + +const charonCATrustLink = "spinifex-ca.pem" + +func installCAIntoCharonTrustStore(caCertPath string) error { + if err := os.MkdirAll(charonCATrustDir, 0750); err != nil { + return fmt.Errorf("mkdir %s: %w", charonCATrustDir, err) + } + link := filepath.Join(charonCATrustDir, charonCATrustLink) + tmp, err := os.MkdirTemp(charonCATrustDir, ".cainstall-") + if err != nil { + return fmt.Errorf("tmpdir: %w", err) + } + defer os.RemoveAll(tmp) + staging := filepath.Join(tmp, charonCATrustLink) + if err := os.Symlink(caCertPath, staging); err != nil { + return fmt.Errorf("symlink %s -> %s: %w", staging, caCertPath, err) + } + if err := os.Rename(staging, link); err != nil { + return fmt.Errorf("rename %s -> %s: %w", staging, link, err) + } + + // strongSwan scans /etc/ipsec.d/cacerts only at charon startup. If + // openvswitch-ipsec.service started before this install (the common + // case — the unit is enabled at provision time), charon is running + // with zero CAs loaded and every IKE_AUTH ends in `no trusted RSA + // public key found for ''`. Trigger a stroke rereadcacerts so + // the symlink we just placed is loaded without bouncing charon. + // Silent failure is expected when charon isn't running yet (admin + // init can run before setup-ovn.sh enables the unit); a later charon + // start will scan cacerts naturally. + if err := charonRereadCAs(); err != nil { + slog.Debug("ipsec: rereadcacerts skipped", "err", err) + } + return nil +} + +// charonRereadCAs triggers strongSwan to re-scan /etc/ipsec.d/cacerts so a +// freshly-symlinked CA is picked up by an already-running charon. Overridden +// in tests to avoid execing ipsec. +var charonRereadCAs = func() error { + out, err := exec.Command("/usr/sbin/ipsec", "rereadcacerts").CombinedOutput() + if err != nil { + return fmt.Errorf("ipsec rereadcacerts: %s: %w", strings.TrimSpace(string(out)), err) + } + return nil +} + +func loadCAKeyPair(caCertPath, caKeyPath string) (*x509.Certificate, *rsa.PrivateKey, error) { + certPEM, err := os.ReadFile(caCertPath) + if err != nil { + return nil, nil, fmt.Errorf("read CA cert %s: %w", caCertPath, err) + } + block, _ := pem.Decode(certPEM) + if block == nil { + return nil, nil, fmt.Errorf("decode CA cert PEM at %s", caCertPath) + } + caCert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("parse CA cert: %w", err) + } + + keyPEM, err := os.ReadFile(caKeyPath) + if err != nil { + return nil, nil, fmt.Errorf("read CA key %s: %w", caKeyPath, err) + } + keyBlock, _ := pem.Decode(keyPEM) + if keyBlock == nil { + return nil, nil, fmt.Errorf("decode CA key PEM at %s", caKeyPath) + } + caKey, err := x509.ParsePKCS8PrivateKey(keyBlock.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("parse CA key: %w", err) + } + rsaKey, ok := caKey.(*rsa.PrivateKey) + if !ok { + return nil, nil, fmt.Errorf("CA key is not RSA") + } + return caCert, rsaKey, nil +} + +func writePEMFile(path, blockType string, der []byte, mode os.FileMode) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return err + } + defer f.Close() + return pem.Encode(f, &pem.Block{Type: blockType, Bytes: der}) +} diff --git a/spinifex/admin/ipsec_test.go b/spinifex/admin/ipsec_test.go new file mode 100644 index 000000000..faefbc3ee --- /dev/null +++ b/spinifex/admin/ipsec_test.go @@ -0,0 +1,189 @@ +package admin + +import ( + "crypto/x509" + "encoding/pem" + "net" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestGenerateIPSecPeerCert shares a CA across subtests so the 4096-bit +// CA-key generation runs once (~0.7s). +func TestGenerateIPSecPeerCert(t *testing.T) { + // Redirect the charon trust-store symlink at a temp dir so the test + // doesn't try to write to /etc/ipsec.d/cacerts on the host. Restored + // in cleanup. Cannot use t.Parallel() with this var override. + origTrustDir := charonCATrustDir + charonCATrustDir = t.TempDir() + t.Cleanup(func() { charonCATrustDir = origTrustDir }) + + // Stub the ipsec rereadcacerts shell-out so tests don't depend on + // strongSwan being installed/running on the test host. + origReread := charonRereadCAs + charonRereadCAs = func() error { return nil } + t.Cleanup(func() { charonRereadCAs = origReread }) + + caDir := t.TempDir() + caCertPath := filepath.Join(caDir, "ca.pem") + caKeyPath := filepath.Join(caDir, "ca.key") + require.NoError(t, GenerateCACert(caCertPath, caKeyPath)) + + t.Run("happy path", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require.NoError(t, GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-1", "10.0.0.5")) + + certPath, keyPath := IPSecCertPaths(dir) + certPEM, err := os.ReadFile(certPath) + require.NoError(t, err) + block, _ := pem.Decode(certPEM) + require.NotNil(t, block) + cert, err := x509.ParseCertificate(block.Bytes) + require.NoError(t, err) + + assert.Equal(t, "node-1", cert.Subject.CommonName) + assert.Equal(t, []string{"node-1"}, cert.DNSNames) + require.Len(t, cert.IPAddresses, 1) + assert.True(t, cert.IPAddresses[0].Equal(net.ParseIP("10.0.0.5"))) + + // IPSec EKU must be present so strongSwan accepts the cert as an IKEv2 peer. + var hasIPSecIKE bool + for _, oid := range cert.UnknownExtKeyUsage { + if oid.Equal(oidExtKeyUsageIPSecIKE) { + hasIPSecIKE = true + break + } + } + assert.True(t, hasIPSecIKE, "cert missing id-kp-ipsecIKE EKU") + + caCertPEM, _ := os.ReadFile(caCertPath) + caBlock, _ := pem.Decode(caCertPEM) + caCert, _ := x509.ParseCertificate(caBlock.Bytes) + pool := x509.NewCertPool() + pool.AddCert(caCert) + _, err = cert.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}}) + assert.NoError(t, err, "cert must verify against cluster CA") + + info, err := os.Stat(keyPath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0600), info.Mode().Perm(), "key must be 0600") + }) + + t.Run("idempotent overwrite", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + require.NoError(t, GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-2", "10.0.0.6")) + require.NoError(t, GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-2", "10.0.0.7")) + + certPath, _ := IPSecCertPaths(dir) + certPEM, _ := os.ReadFile(certPath) + block, _ := pem.Decode(certPEM) + cert, _ := x509.ParseCertificate(block.Bytes) + assert.True(t, cert.IPAddresses[0].Equal(net.ParseIP("10.0.0.7")), "second call should rotate the SAN IP") + }) + + t.Run("rejects missing hostname", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + err := GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "", "10.0.0.5") + require.Error(t, err) + assert.Contains(t, err.Error(), "hostname required") + }) + + t.Run("rejects missing nodeIP", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + err := GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-1", "") + require.Error(t, err) + assert.Contains(t, err.Error(), "nodeIP required") + }) + + t.Run("rejects invalid nodeIP", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + err := GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-1", "not-an-ip") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid nodeIP") + }) + + t.Run("missing CA file", func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + err := GenerateIPSecPeerCert(dir, filepath.Join(dir, "ghost-ca.pem"), caKeyPath, "node-1", "10.0.0.5") + require.Error(t, err) + assert.Contains(t, err.Error(), "read CA cert") + }) +} + +func TestIPSecCertPaths(t *testing.T) { + t.Parallel() + certPath, keyPath := IPSecCertPaths("/etc/spinifex") + assert.Equal(t, "/etc/spinifex/ipsec/peer.pem", certPath) + assert.Equal(t, "/etc/spinifex/ipsec/peer.key", keyPath) +} + +// TestInstallCAIntoCharonTrustStore_TriggersRereadCAs locks the fix for the +// race where openvswitch-ipsec started before admin init's CA install: every +// IKE_AUTH failed with `no trusted RSA public key found for ''` because +// charon had scanned cacerts before the symlink was placed. The trust-store +// installer must trigger `ipsec rereadcacerts` so an already-running charon +// picks up the CA without a restart. +func TestInstallCAIntoCharonTrustStore_TriggersRereadCAs(t *testing.T) { + origTrustDir := charonCATrustDir + charonCATrustDir = t.TempDir() + t.Cleanup(func() { charonCATrustDir = origTrustDir }) + + origReread := charonRereadCAs + var called int + charonRereadCAs = func() error { + called++ + return nil + } + t.Cleanup(func() { charonRereadCAs = origReread }) + + caDir := t.TempDir() + caCertPath := filepath.Join(caDir, "ca.pem") + require.NoError(t, os.WriteFile(caCertPath, []byte("dummy"), 0644)) + + require.NoError(t, installCAIntoCharonTrustStore(caCertPath)) + assert.Equal(t, 1, called, "rereadcacerts must be invoked exactly once after symlink placement") + + link := filepath.Join(charonCATrustDir, charonCATrustLink) + target, err := os.Readlink(link) + require.NoError(t, err) + assert.Equal(t, caCertPath, target, "symlink target must match CA cert path") +} + +// TestInstallCAIntoCharonTrustStore_SwallowsRereadError verifies that a failed +// rereadcacerts (e.g. charon not running, ipsec binary missing on a fresh +// install) does not break admin init. The CA symlink must still be in place so +// a later charon start picks it up via the normal cacerts scan. +func TestInstallCAIntoCharonTrustStore_SwallowsRereadError(t *testing.T) { + origTrustDir := charonCATrustDir + charonCATrustDir = t.TempDir() + t.Cleanup(func() { charonCATrustDir = origTrustDir }) + + origReread := charonRereadCAs + charonRereadCAs = func() error { return stubError("charon not running") } + t.Cleanup(func() { charonRereadCAs = origReread }) + + caDir := t.TempDir() + caCertPath := filepath.Join(caDir, "ca.pem") + require.NoError(t, os.WriteFile(caCertPath, []byte("dummy"), 0644)) + + require.NoError(t, installCAIntoCharonTrustStore(caCertPath), + "reread failure must not propagate; admin init has to succeed even when charon is offline") + + link := filepath.Join(charonCATrustDir, charonCATrustLink) + _, err := os.Readlink(link) + require.NoError(t, err, "symlink must persist even when reread fails") +} + +type stubError string + +func (e stubError) Error() string { return string(e) } diff --git a/spinifex/config/config.go b/spinifex/config/config.go index 9d18fba18..75860481b 100644 --- a/spinifex/config/config.go +++ b/spinifex/config/config.go @@ -3,6 +3,7 @@ package config import ( "fmt" "log/slog" + "net/netip" "os" "slices" "strings" @@ -22,7 +23,7 @@ type ClusterConfig struct { // ExternalPool defines a range of routable IPs that Spinifex manages for public subnets. type ExternalPool struct { Name string `mapstructure:"name"` // Pool identifier (e.g., "wan", "dc1-primary") - Source string `mapstructure:"source"` // IP source: "static" (default) or "dhcp" (from router DHCP) + Source string `mapstructure:"source"` // IP source: "static" (default; only valid value) RangeStart string `mapstructure:"range_start"` // First IP in range (static source only) RangeEnd string `mapstructure:"range_end"` // Last IP in range (static source only) Gateway string `mapstructure:"gateway"` // WAN default gateway (next hop for 0.0.0.0/0) @@ -32,11 +33,11 @@ type ExternalPool struct { Region string `mapstructure:"region"` // Scope to region (optional — empty means any region) AZ string `mapstructure:"az"` // Scope to AZ (optional — empty means any AZ in region) // GwLrpRangeStart/End reserve a sub-range of the LAN for OVN gateway LRP IPs - // in centralized NAT mode (veth/macvlan). Each VPC consumes one IP from this + // in centralized NAT mode (veth). Each VPC consumes one IP from this // range so its gateway router port can ARP with a sender IP on the LAN // subnet — link-local 169.254.0.1/30 makes upstream routers reject ARP per - // RFC 826 (mulga-siv-36). Must NOT overlap [RangeStart, RangeEnd] or IPAM - // and vpcd will fight over the same IP. + // RFC 826. Must NOT overlap [RangeStart, RangeEnd] or IPAM and vpcd will + // fight over the same IP. GwLrpRangeStart string `mapstructure:"gw_lrp_range_start"` GwLrpRangeEnd string `mapstructure:"gw_lrp_range_end"` } @@ -44,8 +45,11 @@ type ExternalPool struct { // NetworkConfig holds cluster-wide external network settings. type NetworkConfig struct { ExternalMode string `mapstructure:"external_mode"` // "pool" or "" (disabled) - ExternalDHCP bool `mapstructure:"external_dhcp"` // Gateway IP obtained via DHCP (pool/dhcp source) ExternalPools []ExternalPool `mapstructure:"external_pools"` // One or more IP pools + // IPSecEnabled toggles OVN native IPsec on every node for intra-AZ Geneve + // (AES-256-GCM via strongSwan). Default true. Operators can flip off for + // trusted single-rack lab topologies; production edge stays on. + IPSecEnabled bool `mapstructure:"ipsec_enabled"` } // BootstrapConfig holds the default VPC infrastructure IDs written by admin init. @@ -103,14 +107,7 @@ type VPCDConfig struct { OVNNBAddr string `json:"OVNNBAddr" mapstructure:"ovn_nb_addr"` // OVN Northbound DB address (e.g., "tcp:127.0.0.1:6641") OVNSBAddr string `json:"OVNSBAddr" mapstructure:"ovn_sb_addr"` // OVN Southbound DB address (e.g., "tcp:127.0.0.1:6642") ExternalInterface string `json:"ExternalInterface" mapstructure:"external_interface"` // WAN NIC name (e.g., "eth1", "enp0s3") — the physical NIC on the WAN bridge - // DhcpBindBridge is the bridge where the DHCP client binds its AF_PACKET - // socket — the interface that physically sees LAN DHCP traffic. On hosts - // where the WAN NIC is enslaved to a Linux bridge (netplan default), this - // is the Linux bridge name (e.g. "br-wan"). On direct-OVS hosts, it is - // the OVS bridge holding the WAN NIC. Never set to the OVN-side bridge - // ("br-ext") — that never sees LAN DHCP traffic. - DhcpBindBridge string `json:"DhcpBindBridge" mapstructure:"dhcp_bind_bridge"` - BridgeMode string `json:"BridgeMode" mapstructure:"bridge_mode"` // "direct" or "veth" (auto-detected if empty) + BridgeMode string `json:"BridgeMode" mapstructure:"bridge_mode"` // "direct" or "veth" (auto-detected if empty) } type PredastoreConfig struct { @@ -212,6 +209,10 @@ func LoadConfig(configPath string) (*ClusterConfig, error) { viper.SetEnvPrefix("SPINIFEX") viper.AutomaticEnv() + // network.ipsec_enabled defaults to true (intra-AZ Geneve encrypted via OVN + // native IPsec). Operators must explicitly set false to disable. + viper.SetDefault("network.ipsec_enabled", true) + // Try to load config file if it exists if configPath != "" { // Check if file exists @@ -244,5 +245,66 @@ func LoadConfig(configPath string) (*ClusterConfig, error) { } } + if err := validateClusterConfig(&config); err != nil { + return nil, err + } + return &config, nil } + +// validateClusterConfig rejects legacy upstream-DHCP config and validates +// external pool ranges. The upstream-DHCP-client model was replaced by static +// WAN-pool allocation; legacy keys must be removed from the operator's TOML. +func validateClusterConfig(cc *ClusterConfig) error { + if viper.IsSet("network.external_dhcp") { + return fmt.Errorf("config: [network] external_dhcp is no longer supported; remove the key (static WAN-pool allocation only)") + } + for nodeName := range cc.Nodes { + if viper.IsSet("nodes." + nodeName + ".vpcd.dhcp_bind_bridge") { + return fmt.Errorf("config: [nodes.%s.vpcd] dhcp_bind_bridge is no longer supported; remove the key (vpcd no longer runs a DHCP client)", nodeName) + } + } + + type poolRange struct { + name string + start netip.Addr + end netip.Addr + } + var ranges []poolRange + for _, p := range cc.Network.ExternalPools { + if p.Source != "" && p.Source != "static" { + return fmt.Errorf("config: [[network.external_pools]] %q: source=%q is no longer supported; use source=\"static\" with explicit range_start/range_end", p.Name, p.Source) + } + if p.RangeStart == "" || p.RangeEnd == "" { + continue + } + start, err := netip.ParseAddr(p.RangeStart) + if err != nil { + return fmt.Errorf("config: pool %q range_start %q: %w", p.Name, p.RangeStart, err) + } + end, err := netip.ParseAddr(p.RangeEnd) + if err != nil { + return fmt.Errorf("config: pool %q range_end %q: %w", p.Name, p.RangeEnd, err) + } + if start.Compare(end) > 0 { + return fmt.Errorf("config: pool %q range_start %s > range_end %s", p.Name, start, end) + } + if p.Gateway != "" && p.PrefixLen > 0 { + gw, err := netip.ParseAddr(p.Gateway) + if err != nil { + return fmt.Errorf("config: pool %q gateway %q: %w", p.Name, p.Gateway, err) + } + cidr := netip.PrefixFrom(gw, p.PrefixLen).Masked() + if !cidr.Contains(start) || !cidr.Contains(end) { + return fmt.Errorf("config: pool %q range [%s, %s] not inside %s", p.Name, start, end, cidr) + } + } + for _, prior := range ranges { + if start.Compare(prior.end) <= 0 && prior.start.Compare(end) <= 0 { + return fmt.Errorf("config: pool %q range [%s, %s] overlaps pool %q [%s, %s]", p.Name, start, end, prior.name, prior.start, prior.end) + } + } + ranges = append(ranges, poolRange{name: p.Name, start: start, end: end}) + } + return nil +} diff --git a/spinifex/config/config_test.go b/spinifex/config/config_test.go index 6c2a9844d..fefea1cf2 100644 --- a/spinifex/config/config_test.go +++ b/spinifex/config/config_test.go @@ -375,7 +375,49 @@ external_interface = "enp0s3" assert.Equal(t, "enp0s3", n.VPCD.ExternalInterface) } -func TestLoadConfig_NetworkPoolDHCPMode(t *testing.T) { +func TestLoadConfig_NetworkIPSecEnabledDefaultsTrue(t *testing.T) { + resetViper(t) + dir := t.TempDir() + path := filepath.Join(dir, "spinifex.toml") + + // No [network] block at all — IPSec must default to true so AWS-parity + // edge deployments encrypt intra-AZ Geneve without operator opt-in. + toml := ` +node = "n1" + +[nodes.n1] +region = "us-east-1" +` + require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) + + cfg, err := LoadConfig(path) + require.NoError(t, err) + assert.True(t, cfg.Network.IPSecEnabled, "default") +} + +func TestLoadConfig_NetworkIPSecEnabledExplicitFalse(t *testing.T) { + resetViper(t) + dir := t.TempDir() + path := filepath.Join(dir, "spinifex.toml") + + // Operator escape hatch for trusted single-rack lab deployments. + toml := ` +node = "n1" + +[network] +ipsec_enabled = false + +[nodes.n1] +region = "us-east-1" +` + require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) + + cfg, err := LoadConfig(path) + require.NoError(t, err) + assert.False(t, cfg.Network.IPSecEnabled) +} + +func TestLoadConfig_NetworkPoolDHCPSourceRejected(t *testing.T) { resetViper(t) dir := t.TempDir() path := filepath.Join(dir, "spinifex.toml") @@ -399,12 +441,117 @@ region = "us-east-1" require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) cfg, err := LoadConfig(path) - require.NoError(t, err) + require.Error(t, err) + assert.Nil(t, cfg) + assert.Contains(t, err.Error(), "source=") + assert.Contains(t, err.Error(), "dhcp") +} - assert.Equal(t, "pool", cfg.Network.ExternalMode) - require.Len(t, cfg.Network.ExternalPools, 1) - assert.Equal(t, "dhcp", cfg.Network.ExternalPools[0].Source) - assert.Equal(t, "192.168.1.100", cfg.Network.ExternalPools[0].GatewayIP) +func TestLoadConfig_ExternalDHCPRejected(t *testing.T) { + resetViper(t) + dir := t.TempDir() + path := filepath.Join(dir, "spinifex.toml") + + toml := ` +node = "n1" + +[network] +external_mode = "pool" +external_dhcp = true + +[nodes.n1] +region = "us-east-1" +` + require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) + + cfg, err := LoadConfig(path) + require.Error(t, err) + assert.Nil(t, cfg) + assert.Contains(t, err.Error(), "external_dhcp") +} + +func TestLoadConfig_DhcpBindBridgeRejected(t *testing.T) { + resetViper(t) + dir := t.TempDir() + path := filepath.Join(dir, "spinifex.toml") + + toml := ` +node = "n1" + +[nodes.n1.vpcd] +dhcp_bind_bridge = "br-wan" +` + require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) + + cfg, err := LoadConfig(path) + require.Error(t, err) + assert.Nil(t, cfg) + assert.Contains(t, err.Error(), "dhcp_bind_bridge") +} + +func TestLoadConfig_PoolRangeOutsideCIDR(t *testing.T) { + resetViper(t) + dir := t.TempDir() + path := filepath.Join(dir, "spinifex.toml") + + toml := ` +node = "n1" + +[network] +external_mode = "pool" + +[[network.external_pools]] +name = "wan" +range_start = "10.99.0.10" +range_end = "10.99.0.50" +gateway = "192.168.1.1" +prefix_len = 24 + +[nodes.n1] +region = "us-east-1" +` + require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) + + cfg, err := LoadConfig(path) + require.Error(t, err) + assert.Nil(t, cfg) + assert.Contains(t, err.Error(), "not inside") +} + +func TestLoadConfig_PoolsOverlap(t *testing.T) { + resetViper(t) + dir := t.TempDir() + path := filepath.Join(dir, "spinifex.toml") + + toml := ` +node = "n1" + +[network] +external_mode = "pool" + +[[network.external_pools]] +name = "a" +range_start = "192.168.1.10" +range_end = "192.168.1.50" +gateway = "192.168.1.1" +prefix_len = 24 + +[[network.external_pools]] +name = "b" +range_start = "192.168.1.40" +range_end = "192.168.1.80" +gateway = "192.168.1.1" +prefix_len = 24 + +[nodes.n1] +region = "us-east-1" +` + require.NoError(t, os.WriteFile(path, []byte(toml), 0600)) + + cfg, err := LoadConfig(path) + require.Error(t, err) + assert.Nil(t, cfg) + assert.Contains(t, err.Error(), "overlaps") } func TestLoadConfig_NetworkDisabledByDefault(t *testing.T) { diff --git a/spinifex/daemon/daemon.go b/spinifex/daemon/daemon.go index d36f22498..ed77933b5 100644 --- a/spinifex/daemon/daemon.go +++ b/spinifex/daemon/daemon.go @@ -49,6 +49,7 @@ import ( handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2" "github.com/mulgadc/spinifex/spinifex/instancetypes" + "github.com/mulgadc/spinifex/spinifex/network/host" "github.com/mulgadc/spinifex/spinifex/objectstore" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" @@ -343,7 +344,7 @@ func (d *Daemon) onNATSReconnect(_ *nats.Conn) { } // execCommand wraps exec.Command so tests can substitute a fake builder. -// Mirrors the sudoCommand seam in network.go — getSystemMemory shells out to +// Mirrors the utils.SudoCommand seam — getSystemMemory shells out to // sysctl (darwin) or grep /proc/meminfo (linux), neither of which can be // faked without an indirection that the test can swap. var execCommand = exec.Command @@ -890,7 +891,7 @@ func (d *Daemon) startLocal() error { if d.config.Daemon.MgmtBridge != "" { mgmtBridge = d.config.Daemon.MgmtBridge } - bridgeIP, bridgeErr := GetBridgeIPv4(mgmtBridge) + bridgeIP, bridgeErr := host.GetBridgeIPv4(mgmtBridge) if bridgeErr != nil { slog.Warn("Management bridge not detected, system instances will not get mgmt NIC", "bridge", mgmtBridge, "err", bridgeErr) } else if bridgeIP == "" { @@ -908,7 +909,7 @@ func (d *Daemon) startLocal() error { // Initialise OVS network plumber (no NATS dep). if d.networkPlumber == nil { - d.networkPlumber = &OVSNetworkPlumber{} + d.networkPlumber = host.NewOVSPlumber() } // Protect daemon from OOM killer (prefer killing QEMU VMs instead). @@ -1019,6 +1020,24 @@ func (d *Daemon) startCluster() error { return fmt.Errorf("initialize JetStream: %w", err) } + // Tombstone the spinifex-dhcp-leases bucket left over from the upstream-DHCP + // client removed in Phase 2.3. Idempotent — first daemon start on an upgraded + // cluster sweeps the bucket; later restarts are no-ops. + if js, jsErr := d.natsConn.JetStream(); jsErr == nil { + if err := utils.DeleteKVBucketIfExists(js, "spinifex-dhcp-leases"); err != nil { + slog.Warn("Failed to delete obsolete spinifex-dhcp-leases KV bucket", "err", err) + } + } + + // Enable OVN native IPsec on intra-AZ Geneve when the cluster flag is set. + // Idempotent — ovs-monitor-ipsec materialises strongSwan configs from the + // cert pointers each time ovn-controller programs a tunnel. + if d.clusterConfig != nil && d.clusterConfig.Network.IPSecEnabled { + if err := host.EnableOVNIPSec(d.configPath, d.clusterConfig); err != nil { + slog.Warn("Failed to enable OVN native IPsec; intra-AZ Geneve will be plaintext", "err", err) + } + } + // Write service manifest so other nodes know what this node runs if d.jsManager != nil { if err := d.jsManager.WriteServiceManifest( @@ -1103,19 +1122,9 @@ func (d *Daemon) startCluster() error { slog.Warn("Failed to get JetStream for external IPAM", "err", jsErr) } else { var pools []handlers_ec2_vpc.ExternalPoolConfig - // Resolve DHCP bind bridge name for DHCP pools (where AF_PACKET binds). - dhcpBindBridge := "" - if node, ok := d.clusterConfig.Nodes[d.clusterConfig.Node]; ok { - dhcpBindBridge = node.VPCD.DhcpBindBridge - } - gwMAC := "" - if d.clusterConfig.Bootstrap.VpcId != "" { - gwMAC = utils.HashMAC("gw-" + d.clusterConfig.Bootstrap.VpcId) - } for _, p := range d.clusterConfig.Network.ExternalPools { pools = append(pools, handlers_ec2_vpc.ExternalPoolConfig{ Name: p.Name, - Source: p.Source, RangeStart: p.RangeStart, RangeEnd: p.RangeEnd, Gateway: p.Gateway, @@ -1123,13 +1132,11 @@ func (d *Daemon) startCluster() error { PrefixLen: p.PrefixLen, Region: p.Region, AZ: p.AZ, - DhcpBindBridge: dhcpBindBridge, - GatewayMAC: gwMAC, GwLrpRangeStart: p.GwLrpRangeStart, GwLrpRangeEnd: p.GwLrpRangeEnd, }) } - d.externalIPAM, err = handlers_ec2_vpc.NewExternalIPAM(d.natsConn, js, pools) + d.externalIPAM, err = handlers_ec2_vpc.NewExternalIPAM(js, pools) if err != nil { slog.Warn("Failed to initialize external IPAM", "err", err) } else { @@ -1563,7 +1570,7 @@ func (d *Daemon) ClusterManager() error { } // Check OVN networking readiness - ovnHealth := CheckOVNHealth() + ovnHealth := host.HealthStatus() if ovnHealth.BrIntExists { serviceHealth["br-int"] = "ok" } else { @@ -1792,7 +1799,7 @@ func (d *Daemon) setupShutdown() { // Cancel context to stop heartbeat and other goroutines d.cancel() - // DDIL Tier 1 (mulga-siv-58): SIGTERM alone never stops local VMs. + // DDIL Tier 1: SIGTERM alone never stops local VMs. // `systemctl restart spinifex-daemon` must preserve every QEMU child so // the new daemon can reattach via the persisted local state file // (Scenario B). VMs are only stopped when coordinated cluster shutdown diff --git a/spinifex/daemon/daemon_handlers_test.go b/spinifex/daemon/daemon_handlers_test.go index 0a5c4b9ac..4554e0fd1 100644 --- a/spinifex/daemon/daemon_handlers_test.go +++ b/spinifex/daemon/daemon_handlers_test.go @@ -3548,7 +3548,7 @@ func TestDelegateHandlers_EIP(t *testing.T) { js, err := nc.JetStream() require.NoError(t, err) - ipam, err := handlers_ec2_vpc.NewExternalIPAM(nc, js, []handlers_ec2_vpc.ExternalPoolConfig{ + ipam, err := handlers_ec2_vpc.NewExternalIPAM(js, []handlers_ec2_vpc.ExternalPoolConfig{ {Name: "test-pool", RangeStart: "192.168.100.2", RangeEnd: "192.168.100.254", Gateway: "192.168.100.1", PrefixLen: 24}, }) require.NoError(t, err) diff --git a/spinifex/daemon/daemon_system_instance.go b/spinifex/daemon/daemon_system_instance.go index 830cdf670..948c20f80 100644 --- a/spinifex/daemon/daemon_system_instance.go +++ b/spinifex/daemon/daemon_system_instance.go @@ -10,7 +10,9 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" handlers_elbv2 "github.com/mulgadc/spinifex/spinifex/handlers/elbv2" + "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/tags" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" @@ -209,7 +211,7 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) region = d.config.Region az = d.config.AZ } - allocatedIP, poolName, allocErr := d.externalIPAM.AllocateIP(region, az, "auto_assign", "", instance.ENIId, instance.ID) + allocatedIP, poolName, allocErr := d.externalIPAM.AllocateIP(region, az, handlers_ec2_vpc.PurposeENIPublic, "", instance.ENIId, instance.ID) if allocErr != nil { slog.Error("LaunchSystemInstance: failed to allocate public IP for internet-facing ALB", "instanceId", instance.ID, "err", allocErr) d.cleanupFailedSystemInstance(instance, instanceType) @@ -226,7 +228,7 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) if descErr == nil && len(result.NetworkInterfaces) > 0 && result.NetworkInterfaces[0].VpcId != nil { vpcID = *result.NetworkInterfaces[0].VpcId } - portName := "port-" + instance.ENIId + portName := topology.Port(instance.ENIId) if natErr := utils.AddNAT(d.natsConn, vpcID, publicIP, privateIP, portName, instance.ENIMac); natErr != nil { slog.Error("LaunchSystemInstance: vpc.add-nat failed for ALB public IP — rolling back to avoid surfacing an unreachable address", "instanceId", instance.ID, "publicIp", publicIP, "pool", poolName, "err", natErr) @@ -290,7 +292,7 @@ func (d *Daemon) LaunchSystemInstance(input *handlers_elbv2.SystemInstanceInput) } slog.Warn("LaunchSystemInstance: failed to allocate mgmt IP, skipping mgmt NIC", "instanceId", instance.ID, "err", allocErr) } else { - instance.MgmtMAC = generateMgmtMAC(instance.ID) + instance.MgmtMAC = vm.GenerateMgmtMAC(instance.ID) instance.MgmtIP = mgmtIP // Inject the allocated MAC and CIDR into NIC[1] so writeFwCfgBlobs diff --git a/spinifex/daemon/daemon_system_instance_test.go b/spinifex/daemon/daemon_system_instance_test.go index 424ec8555..c9e5893d2 100644 --- a/spinifex/daemon/daemon_system_instance_test.go +++ b/spinifex/daemon/daemon_system_instance_test.go @@ -485,7 +485,7 @@ func TestLaunchSystemInstance_NATFailureRollsBackPublicIP(t *testing.T) { js, err := jsNC.JetStream() require.NoError(t, err) - ipam, err := handlers_ec2_vpc.NewExternalIPAM(jsNC, js, []handlers_ec2_vpc.ExternalPoolConfig{ + ipam, err := handlers_ec2_vpc.NewExternalIPAM(js, []handlers_ec2_vpc.ExternalPoolConfig{ {Name: "wan-test", RangeStart: "203.0.113.10", RangeEnd: "203.0.113.20", Gateway: "203.0.113.1", PrefixLen: 24}, }) require.NoError(t, err) diff --git a/spinifex/daemon/network.go b/spinifex/daemon/network.go deleted file mode 100644 index 9367194a5..000000000 --- a/spinifex/daemon/network.go +++ /dev/null @@ -1,320 +0,0 @@ -package daemon - -import ( - "fmt" - "log/slog" - "net" - "os" - "os/exec" - "sort" - "strconv" - "strings" - - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/mulgadc/spinifex/spinifex/vm" -) - -// sudoCommand wraps exec.Command with sudo when running as non-root. -// OVS/OVN and ip commands require elevated privileges; in Docker and -// production the daemon runs as root, but in dev environments it may not. -// -// Bound to a var (not a plain func) so tests can swap in a stub — running -// the live binary against the dev host's OVS would mutate `external_ids` on -// the running cluster (see TestSetupComputeNode_ValidatesArgs). -var sudoCommand = func(name string, args ...string) *exec.Cmd { - if os.Getuid() == 0 { - return exec.Command(name, args...) - } - return exec.Command("sudo", append([]string{name}, args...)...) -} - -// OVSNetworkPlumber implements vm.NetworkPlumber using system commands -// (ip, ovs-vsctl); tests use a mock satisfying the same interface. -type OVSNetworkPlumber struct{} - -var _ vm.NetworkPlumber = (*OVSNetworkPlumber)(nil) - -// SetupTap creates the kernel tap, brings it up, and attaches it to spec.Bridge. -// The pre-create del-port is unconditional because OVS conf.db survives reboot -// while kernel taps don't, so a /sys/class/net check would miss orphan ports. -// Rejected alternative: `--may-exist add-port` — it would silently keep stale -// external_ids from a prior launch with a different ENI/MAC. -func (p *OVSNetworkPlumber) SetupTap(spec vm.TapSpec) error { - if err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", spec.Bridge, spec.Name).Run(); err != nil { - slog.Warn("Pre-create del-port failed (continuing)", "tap", spec.Name, "bridge", spec.Bridge, "err", err) - } - if _, err := os.Stat("/sys/class/net/" + spec.Name); err == nil { - if err := sudoCommand("ip", "tuntap", "del", "dev", spec.Name, "mode", "tap").Run(); err != nil { - slog.Warn("Pre-create tap del failed (continuing)", "tap", spec.Name, "err", err) - } - } - - // When the daemon runs non-root (production: User=spinifex-daemon), child - // QEMU processes inherit that uid without CAP_NET_ADMIN. The kernel's tun - // attach check then requires the tap to be owned by the calling euid - // (drivers/net/tun.c: tun_chr_open + TUNSETIFF perm check). Without the - // `user` flag the tap defaults to root:root and qemu's TUNSETIFF fails - // with EPERM ("could not configure /dev/net/tun: Operation not permitted"). - // Pass numeric uid/gid directly — user.Current() can fail silently under - // hardened systemd units / static builds, dropping the flags and leaving - // the tap root-owned (LB microvm in cell-17/tofu-examples). Numeric values - // always work and skip the NSS lookup. Root daemons skip this — they have - // CAP_NET_ADMIN ambient. - addArgs := []string{"tuntap", "add", "dev", spec.Name, "mode", "tap"} - if uid := os.Geteuid(); uid != 0 { - addArgs = append(addArgs, "user", strconv.Itoa(uid), "group", strconv.Itoa(os.Getegid())) - } - if out, err := sudoCommand("ip", addArgs...).CombinedOutput(); err != nil { - return fmt.Errorf("create tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err) - } - - if out, err := sudoCommand("ip", "link", "set", spec.Name, "up").CombinedOutput(); err != nil { - if cleanErr := sudoCommand("ip", "tuntap", "del", "dev", spec.Name, "mode", "tap").Run(); cleanErr != nil { - slog.Warn("Failed to clean up tap after bring-up failure", "tap", spec.Name, "err", cleanErr) - } - return fmt.Errorf("bring up tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err) - } - - addPortArgs := []string{"add-port", spec.Bridge, spec.Name} - if len(spec.ExternalIDs) > 0 { - addPortArgs = append(addPortArgs, "--", "set", "Interface", spec.Name) - // Sort keys for deterministic command construction (test assertions, log readability). - keys := make([]string, 0, len(spec.ExternalIDs)) - for k := range spec.ExternalIDs { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - addPortArgs = append(addPortArgs, fmt.Sprintf("external_ids:%s=%s", k, spec.ExternalIDs[k])) - } - } - if out, err := sudoCommand("ovs-vsctl", addPortArgs...).CombinedOutput(); err != nil { - if cleanErr := sudoCommand("ip", "tuntap", "del", "dev", spec.Name, "mode", "tap").Run(); cleanErr != nil { - slog.Warn("Failed to clean up tap after OVS failure", "tap", spec.Name, "err", cleanErr) - } - return fmt.Errorf("add tap %s to %s: %s: %w", spec.Name, spec.Bridge, strings.TrimSpace(string(out)), err) - } - - slog.Info("Tap plumbing complete", "tap", spec.Name, "bridge", spec.Bridge, "external_ids", spec.ExternalIDs) - return nil -} - -// CleanupTap removes the named tap from OVS (any bridge) and the kernel. -// Idempotent: callers may invoke for an instance that never reached SetupTap -// (e.g. terminate that races mid-launch). -func (p *OVSNetworkPlumber) CleanupTap(name string) error { - if out, err := sudoCommand("ovs-vsctl", "--if-exists", "del-port", name).CombinedOutput(); err != nil { - slog.Warn("Failed to remove tap from OVS", "tap", name, "err", err, "out", strings.TrimSpace(string(out))) - } - - if _, err := os.Stat("/sys/class/net/" + name); os.IsNotExist(err) { - slog.Info("Tap already absent", "tap", name) - return nil - } - - if out, err := sudoCommand("ip", "tuntap", "del", "dev", name, "mode", "tap").CombinedOutput(); err != nil { - return fmt.Errorf("delete tap %s: %s: %w", name, strings.TrimSpace(string(out)), err) - } - - slog.Info("Tap cleanup complete", "tap", name) - return nil -} - -// generateMgmtMAC creates a locally-administered unicast MAC for the -// management NIC. The "mgmt:" tag disambiguates from the dev NIC of the -// same instance (which shares instanceId). -func generateMgmtMAC(instanceId string) string { - return utils.HashMAC("mgmt:" + instanceId) -} - -// GetBridgeIPv4 returns the first IPv4 address on the named Linux bridge. -// Returns "", nil if the bridge does not exist. -func GetBridgeIPv4(bridgeName string) (string, error) { - iface, err := net.InterfaceByName(bridgeName) - if err != nil { - // "no such network interface" means the bridge doesn't exist yet — expected. - // Other errors (permission denied on /sys/class/net/, etc.) are real failures. - if strings.Contains(err.Error(), "no such network interface") { - return "", nil - } - return "", fmt.Errorf("lookup %s: %w", bridgeName, err) - } - addrs, err := iface.Addrs() - if err != nil { - return "", fmt.Errorf("list addrs on %s: %w", bridgeName, err) - } - for _, addr := range addrs { - ipNet, ok := addr.(*net.IPNet) - if !ok { - continue - } - if ipNet.IP.To4() != nil { - return ipNet.IP.String(), nil - } - } - return "", fmt.Errorf("no IPv4 address on %s", bridgeName) -} - -// OVNHealthStatus reports the readiness of OVN networking on this compute node. -type OVNHealthStatus struct { - BrIntExists bool `json:"br_int_exists"` - OVNControllerUp bool `json:"ovn_controller_up"` - ChassisID string `json:"chassis_id,omitempty"` - EncapIP string `json:"encap_ip,omitempty"` - OVNRemote string `json:"ovn_remote,omitempty"` -} - -// CheckOVNHealth probes local OVS/OVN state to determine network readiness. -func CheckOVNHealth() OVNHealthStatus { - status := OVNHealthStatus{} - - // Check br-int exists - if err := sudoCommand("ovs-vsctl", "br-exists", "br-int").Run(); err == nil { - status.BrIntExists = true - } - - // Check ovn-controller is running via ovs-appctl. Trixie OVN moved the - // pid file to /var/run/ovn/; older installs use /var/run/openvswitch/. - // Probe at runtime so the same binary works on both layouts. - ovnTarget := "ovn-controller" - if _, err := os.Stat("/var/run/ovn/ovn-controller.pid"); err == nil { - ovnTarget = "/var/run/ovn/ovn-controller" - } - if out, err := sudoCommand("ovs-appctl", "-t", ovnTarget, "version").CombinedOutput(); err == nil && len(out) > 0 { - status.OVNControllerUp = true - } - - // Read chassis identity from OVS external_ids - if out, err := sudoCommand("ovs-vsctl", "get", "Open_vSwitch", ".", "external_ids:system-id").CombinedOutput(); err == nil { - status.ChassisID = strings.Trim(strings.TrimSpace(string(out)), "\"") - } - if out, err := sudoCommand("ovs-vsctl", "get", "Open_vSwitch", ".", "external_ids:ovn-encap-ip").CombinedOutput(); err == nil { - status.EncapIP = strings.Trim(strings.TrimSpace(string(out)), "\"") - } - if out, err := sudoCommand("ovs-vsctl", "get", "Open_vSwitch", ".", "external_ids:ovn-remote").CombinedOutput(); err == nil { - status.OVNRemote = strings.Trim(strings.TrimSpace(string(out)), "\"") - } - - return status -} - -// SetupComputeNode configures OVS for OVN on this compute node. -// It creates br-int with secure fail-mode and sets the OVN external_ids. -func SetupComputeNode(chassisID, ovnRemote, encapIP string) error { - // Create br-int if it doesn't exist - if out, err := sudoCommand("ovs-vsctl", "--may-exist", "add-br", "br-int").CombinedOutput(); err != nil { - return fmt.Errorf("create br-int: %s: %w", strings.TrimSpace(string(out)), err) - } - - // Set fail-mode=secure (preserves flows during ovn-controller restart) - if out, err := sudoCommand("ovs-vsctl", "set", "Bridge", "br-int", "fail-mode=secure").CombinedOutput(); err != nil { - return fmt.Errorf("set br-int fail-mode: %s: %w", strings.TrimSpace(string(out)), err) - } - - // Disable in-band management (prevents OVS from adding its own flows) - if out, err := sudoCommand("ovs-vsctl", "set", "Bridge", "br-int", "other-config:disable-in-band=true").CombinedOutput(); err != nil { - return fmt.Errorf("set br-int disable-in-band: %s: %w", strings.TrimSpace(string(out)), err) - } - - // Bring br-int up - if out, err := sudoCommand("ip", "link", "set", "br-int", "up").CombinedOutput(); err != nil { - return fmt.Errorf("bring up br-int: %s: %w", strings.TrimSpace(string(out)), err) - } - - // Set OVN external_ids on the Open_vSwitch table - if out, err := sudoCommand("ovs-vsctl", "set", "Open_vSwitch", ".", - fmt.Sprintf("external_ids:system-id=%s", chassisID), - fmt.Sprintf("external_ids:ovn-remote=%s", ovnRemote), - fmt.Sprintf("external_ids:ovn-encap-ip=%s", encapIP), - "external_ids:ovn-encap-type=geneve", - ).CombinedOutput(); err != nil { - return fmt.Errorf("set OVN external_ids: %s: %w", strings.TrimSpace(string(out)), err) - } - - slog.Info("OVN compute node configured", - "chassis_id", chassisID, - "ovn_remote", ovnRemote, - "encap_ip", encapIP, - ) - - // Ensure the data NIC is preferred for Geneve tunnel routing. - if err := EnsureDataRoute(encapIP); err != nil { - slog.Warn("Failed to configure data NIC routing (Geneve tunnels may use wrong source IP)", "err", err) - } - - return nil -} - -// EnsureDataRoute ensures the kernel routes Geneve tunnel traffic through the -// data NIC (the interface holding the encap IP). When management and data NICs -// share the same subnet with equal route metrics, the kernel may pick the -// management NIC, causing Geneve packets to have the wrong source IP. Remote -// OVS nodes then drop these packets because the source doesn't match the -// expected tunnel remote_ip. -// -// Fix: find the data NIC's subnet route and replace it with a lower metric (50) -// so it's preferred over the management NIC's route (typically metric 100+). -func EnsureDataRoute(encapIP string) error { - dataIface, err := findInterfaceByIP(encapIP) - if err != nil { - return fmt.Errorf("find data interface for %s: %w", encapIP, err) - } - - // Read the existing subnet route for the data interface - out, err := sudoCommand("ip", "-o", "-4", "route", "show", "dev", dataIface, "proto", "kernel", "scope", "link").CombinedOutput() - if err != nil { - return fmt.Errorf("read routes for %s: %s: %w", dataIface, strings.TrimSpace(string(out)), err) - } - - // Parse the subnet CIDR from the route output (e.g. "10.1.0.0/16 dev eth1 ...") - fields := strings.Fields(strings.TrimSpace(string(out))) - if len(fields) == 0 { - return fmt.Errorf("no kernel route found for %s", dataIface) - } - subnet := fields[0] - - // Replace the route with a lower metric so the data NIC is preferred - if out, err := sudoCommand("ip", "route", "replace", subnet, - "dev", dataIface, "src", encapIP, "metric", "50", - ).CombinedOutput(); err != nil { - return fmt.Errorf("replace route for %s: %s: %w", subnet, strings.TrimSpace(string(out)), err) - } - - slog.Info("Data NIC route configured for Geneve tunnels", - "interface", dataIface, - "subnet", subnet, - "encap_ip", encapIP, - "metric", 50, - ) - return nil -} - -// findInterfaceByIP returns the network interface name that holds the given IP address. -func findInterfaceByIP(ipAddr string) (string, error) { - targetIP := net.ParseIP(ipAddr) - if targetIP == nil { - return "", fmt.Errorf("invalid IP address: %s", ipAddr) - } - - ifaces, err := net.Interfaces() - if err != nil { - return "", fmt.Errorf("list interfaces: %w", err) - } - - for _, iface := range ifaces { - addrs, err := iface.Addrs() - if err != nil { - continue - } - for _, addr := range addrs { - ipNet, ok := addr.(*net.IPNet) - if !ok { - continue - } - if ipNet.IP.Equal(targetIP) { - return iface.Name, nil - } - } - } - return "", fmt.Errorf("no interface found with IP %s", ipAddr) -} diff --git a/spinifex/daemon/network_test.go b/spinifex/daemon/network_test.go deleted file mode 100644 index 83d2f24c2..000000000 --- a/spinifex/daemon/network_test.go +++ /dev/null @@ -1,538 +0,0 @@ -package daemon - -import ( - "net" - "os" - "os/exec" - "slices" - "strings" - "testing" - - "github.com/mulgadc/spinifex/spinifex/vm" - "github.com/stretchr/testify/require" -) - -func TestTapDeviceName(t *testing.T) { - tests := []struct { - eniId string - expected string - }{ - {"eni-abc123", "tapabc123"}, // Short ID - {"eni-abc123def456789", "tapabc123def456"}, // Truncated to 15 chars - {"eni-a", "tapa"}, // Minimal - {"eni-123456789abcdef", "tap123456789abc"}, // Exactly 15 chars - {"eni-123456789abcdefg", "tap123456789abc"}, // Truncated at 15 - } - - for _, tt := range tests { - t.Run(tt.eniId, func(t *testing.T) { - got := vm.TapDeviceName(tt.eniId) - if got != tt.expected { - t.Errorf("vm.TapDeviceName(%q) = %q, want %q", tt.eniId, got, tt.expected) - } - if len(got) > 15 { - t.Errorf("vm.TapDeviceName(%q) = %q (len %d), exceeds IFNAMSIZ limit of 15", tt.eniId, got, len(got)) - } - }) - } -} - -func TestSudoCommand_NonRoot(t *testing.T) { - // When not root, sudoCommand should prefix with sudo - cmd := sudoCommand("ovs-vsctl", "br-exists", "br-int") - args := cmd.Args - - if os.Getuid() == 0 { - // Running as root: should NOT use sudo - if args[0] != "ovs-vsctl" { - t.Errorf("as root, expected args[0]='ovs-vsctl', got %q", args[0]) - } - } else { - // Running as non-root: should use sudo - if args[0] != "sudo" { - t.Errorf("as non-root, expected args[0]='sudo', got %q", args[0]) - } - if args[1] != "ovs-vsctl" { - t.Errorf("as non-root, expected args[1]='ovs-vsctl', got %q", args[1]) - } - if len(args) != 4 { - t.Errorf("expected 4 args [sudo ovs-vsctl br-exists br-int], got %d: %v", len(args), args) - } - } -} - -func TestGenerateDevMAC(t *testing.T) { - tests := []struct { - instanceId string - }{ - {"i-abc123"}, - {"i-def456"}, - {"i-ghi789"}, - } - - // All MACs must be valid locally-administered unicast and unique. First - // octet is hash-derived (not the literal class prefix of the old impl). - seen := make(map[string]bool) - for _, tt := range tests { - mac := vm.GenerateDevMAC(tt.instanceId) - hw, err := net.ParseMAC(mac) - if err != nil { - t.Errorf("vm.GenerateDevMAC(%q) = %q: invalid MAC: %v", tt.instanceId, mac, err) - continue - } - if hw[0]&0x03 != 0x02 { - t.Errorf("vm.GenerateDevMAC(%q) = %q: expected unicast+LAA bits, got %#x", - tt.instanceId, mac, hw[0]) - } - if seen[mac] { - t.Errorf("vm.GenerateDevMAC(%q) = %q, duplicate MAC", tt.instanceId, mac) - } - seen[mac] = true - } - - // Class separation: dev and mgmt MACs for the same instance must differ. - if vm.GenerateDevMAC("i-abc123") == generateMgmtMAC("i-abc123") { - t.Error("expected dev and mgmt MACs for same instance to differ") - } - - // Same input should produce same output (deterministic) - mac1 := vm.GenerateDevMAC("i-test123") - mac2 := vm.GenerateDevMAC("i-test123") - if mac1 != mac2 { - t.Errorf("generateDevMAC not deterministic: %q != %q", mac1, mac2) - } -} - -func TestFindInterfaceByIP_InvalidIP(t *testing.T) { - _, err := findInterfaceByIP("not-an-ip") - if err == nil { - t.Fatal("expected error for invalid IP") - } - if !strings.Contains(err.Error(), "invalid IP address") { - t.Errorf("expected 'invalid IP address' error, got: %v", err) - } -} - -func TestFindInterfaceByIP_Loopback(t *testing.T) { - // 127.0.0.1 should always be on the loopback interface - iface, err := findInterfaceByIP("127.0.0.1") - if err != nil { - t.Fatalf("findInterfaceByIP(127.0.0.1): %v", err) - } - if iface != "lo" { - t.Errorf("findInterfaceByIP(127.0.0.1) = %q, want 'lo'", iface) - } -} - -func TestFindInterfaceByIP_NotFound(t *testing.T) { - _, err := findInterfaceByIP("192.0.2.1") - if err == nil { - t.Fatal("expected error for non-existent IP") - } - if !strings.Contains(err.Error(), "no interface found") { - t.Errorf("expected 'no interface found' error, got: %v", err) - } -} - -// TestOVSNetworkPlumber_SetupTap_AddPortArgs captures the ovs-vsctl invocations -// for both VPC-style (populated ExternalIDs → `set Interface external_ids:k=v`) -// and management-style (empty ExternalIDs → bare `add-port`) calls. This is the -// functional difference that previously lived in two diverged setup functions. -func TestOVSNetworkPlumber_SetupTap_AddPortArgs(t *testing.T) { - cases := []struct { - name string - spec vm.TapSpec - wantAddPort []string // expected tail of ovs-vsctl args (after add-port ) - }{ - { - name: "vpc style with external_ids", - spec: vm.TapSpec{ - Name: "tapeni-test", - Bridge: "br-int", - ExternalIDs: map[string]string{ - "iface-id": "port-eni-test", - "attached-mac": "02:00:00:aa:bb:cc", - }, - }, - wantAddPort: []string{ - "add-port", "br-int", "tapeni-test", - "--", "set", "Interface", "tapeni-test", - "external_ids:attached-mac=02:00:00:aa:bb:cc", - "external_ids:iface-id=port-eni-test", - }, - }, - { - name: "mgmt style without external_ids", - spec: vm.TapSpec{ - Name: "mg-test", - Bridge: "br-mgmt", - }, - wantAddPort: []string{"add-port", "br-mgmt", "mg-test"}, - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - orig := sudoCommand - t.Cleanup(func() { sudoCommand = orig }) - - var calls [][]string - sudoCommand = func(name string, args ...string) *exec.Cmd { - call := append([]string{name}, args...) - calls = append(calls, call) - return exec.Command("/bin/true") - } - - p := &OVSNetworkPlumber{} - if err := p.SetupTap(tc.spec); err != nil { - t.Fatalf("SetupTap: %v", err) - } - - // Find the add-port invocation (last ovs-vsctl call). - var addPort []string - for _, c := range calls { - if len(c) >= 2 && c[0] == "ovs-vsctl" && c[1] != "--if-exists" { - addPort = c[1:] - } - } - if addPort == nil { - t.Fatalf("no add-port invocation captured; calls=%v", calls) - } - if !slices.Equal(addPort, tc.wantAddPort) { - t.Errorf("add-port args = %v, want %v", addPort, tc.wantAddPort) - } - }) - } -} - -// TestOVSNetworkPlumber_SetupTap_PreExistingKernelTap exercises the -// pre-create kernel-side cleanup branch (lines guarded by /sys/class/net/ -// stat). Uses "lo" as a name that's always present so os.Stat returns success. -func TestOVSNetworkPlumber_SetupTap_PreExistingKernelTap(t *testing.T) { - orig := sudoCommand - t.Cleanup(func() { sudoCommand = orig }) - - failTuntapDel := false - sudoCommand = func(name string, args ...string) *exec.Cmd { - if name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "del" && failTuntapDel { - failTuntapDel = false - return exec.Command("/bin/false") - } - return exec.Command("/bin/true") - } - - failTuntapDel = true - p := &OVSNetworkPlumber{} - if err := p.SetupTap(vm.TapSpec{Name: "lo", Bridge: "br-int"}); err != nil { - t.Fatalf("SetupTap with stubs should succeed: %v", err) - } -} - -// TestOVSNetworkPlumber_SetupTap_ErrorBranches stubs sudoCommand to fail at -// each step in turn and asserts the corresponding error is returned. Covers -// the cleanup-on-failure paths that the success-path tests miss. -func TestOVSNetworkPlumber_SetupTap_ErrorBranches(t *testing.T) { - cases := []struct { - name string - failMatch func(name string, args []string) bool - wantErr string - }{ - { - name: "ovs del-port failure is logged-warn (continues)", - failMatch: func(name string, args []string) bool { - return name == "ovs-vsctl" && len(args) >= 1 && args[0] == "--if-exists" - }, - wantErr: "", // pre-clear failure is swallowed; SetupTap returns nil - }, - { - name: "tuntap add failure surfaces", - failMatch: func(name string, args []string) bool { - return name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "add" - }, - wantErr: "create tap", - }, - { - name: "ip link set up failure surfaces and triggers tap cleanup", - failMatch: func(name string, args []string) bool { - return name == "ip" && len(args) >= 1 && args[0] == "link" - }, - wantErr: "bring up tap", - }, - { - name: "ovs add-port failure surfaces and triggers tap cleanup", - failMatch: func(name string, args []string) bool { - return name == "ovs-vsctl" && len(args) >= 1 && args[0] == "add-port" - }, - wantErr: "add tap", - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - orig := sudoCommand - t.Cleanup(func() { sudoCommand = orig }) - - sudoCommand = func(name string, args ...string) *exec.Cmd { - if tc.failMatch(name, args) { - return exec.Command("/bin/false") - } - return exec.Command("/bin/true") - } - - p := &OVSNetworkPlumber{} - err := p.SetupTap(vm.TapSpec{Name: "tap-test-noexist", Bridge: "br-int"}) - if tc.wantErr == "" { - if err != nil { - t.Fatalf("expected nil, got %v", err) - } - return - } - if err == nil || !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("expected error containing %q, got %v", tc.wantErr, err) - } - }) - } -} - -// TestOVSNetworkPlumber_CleanupTap_PresentKernelTap exercises the success -// path where the kernel device is present (uses "lo" so /sys/class/net/lo -// stat succeeds). Asserts no error and that ip tuntap del was attempted. -func TestOVSNetworkPlumber_CleanupTap_PresentKernelTap(t *testing.T) { - orig := sudoCommand - t.Cleanup(func() { sudoCommand = orig }) - - var sawTuntapDel bool - sudoCommand = func(name string, args ...string) *exec.Cmd { - if name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "del" { - sawTuntapDel = true - } - return exec.Command("/bin/true") - } - - p := &OVSNetworkPlumber{} - if err := p.CleanupTap("lo"); err != nil { - t.Fatalf("CleanupTap: %v", err) - } - if !sawTuntapDel { - t.Errorf("expected ip tuntap del to be invoked, got no call") - } -} - -// TestOVSNetworkPlumber_CleanupTap_DelPortLoggedWarn covers the OVS del-port -// failure branch (logged warn, continues to kernel-side cleanup). -func TestOVSNetworkPlumber_CleanupTap_DelPortLoggedWarn(t *testing.T) { - orig := sudoCommand - t.Cleanup(func() { sudoCommand = orig }) - - sudoCommand = func(name string, args ...string) *exec.Cmd { - if name == "ovs-vsctl" { - return exec.Command("/bin/false") - } - return exec.Command("/bin/true") - } - - p := &OVSNetworkPlumber{} - // Tap doesn't exist on this test system → kernel-presence gate short-circuits - // after the OVS warn, so CleanupTap returns nil. - if err := p.CleanupTap("tap-test-noexist"); err != nil { - t.Fatalf("CleanupTap: %v", err) - } -} - -// TestOVSNetworkPlumber_CleanupTap_TuntapDelFailure exercises the kernel-side -// failure branch (real kernel device present + ip tuntap del returns nonzero). -func TestOVSNetworkPlumber_CleanupTap_TuntapDelFailure(t *testing.T) { - orig := sudoCommand - t.Cleanup(func() { sudoCommand = orig }) - - sudoCommand = func(name string, args ...string) *exec.Cmd { - if name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "del" { - return exec.Command("/bin/false") - } - return exec.Command("/bin/true") - } - - p := &OVSNetworkPlumber{} - err := p.CleanupTap("lo") // /sys/class/net/lo is present, so we reach tuntap del - if err == nil || !strings.Contains(err.Error(), "delete tap") { - t.Fatalf("expected 'delete tap' error, got %v", err) - } -} - -// TestOVSNetworkPlumber_CleanupTap_MissingKernelTap verifies the nil-safe branch: -// callers may invoke CleanupTap on a name that has neither an OVS port nor a -// kernel tap (e.g. a terminate that races mid-launch, or an instance that -// never reached SetupTap) without producing a misleading "Device does not -// exist" error from `ip tuntap del`. -func TestOVSNetworkPlumber_CleanupTap_MissingKernelTap(t *testing.T) { - // 15-char IFNAMSIZ-compliant name that does not exist in /sys/class/net. - // The OVS del-port call may fail on CI without ovs-vsctl, but is - // best-effort + logged-warn; the kernel-presence gate short-circuits - // before `ip tuntap del` runs. - p := &OVSNetworkPlumber{} - if err := p.CleanupTap("mg-test-noexist"); err != nil { - t.Fatalf("expected nil for missing kernel tap, got: %v", err) - } -} - -func TestEnsureDataRoute_NoKernelRoute(t *testing.T) { - // 127.0.0.1 resolves to "lo", which has no kernel/scope-link subnet route. - // EnsureDataRoute must surface this as an error rather than silently - // succeed against a non-existent route — a regression that drops the - // error would leave Geneve traffic egressing the wrong NIC. - err := EnsureDataRoute("127.0.0.1") - require.Error(t, err, "expected error for IP without a kernel subnet route") - require.ErrorContains(t, err, "no kernel route found", - "error must identify the missing-route condition, not a generic interface lookup failure") -} - -func TestSetupComputeNode_ValidatesArgs(t *testing.T) { - // Stub sudoCommand so the test never shells out to the host's real - // ovs-vsctl. Without this stub, on a dev box with OVS installed the call - // silently mutated external_ids:system-id (and ovn-remote, ovn-encap-ip) - // on the live cluster, breaking vpcd's chassis discovery until reboot. - orig := sudoCommand - t.Cleanup(func() { sudoCommand = orig }) - sudoCommand = func(string, ...string) *exec.Cmd { - return exec.Command("/bin/false") - } - - if err := SetupComputeNode("chassis-test", "tcp:127.0.0.1:6642", "10.0.0.1"); err == nil { - t.Fatal("expected error from stubbed sudoCommand, got nil") - } -} - -func TestGenerateDevMAC_Format(t *testing.T) { - tests := []string{ - "i-abc123", - "i-def456", - "i-00000000", - "i-ffffffff", - "i-a", - "i-very-long-instance-id-with-many-characters", - } - for _, id := range tests { - mac := vm.GenerateDevMAC(id) - hw, err := net.ParseMAC(mac) - if err != nil { - t.Errorf("vm.GenerateDevMAC(%q) = %q: invalid MAC: %v", id, mac, err) - continue - } - if hw[0]&0x03 != 0x02 { - t.Errorf("vm.GenerateDevMAC(%q) = %q: expected unicast+LAA bits, got %#x", - id, mac, hw[0]) - } - } -} - -func TestTapDeviceName_EmptyInput(t *testing.T) { - // Even with empty string (no eni- prefix), should not panic - name := vm.TapDeviceName("") - if name != "tap" { - t.Errorf("vm.TapDeviceName('') = %q, want 'tap'", name) - } -} - -func TestOVSIfaceID_Format(t *testing.T) { - tests := []struct { - eniId string - expected string - }{ - {"eni-abc123", "port-eni-abc123"}, - {"eni-abc123def456789", "port-eni-abc123def456789"}, - {"eni-short", "port-eni-short"}, - {"eni-", "port-eni-"}, - {"", "port-"}, - } - for _, tt := range tests { - got := vm.OVSIfaceID(tt.eniId) - if got != tt.expected { - t.Errorf("vm.OVSIfaceID(%q) = %q, want %q", tt.eniId, got, tt.expected) - } - } -} - -func TestGenerateMgmtMAC(t *testing.T) { - tests := []string{ - "i-abc123", - "i-def456", - "i-ghi789", - } - - seen := make(map[string]bool) - for _, id := range tests { - mac := generateMgmtMAC(id) - hw, err := net.ParseMAC(mac) - if err != nil { - t.Errorf("generateMgmtMAC(%q) = %q: invalid MAC: %v", id, mac, err) - continue - } - if hw[0]&0x03 != 0x02 { - t.Errorf("generateMgmtMAC(%q) = %q: expected unicast+LAA bits, got %#x", - id, mac, hw[0]) - } - if seen[mac] { - t.Errorf("generateMgmtMAC(%q) = %q, duplicate MAC", id, mac) - } - seen[mac] = true - } - - // Deterministic - mac1 := generateMgmtMAC("i-test123") - mac2 := generateMgmtMAC("i-test123") - if mac1 != mac2 { - t.Errorf("generateMgmtMAC not deterministic: %q != %q", mac1, mac2) - } - - // Different from dev MAC for same instance - devMAC := vm.GenerateDevMAC("i-test123") - mgmtMAC := generateMgmtMAC("i-test123") - if devMAC == mgmtMAC { - t.Errorf("dev and mgmt MACs should differ for same instance: both %q", devMAC) - } -} - -func TestMgmtTapName(t *testing.T) { - tests := []struct { - instanceID string - expected string - }{ - {"i-abc123", "mgabc123"}, - {"i-abc123def456789", "mgabc123def4567"}, // Truncated to 15 chars - {"i-a", "mga"}, - {"abc123", "mgabc123"}, // No i- prefix - } - - for _, tt := range tests { - t.Run(tt.instanceID, func(t *testing.T) { - got := vm.MgmtTapName(tt.instanceID) - if got != tt.expected { - t.Errorf("vm.MgmtTapName(%q) = %q, want %q", tt.instanceID, got, tt.expected) - } - if len(got) > 15 { - t.Errorf("vm.MgmtTapName(%q) = %q (len %d), exceeds IFNAMSIZ limit of 15", tt.instanceID, got, len(got)) - } - }) - } -} - -func TestGetBridgeIPv4_Loopback(t *testing.T) { - // "lo" is always present and has 127.0.0.1 - ip, err := GetBridgeIPv4("lo") - if err != nil { - t.Fatalf("GetBridgeIPv4(lo): %v", err) - } - if ip != "127.0.0.1" { - t.Errorf("GetBridgeIPv4(lo) = %q, want 127.0.0.1", ip) - } -} - -func TestGetBridgeIPv4_NonexistentBridge(t *testing.T) { - ip, err := GetBridgeIPv4("br-nonexistent-test-xyz") - if err != nil { - t.Fatalf("expected nil error for absent bridge, got: %v", err) - } - if ip != "" { - t.Errorf("expected empty IP for absent bridge, got %q", ip) - } -} diff --git a/spinifex/daemon/vm_adapters.go b/spinifex/daemon/vm_adapters.go index b4a0c8669..e479268ca 100644 --- a/spinifex/daemon/vm_adapters.go +++ b/spinifex/daemon/vm_adapters.go @@ -11,6 +11,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/awserrors" handlers_ec2_placementgroup "github.com/mulgadc/spinifex/spinifex/handlers/ec2/placementgroup" + "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/mulgadc/spinifex/spinifex/vm" @@ -431,7 +432,7 @@ func (d *Daemon) onInstanceUpHook() func(*vm.VM) error { privateIP = *instance.Instance.PrivateIpAddress } if vpcID != "" && privateIP != "" { - portName := "port-" + instance.ENIId + portName := topology.Port(instance.ENIId) utils.PublishNATEvent(d.natsConn, "vpc.add-nat", vpcID, instance.PublicIP, privateIP, portName, instance.ENIMac) } } @@ -586,7 +587,7 @@ func (a *instanceCleanerAdapter) ReleasePublicIP(instance *vm.VM) { return } - portName := "port-" + instance.ENIId + portName := topology.Port(instance.ENIId) vpcId := "" logicalIP := "" if instance.Instance != nil { diff --git a/spinifex/formation/formation.go b/spinifex/formation/formation.go index d0e72ced2..35a6447d2 100644 --- a/spinifex/formation/formation.go +++ b/spinifex/formation/formation.go @@ -62,7 +62,6 @@ type StatusResponse struct { // propagated from the init node to joining nodes during formation. type NetworkConfig struct { ExternalMode string `json:"external_mode"` - ExternalDHCP bool `json:"external_dhcp,omitempty"` PoolName string `json:"pool_name"` PoolSource string `json:"pool_source,omitempty"` PoolStart string `json:"pool_start,omitempty"` @@ -72,6 +71,11 @@ type NetworkConfig struct { PoolPrefixLen int `json:"pool_prefix_len"` PoolDNSServers []string `json:"pool_dns_servers,omitempty"` + // IPSecEnabled propagates the cluster-wide intra-AZ IPsec toggle so the + // joining node provisions strongSwan and flips OVS ipsec_encapsulation to + // match the rest of the cluster. + IPSecEnabled bool `json:"ipsec_enabled"` + BootstrapAccountId string `json:"bootstrap_account_id,omitempty"` BootstrapVpcId string `json:"bootstrap_vpc_id,omitempty"` BootstrapSubnetId string `json:"bootstrap_subnet_id,omitempty"` diff --git a/spinifex/handlers/ec2/eip/service_impl.go b/spinifex/handlers/ec2/eip/service_impl.go index e5dba061d..02526a326 100644 --- a/spinifex/handlers/ec2/eip/service_impl.go +++ b/spinifex/handlers/ec2/eip/service_impl.go @@ -14,6 +14,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/filterutil" handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" "github.com/mulgadc/spinifex/spinifex/migrate" + "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" ) @@ -73,7 +74,7 @@ func (s *EIPServiceImpl) AllocateAddress(input *ec2.AllocateAddressInput, accoun if input.PublicIpv4Pool != nil && *input.PublicIpv4Pool != "" { // Allocate from a specific named pool. poolName = *input.PublicIpv4Pool - publicIP, err = s.externalIPAM.AllocateFromPool(poolName, "elastic_ip", allocID, "", "") + publicIP, err = s.externalIPAM.AllocateFromPool(poolName, handlers_ec2_vpc.PurposeEIP, allocID, "", "") if err != nil { slog.Error("AllocateAddress: IPAM pool allocation failed", "pool", poolName, "err", err) return nil, errors.New(awserrors.ErrorInsufficientAddressCapacity) @@ -82,7 +83,7 @@ func (s *EIPServiceImpl) AllocateAddress(input *ec2.AllocateAddressInput, accoun // Allocate from the best pool matching region/AZ (empty strings = global fallback). region := "" az := "" - publicIP, poolName, err = s.externalIPAM.AllocateIP(region, az, "elastic_ip", allocID, "", "") + publicIP, poolName, err = s.externalIPAM.AllocateIP(region, az, handlers_ec2_vpc.PurposeEIP, allocID, "", "") if err != nil { slog.Error("AllocateAddress: IPAM allocation failed", "err", err) return nil, errors.New(awserrors.ErrorInsufficientAddressCapacity) @@ -579,7 +580,7 @@ func (s *EIPServiceImpl) publishNATEvent(topic, vpcID, externalIP, logicalIP, en VpcId: vpcID, ExternalIP: externalIP, LogicalIP: logicalIP, - PortName: "port-" + eniID, + PortName: topology.Port(eniID), MAC: mac, }) } diff --git a/spinifex/handlers/ec2/eip/service_impl_test.go b/spinifex/handlers/ec2/eip/service_impl_test.go index 48e687977..f0c35df94 100644 --- a/spinifex/handlers/ec2/eip/service_impl_test.go +++ b/spinifex/handlers/ec2/eip/service_impl_test.go @@ -30,7 +30,7 @@ func setupTestEIP(t *testing.T) (*EIPServiceImpl, *handlers_ec2_vpc.ExternalIPAM _, nc, js := testutil.StartTestJetStream(t) pool := testPool() - ipam, err := handlers_ec2_vpc.NewExternalIPAM(nc, js, []handlers_ec2_vpc.ExternalPoolConfig{pool}) + ipam, err := handlers_ec2_vpc.NewExternalIPAM(js, []handlers_ec2_vpc.ExternalPoolConfig{pool}) require.NoError(t, err) svc, err := NewEIPServiceImpl(nc, ipam, nil) @@ -562,7 +562,7 @@ func TestEIP_DescribeAddresses_FilterByTag(t *testing.T) { // writes PortName into NAT.LogicalPort in distributed NAT mode (direct bridge), // so a mismatch with the OVN logical switch port name ("port-") produces // a dnat_and_snat row pointing at a nonexistent port — OVN never programs the -// flow and the EIP black-holes. See mulga-js-17. +// flow and the EIP black-holes. func TestEIP_PublishNATEvent_PortNameHasPortPrefix(t *testing.T) { svc, _ := setupTestEIP(t) diff --git a/spinifex/handlers/ec2/igw/service_impl.go b/spinifex/handlers/ec2/igw/service_impl.go index 52b7f7a17..fcb0d7493 100644 --- a/spinifex/handlers/ec2/igw/service_impl.go +++ b/spinifex/handlers/ec2/igw/service_impl.go @@ -32,7 +32,7 @@ const ( type IGWRecord struct { InternetGatewayId string `json:"internet_gateway_id"` VpcId string `json:"vpc_id,omitempty"` // empty when detached - State string `json:"state"` // "available", "attached", "detached" + State string `json:"state"` // "available" — AWS attachment.state is "available" when attached Tags map[string]string `json:"tags"` CreatedAt time.Time `json:"created_at"` } diff --git a/spinifex/handlers/ec2/instance/service_impl.go b/spinifex/handlers/ec2/instance/service_impl.go index 3b32116b4..5870a0b9a 100644 --- a/spinifex/handlers/ec2/instance/service_impl.go +++ b/spinifex/handlers/ec2/instance/service_impl.go @@ -22,7 +22,9 @@ import ( "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/config" "github.com/mulgadc/spinifex/spinifex/filterutil" + handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" "github.com/mulgadc/spinifex/spinifex/instancetypes" + "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/objectstore" spxtypes "github.com/mulgadc/spinifex/spinifex/types" "github.com/mulgadc/spinifex/spinifex/utils" @@ -698,17 +700,21 @@ func (s *InstanceServiceImpl) PrepareRunInstances(input *ec2.RunInstancesInput, if s.ipAllocator != nil { subnet, subErr := s.eniCreator.GetSubnet(accountID, *input.SubnetId) - if subErr == nil && subnet != nil && subnet.MapPublicIpOnLaunch { + wantPublic := subErr == nil && subnet != nil && subnet.MapPublicIpOnLaunch + if len(input.NetworkInterfaces) > 0 && input.NetworkInterfaces[0] != nil && input.NetworkInterfaces[0].AssociatePublicIpAddress != nil { + wantPublic = *input.NetworkInterfaces[0].AssociatePublicIpAddress + } + if wantPublic { region := s.config.Region az := s.config.AZ - publicIP, poolName, allocErr := s.ipAllocator.AllocateIP(region, az, "auto_assign", "", *eni.NetworkInterfaceId, instance.ID) + publicIP, poolName, allocErr := s.ipAllocator.AllocateIP(region, az, handlers_ec2_vpc.PurposeENIPublic, "", *eni.NetworkInterfaceId, instance.ID) if allocErr != nil { slog.Warn("PrepareRunInstances: failed to allocate public IP", "instanceId", instance.ID, "err", allocErr) } else { if updateErr := s.eniCreator.UpdateENIPublicIP(accountID, *eni.NetworkInterfaceId, publicIP, poolName); updateErr != nil { slog.Warn("PrepareRunInstances: failed to update ENI with public IP", "eniId", *eni.NetworkInterfaceId, "err", updateErr) } - portName := "port-" + *eni.NetworkInterfaceId + portName := topology.Port(*eni.NetworkInterfaceId) if natErr := utils.AddNAT(s.natsConn, *eni.VpcId, publicIP, *eni.PrivateIpAddress, portName, *eni.MacAddress); natErr != nil { slog.Error("PrepareRunInstances: vpc.add-nat failed — rolling back public IP to avoid surfacing an unreachable address", "instanceId", instance.ID, "publicIp", publicIP, "pool", poolName, "err", natErr) @@ -1365,8 +1371,7 @@ func (s *InstanceServiceImpl) newViperblock(volumeName string, size int, volumeC // restoreSlogDefault re-installs the daemon's Info-level slog handler after // viperblock.New mutates the global slog default via its SetDebug method // (see viperblock.go SetDebug — it calls slog.SetDefault with LevelError, -// silencing every Info/Warn in the entire process). Tracked for proper fix -// in viperblock as mulga-siv-70. +// silencing every Info/Warn in the entire process). func restoreSlogDefault() { slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ Level: slog.LevelInfo, @@ -2602,7 +2607,7 @@ func (s *InstanceServiceImpl) releaseInstancePublicIP(instance *vm.VM, instanceI if instance.PublicIP == "" || instance.PublicIPPool == "" || s.ipReleaser == nil { return } - portName := "port-" + instance.ENIId + portName := topology.Port(instance.ENIId) vpcID := "" logicalIP := "" if instance.Instance != nil { diff --git a/spinifex/handlers/ec2/instance/service_impl_test.go b/spinifex/handlers/ec2/instance/service_impl_test.go index b7addd222..d489f1560 100644 --- a/spinifex/handlers/ec2/instance/service_impl_test.go +++ b/spinifex/handlers/ec2/instance/service_impl_test.go @@ -2536,40 +2536,68 @@ func TestPrepareRunInstances_DefaultSubnetResolved(t *testing.T) { } func TestPrepareRunInstances_PublicIPAutoAssigned(t *testing.T) { - eni := &fakeENICreator{ - subnet: &SubnetInfo{SubnetID: "subnet-1", VpcID: "vpc-1", MapPublicIpOnLaunch: true}, - createOut: &ec2.CreateNetworkInterfaceOutput{ - NetworkInterface: &ec2.NetworkInterface{ - NetworkInterfaceId: aws.String("eni-2"), - MacAddress: aws.String("aa:bb:cc:dd:ee:00"), - PrivateIpAddress: aws.String("10.0.0.20"), - VpcId: aws.String("vpc-1"), - }, - }, + cases := []struct { + name string + mapOnLaunch bool + nicOverride *bool + wantPublic bool + }{ + {name: "subnet_true_no_override", mapOnLaunch: true, nicOverride: nil, wantPublic: true}, + {name: "subnet_false_override_true", mapOnLaunch: false, nicOverride: aws.Bool(true), wantPublic: true}, + {name: "subnet_true_override_false", mapOnLaunch: true, nicOverride: aws.Bool(false), wantPublic: false}, } - ipam := &fakeIPAllocator{publicIP: "203.0.113.5", poolName: "pool-a"} - svc, _ := prepareSvcWithENI(t, eni, ipam) + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + eni := &fakeENICreator{ + subnet: &SubnetInfo{SubnetID: "subnet-1", VpcID: "vpc-1", MapPublicIpOnLaunch: tc.mapOnLaunch}, + createOut: &ec2.CreateNetworkInterfaceOutput{ + NetworkInterface: &ec2.NetworkInterface{ + NetworkInterfaceId: aws.String("eni-2"), + MacAddress: aws.String("aa:bb:cc:dd:ee:00"), + PrivateIpAddress: aws.String("10.0.0.20"), + VpcId: aws.String("vpc-1"), + }, + }, + } + ipam := &fakeIPAllocator{publicIP: "203.0.113.5", poolName: "pool-a"} + svc, _ := prepareSvcWithENI(t, eni, ipam) - // vpc.add-nat is now request-reply: the helper waits for vpcd to ack - // before the launch proceeds. The happy path needs a success responder. - sub, err := svc.natsConn.Subscribe("vpc.add-nat", func(msg *nats.Msg) { - _ = msg.Respond([]byte(`{"success":true}`)) - }) - require.NoError(t, err) - defer func() { _ = sub.Unsubscribe() }() + // vpc.add-nat is now request-reply: the helper waits for vpcd to + // ack before the launch proceeds. The happy path needs a success + // responder. + sub, err := svc.natsConn.Subscribe("vpc.add-nat", func(msg *nats.Msg) { + _ = msg.Respond([]byte(`{"success":true}`)) + }) + require.NoError(t, err) + defer func() { _ = sub.Unsubscribe() }() - _, instances, _, err := svc.PrepareRunInstances(&ec2.RunInstancesInput{ - InstanceType: aws.String("t3.micro"), - ImageId: aws.String("ami-1"), - SubnetId: aws.String("subnet-1"), - MinCount: aws.Int64(1), - MaxCount: aws.Int64(1), - }, "acc") - require.NoError(t, err) - require.Len(t, instances, 1) - assert.Equal(t, "203.0.113.5", instances[0].PublicIP) - assert.Equal(t, "pool-a", instances[0].PublicIPPool) - assert.Equal(t, 1, eni.updateCalls) + input := &ec2.RunInstancesInput{ + InstanceType: aws.String("t3.micro"), + ImageId: aws.String("ami-1"), + SubnetId: aws.String("subnet-1"), + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + } + if tc.nicOverride != nil { + input.NetworkInterfaces = []*ec2.InstanceNetworkInterfaceSpecification{ + {AssociatePublicIpAddress: tc.nicOverride}, + } + } + + _, instances, _, err := svc.PrepareRunInstances(input, "acc") + require.NoError(t, err) + require.Len(t, instances, 1) + if tc.wantPublic { + assert.Equal(t, "203.0.113.5", instances[0].PublicIP) + assert.Equal(t, "pool-a", instances[0].PublicIPPool) + assert.Equal(t, 1, eni.updateCalls) + } else { + assert.Empty(t, instances[0].PublicIP) + assert.Empty(t, instances[0].PublicIPPool) + assert.Equal(t, 0, eni.updateCalls) + } + }) + } } // TestPrepareRunInstances_NATFailureRollsBackPublicIP regresses the silent diff --git a/spinifex/handlers/ec2/routetable/service_impl_test.go b/spinifex/handlers/ec2/routetable/service_impl_test.go index c0a1fbc21..f8ae2c3ff 100644 --- a/spinifex/handlers/ec2/routetable/service_impl_test.go +++ b/spinifex/handlers/ec2/routetable/service_impl_test.go @@ -31,7 +31,7 @@ func setupTestService(t *testing.T) *RouteTableServiceImpl { // Seed IGW KV (attached to vpc-test1) testutil.SeedKV(t, js, handlers_ec2_igw.KVBucketIGW, map[string][]byte{ - utils.AccountKey(testAccountID, "igw-test1"): []byte(`{"internet_gateway_id":"igw-test1","vpc_id":"vpc-test1","state":"attached"}`), + utils.AccountKey(testAccountID, "igw-test1"): []byte(`{"internet_gateway_id":"igw-test1","vpc_id":"vpc-test1","state":"available"}`), }) // Seed subnet KV diff --git a/spinifex/handlers/ec2/vpc/eni.go b/spinifex/handlers/ec2/vpc/eni.go index 30b28f72a..66b1969e4 100644 --- a/spinifex/handlers/ec2/vpc/eni.go +++ b/spinifex/handlers/ec2/vpc/eni.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" "github.com/mulgadc/spinifex/spinifex/awserrors" "github.com/mulgadc/spinifex/spinifex/filterutil" + "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" ) @@ -78,21 +79,21 @@ func (s *VPCServiceImpl) CreateNetworkInterface(input *ec2.CreateNetworkInterfac return nil, err } + eniId := utils.GenerateResourceID("eni") + // Allocate IP from subnet var privateIP string if input.PrivateIpAddress != nil && *input.PrivateIpAddress != "" { // TODO: validate the requested IP is in the subnet range and not already allocated privateIP = *input.PrivateIpAddress } else { - ip, err := s.ipam.AllocateIP(subnetId, subnet.CidrBlock) + ip, err := s.ipam.AllocateIP(subnetId, subnet.CidrBlock, PurposeENIPrimary, eniId) if err != nil { return nil, errors.New(awserrors.ErrorServerInternal) } privateIP = ip } - eniId := utils.GenerateResourceID("eni") - // Generate a deterministic MAC address macAddr := generateENIMac(eniId) @@ -178,7 +179,7 @@ func (s *VPCServiceImpl) DeleteNetworkInterface(input *ec2.DeleteNetworkInterfac } else if owned { slog.Info("DeleteNetworkInterface: public IP owned by EIP, skipping release", "eniId", eniId, "publicIp", record.PublicIpAddress) } else { - portName := "port-" + eniId + portName := topology.Port(eniId) s.publishNATEvent("vpc.delete-nat", record.VpcId, record.PublicIpAddress, record.PrivateIpAddress, portName, record.MacAddress) if err := s.externalIPAM.ReleaseIP(record.PublicIpPool, record.PublicIpAddress); err != nil { slog.Warn("Failed to release public IP during ENI delete", "eni", eniId, "ip", record.PublicIpAddress, "pool", record.PublicIpPool, "err", err) @@ -588,8 +589,8 @@ func generateENIMac(eniId string) string { } // portEventPayload is the wire shape for vpc.create-port / vpc.delete-port. -// Mirrors vpcd.PortEvent — duplicated here to avoid a vpcd → handlers import -// cycle. +// Mirrors network/subscribers.PortEvent — duplicated here to avoid a +// subscribers → handlers import cycle. type portEventPayload struct { NetworkInterfaceId string `json:"network_interface_id"` SubnetId string `json:"subnet_id"` diff --git a/spinifex/handlers/ec2/vpc/external_ipam.go b/spinifex/handlers/ec2/vpc/external_ipam.go index e8f9d19ef..e3661d884 100644 --- a/spinifex/handlers/ec2/vpc/external_ipam.go +++ b/spinifex/handlers/ec2/vpc/external_ipam.go @@ -9,38 +9,29 @@ import ( "net" "github.com/mulgadc/spinifex/spinifex/migrate" - "github.com/mulgadc/spinifex/spinifex/services/vpcd/dhcp" "github.com/mulgadc/spinifex/spinifex/utils" "github.com/nats-io/nats.go" ) const ( KVBucketExternalIPAM = "spinifex-external-ipam" - KVBucketExternalIPAMVersion = 1 + KVBucketExternalIPAMVersion = 2 ) -// ExternalIPAllocation describes how an external IP is being used. +// ExternalIPAllocation describes how an external IP is being used. Purpose +// values come from the Purpose enum in purpose.go. type ExternalIPAllocation struct { - Type string `json:"type"` // "gateway", "auto_assign", "elastic_ip" + Purpose string `json:"purpose"` // Purpose enum: igw-lrp, eni-public, eip, natgw-external AllocationID string `json:"allocation_id,omitempty"` // For elastic IPs Association string `json:"association,omitempty"` // ENI ID for elastic IPs ENIId string `json:"eni_id,omitempty"` // ENI owning this IP InstanceId string `json:"instance_id,omitempty"` // Instance owning this IP Note string `json:"note,omitempty"` // Human-readable note - - // DHCP-sourced metadata — set only for allocations whose pool has - // source="dhcp" (see dhcp.RequestAcquire). These fields let describe - // surfaces and operators trace a given IP back to the upstream lease - // without re-querying vpcd. - LeaseExpiresUnix int64 `json:"lease_expires_unix,omitempty"` // Absolute expiry from the server's ACK - DHCPServerID string `json:"dhcp_server_id,omitempty"` // Option 54 of the ACK - HWAddr string `json:"hw_addr,omitempty"` // chaddr placed in the DHCP payload } // ExternalIPAMRecord tracks allocated external IPs for a single pool. type ExternalIPAMRecord struct { PoolName string `json:"pool_name"` - Source string `json:"source"` // "static" (default) or "dhcp" RangeStart string `json:"range_start"` RangeEnd string `json:"range_end"` Gateway string `json:"gateway"` @@ -50,7 +41,7 @@ type ExternalIPAMRecord struct { AZ string `json:"az,omitempty"` // GwLrpRangeStart/End mirrors ExternalPoolConfig — IPAM skips this // sub-range so it doesn't collide with vpcd's gateway LRP IPs in - // centralized NAT (mulga-siv-36). + // centralized NAT. GwLrpRangeStart string `json:"gw_lrp_range_start,omitempty"` GwLrpRangeEnd string `json:"gw_lrp_range_end,omitempty"` Allocated map[string]ExternalIPAllocation `json:"allocated"` @@ -58,49 +49,29 @@ type ExternalIPAMRecord struct { // ExternalPoolConfig is the admin-defined pool from spinifex.toml. type ExternalPoolConfig struct { - Name string - Source string // "static" (default) or "dhcp" - RangeStart string - RangeEnd string - Gateway string - GatewayIP string - PrefixLen int - Region string - AZ string - DhcpBindBridge string // Bridge where the DHCP AF_PACKET socket binds (e.g. "br-wan"). Linux bridge in veth mode; OVS bridge in direct mode. Never "br-ext". - // GatewayMAC, when set, is used as the chaddr in per-VM DHCP DORAs so - // that the upstream DHCP binding matches the MAC that OVN GARPs use (the - // gateway LRP MAC). Without this, the DHCP server binds the public IP to - // the generated ENI-derived MAC while OVN announces a different MAC via - // GARP, causing home routers that use their DHCP lease table as ARP to - // forward traffic to a non-existent interface. - GatewayMAC string + Name string + RangeStart string + RangeEnd string + Gateway string + GatewayIP string + PrefixLen int + Region string + AZ string // GwLrpRangeStart/End reserves a sub-range of the LAN for OVN gateway - // LRP IPs in centralized NAT mode (mulga-siv-36). IPAM must skip these - // addresses or the per-VM EIP allocator and vpcd will fight over them. + // LRP IPs in centralized NAT mode. IPAM must skip these addresses or + // the per-VM EIP allocator and vpcd will fight over them. GwLrpRangeStart string GwLrpRangeEnd string } -// IsDHCP returns true if this pool obtains IPs from router DHCP. -func (p *ExternalPoolConfig) IsDHCP() bool { - return p.Source == "dhcp" -} - // ExternalIPAM manages external IP allocation from admin-defined pools using NATS KV with CAS. type ExternalIPAM struct { kv nats.KeyValue pools []ExternalPoolConfig - // nc is used to talk to spinifex-vpcd for DHCP-sourced pools via - // vpc.dhcp.acquire / vpc.dhcp.release. May be nil when no pool has - // Source="dhcp" (callers should still pass it if available). - nc *nats.Conn } // NewExternalIPAM creates a new ExternalIPAM backed by NATS JetStream KV. -// nc is required for DHCP-sourced pools and may be nil for static-only -// deployments. -func NewExternalIPAM(nc *nats.Conn, js nats.JetStreamContext, pools []ExternalPoolConfig) (*ExternalIPAM, error) { +func NewExternalIPAM(js nats.JetStreamContext, pools []ExternalPoolConfig) (*ExternalIPAM, error) { kv, err := utils.GetOrCreateKVBucket(js, KVBucketExternalIPAM, 5) if err != nil { return nil, fmt.Errorf("create external IPAM KV bucket: %w", err) @@ -108,17 +79,16 @@ func NewExternalIPAM(nc *nats.Conn, js nats.JetStreamContext, pools []ExternalPo if err := migrate.DefaultRegistry.RunKV(KVBucketExternalIPAM, kv, KVBucketExternalIPAMVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketExternalIPAM, err) } - ipam := &ExternalIPAM{kv: kv, pools: pools, nc: nc} + ipam := &ExternalIPAM{kv: kv, pools: pools} if err := ipam.initPools(); err != nil { return nil, fmt.Errorf("init external IPAM pools: %w", err) } return ipam, nil } -// NewExternalIPAMWithKV creates an ExternalIPAM with an existing KV bucket -// (for testing). nc may be nil when the test exercises only static pools. -func NewExternalIPAMWithKV(nc *nats.Conn, kv nats.KeyValue, pools []ExternalPoolConfig) *ExternalIPAM { - return &ExternalIPAM{kv: kv, pools: pools, nc: nc} +// NewExternalIPAMWithKV creates an ExternalIPAM with an existing KV bucket (for testing). +func NewExternalIPAMWithKV(kv nats.KeyValue, pools []ExternalPoolConfig) *ExternalIPAM { + return &ExternalIPAM{kv: kv, pools: pools} } // initPools ensures each configured pool has a KV record. Idempotent — safe to call @@ -140,18 +110,16 @@ func (m *ExternalIPAM) initPool(pool ExternalPoolConfig) error { } if err == nil { - if chk.RangeStart != pool.RangeStart || chk.RangeEnd != pool.RangeEnd || chk.Source != pool.Source || + if chk.RangeStart != pool.RangeStart || chk.RangeEnd != pool.RangeEnd || chk.GwLrpRangeStart != pool.GwLrpRangeStart || chk.GwLrpRangeEnd != pool.GwLrpRangeEnd { slog.Info("external IPAM pool config drift, reconciling KV", "pool", pool.Name, "old_range", chk.RangeStart+"-"+chk.RangeEnd, "new_range", pool.RangeStart+"-"+pool.RangeEnd, - "old_source", chk.Source, "new_source", pool.Source, "old_gw_lrp_range", chk.GwLrpRangeStart+"-"+chk.GwLrpRangeEnd, "new_gw_lrp_range", pool.GwLrpRangeStart+"-"+pool.GwLrpRangeEnd) chk.RangeStart = pool.RangeStart chk.RangeEnd = pool.RangeEnd - chk.Source = pool.Source chk.GwLrpRangeStart = pool.GwLrpRangeStart chk.GwLrpRangeEnd = pool.GwLrpRangeEnd @@ -172,39 +140,11 @@ func (m *ExternalIPAM) initPool(pool ExternalPoolConfig) error { slog.Info("external IPAM pool not found, creating", "pool", pool.Name) - // TODO: Confirm gateway IP defined, should error, given we route traffic to the upstream DHCP gateway gwIP := pool.GatewayIP - if gwIP == "" && !pool.IsDHCP() { + if gwIP == "" { gwIP = pool.RangeStart } - // For DHCP pools, obtain the gateway IP from router DHCP if not set. - // The gateway lease is long-lived: vpcd's renewal goroutine keeps it - // refreshed until the pool is torn down. - var gatewayLease *dhcp.LeaseResult - if gwIP == "" && pool.IsDHCP() { - lease, dhcpErr := dhcp.RequestAcquire(m.nc, pool.DhcpBindBridge, - "gateway-"+pool.Name, - "gateway-"+pool.Name, - "mulga-spinifex-gw", - pool.Name, - "", - ) - if dhcpErr != nil { - return fmt.Errorf("DHCP gateway lease for pool %q: %w", pool.Name, dhcpErr) - } - gatewayLease = &lease - gwIP = lease.IP - slog.Info("external IPAM obtained gateway IP via DHCP", "pool", pool.Name, "gateway_ip", gwIP) - } - - gatewayAlloc := ExternalIPAllocation{Type: "gateway", Note: "OVN router SNAT address"} - if gatewayLease != nil { - gatewayAlloc.LeaseExpiresUnix = gatewayLease.ExpiresUnix - gatewayAlloc.DHCPServerID = gatewayLease.ServerID - gatewayAlloc.HWAddr = gatewayLease.HWAddr - } - record := &ExternalIPAMRecord{ PoolName: pool.Name, RangeStart: pool.RangeStart, @@ -217,7 +157,7 @@ func (m *ExternalIPAM) initPool(pool ExternalPoolConfig) error { GwLrpRangeStart: pool.GwLrpRangeStart, GwLrpRangeEnd: pool.GwLrpRangeEnd, Allocated: map[string]ExternalIPAllocation{ - gwIP: gatewayAlloc, + gwIP: {Purpose: PurposeIGWLRP, Note: "OVN router SNAT address"}, }, } @@ -234,19 +174,20 @@ func (m *ExternalIPAM) initPool(pool ExternalPoolConfig) error { return fmt.Errorf("create pool KV entry: %w", err) } - slog.Info("external IPAM pool initialized", "pool", pool.Name, "gateway_ip", gwIP, "source", pool.Source) + slog.Info("external IPAM pool initialized", "pool", pool.Name, "gateway_ip", gwIP) return nil } // AllocateIP allocates the next available external IP from the best pool // matching the given region/AZ. Returns the allocated IP and pool name. -// allocID is the EIP allocation ID (for elastic IPs); pass "" for other types. -func (m *ExternalIPAM) AllocateIP(region, az, allocType, allocID, eniID, instanceID string) (string, string, error) { +// purpose is one of the Purpose* constants in purpose.go; allocID is the +// EIP allocation ID (for PurposeEIP) and "" otherwise. +func (m *ExternalIPAM) AllocateIP(region, az, purpose, allocID, eniID, instanceID string) (string, string, error) { pool := m.findPool(region, az) if pool == nil { return "", "", fmt.Errorf("InsufficientAddressCapacity: no external pool available for region=%q az=%q", region, az) } - ip, err := m.allocateFromPool(pool.Name, allocType, allocID, eniID, instanceID) + ip, err := m.allocateFromPool(pool.Name, purpose, allocID, eniID, instanceID) if err != nil { return "", "", err } @@ -254,76 +195,31 @@ func (m *ExternalIPAM) AllocateIP(region, az, allocType, allocID, eniID, instanc } // AllocateFromPool allocates an IP from a specific named pool. -func (m *ExternalIPAM) AllocateFromPool(poolName, allocType, allocID, eniID, instanceID string) (string, error) { - return m.allocateFromPool(poolName, allocType, allocID, eniID, instanceID) +func (m *ExternalIPAM) AllocateFromPool(poolName, purpose, allocID, eniID, instanceID string) (string, error) { + return m.allocateFromPool(poolName, purpose, allocID, eniID, instanceID) } -func (m *ExternalIPAM) allocateFromPool(poolName, allocType, allocID, eniID, instanceID string) (string, error) { - pool := m.findPoolByName(poolName) - - // DHCP client-id: prefer allocID (EIPs), then eniID (ENIs), then instanceID. - // At least one must be non-empty for DHCP pools — the upstream DHCP server - // uses this to keep per-client leases distinct. - clientID := allocID - if clientID == "" { - clientID = eniID - } - if clientID == "" { - clientID = instanceID - } - - // For DHCP-sourced pools: acquire the lease once, up front. vpcd's - // Manager is idempotent on ClientID, so CAS retries below will not - // trigger a second DORA — we only release at the end if no CAS - // attempt succeeded. - var dhcpLease *dhcp.LeaseResult - if pool != nil && pool.IsDHCP() { - hostname, vendorClass := dhcpIdentityOptions(eniID, instanceID, poolName) - lease, err := dhcp.RequestAcquire(m.nc, pool.DhcpBindBridge, clientID, hostname, vendorClass, poolName, pool.GatewayMAC) - if err != nil { - return "", fmt.Errorf("DHCP lease for %s: %w", clientID, err) - } - dhcpLease = &lease - } - +func (m *ExternalIPAM) allocateFromPool(poolName, purpose, allocID, eniID, instanceID string) (string, error) { for attempt := range 5 { record, revision, err := m.getRecord(poolName) if err != nil { - if dhcpLease != nil { - _ = dhcp.RequestRelease(m.nc, clientID) - } return "", fmt.Errorf("get external IPAM record: %w", err) } - var ip string - if dhcpLease != nil { - ip = dhcpLease.IP - } else { - // Static source: pick next IP from range - ip, err = nextAvailableExternalIP(record) - if err != nil { - return "", err - } + ip, err := nextAvailableExternalIP(record) + if err != nil { + return "", err } - alloc := ExternalIPAllocation{ - Type: allocType, + record.Allocated[ip] = ExternalIPAllocation{ + Purpose: purpose, AllocationID: allocID, ENIId: eniID, InstanceId: instanceID, } - if dhcpLease != nil { - alloc.LeaseExpiresUnix = dhcpLease.ExpiresUnix - alloc.DHCPServerID = dhcpLease.ServerID - alloc.HWAddr = dhcpLease.HWAddr - } - record.Allocated[ip] = alloc data, err := json.Marshal(record) if err != nil { - if dhcpLease != nil { - _ = dhcp.RequestRelease(m.nc, clientID) - } return "", fmt.Errorf("marshal external IPAM record: %w", err) } @@ -332,47 +228,15 @@ func (m *ExternalIPAM) allocateFromPool(poolName, allocType, allocID, eniID, ins continue } - source := "static" - if pool != nil { - source = pool.Source - } - slog.Info("external IPAM allocated IP", "pool", poolName, "ip", ip, "type", allocType, "source", source) + slog.Info("external IPAM allocated IP", "pool", poolName, "ip", ip, "purpose", purpose) return ip, nil } - // All CAS attempts exhausted — release any DHCP lease we acquired so - // vpcd doesn't hold a binding for an IP we won't use. - if dhcpLease != nil { - _ = dhcp.RequestRelease(m.nc, clientID) - } return "", fmt.Errorf("external IPAM allocation failed after CAS retries for pool %s", poolName) } -// dhcpIdentityOptions composes the option 12 (hostname) and option 60 -// (vendor class) values for a per-allocation lease. Empty eniID / -// instanceID fall through to identifiers that still group usefully on -// the upstream server's lease table. Instance and ENI identifiers are -// used as-is (they already carry their "i-" / "eni-" prefixes). -func dhcpIdentityOptions(eniID, instanceID, poolName string) (hostname, vendorClass string) { - hostname = eniID - if hostname == "" { - if instanceID != "" { - hostname = "spinifex-" + instanceID - } else { - hostname = "spinifex-" + poolName - } - } - vendorClass = "mulga-spinifex" - if instanceID != "" { - vendorClass = instanceID - } - return hostname, vendorClass -} - // ReleaseIP releases a previously allocated external IP back to its pool. func (m *ExternalIPAM) ReleaseIP(poolName, ip string) error { - pool := m.findPoolByName(poolName) - for attempt := range 5 { record, revision, err := m.getRecord(poolName) if err != nil { @@ -383,26 +247,10 @@ func (m *ExternalIPAM) ReleaseIP(poolName, ip string) error { if !ok { return fmt.Errorf("IP %s not allocated in pool %s", ip, poolName) } - if alloc.Type == "gateway" { + if alloc.Purpose == PurposeIGWLRP { return fmt.Errorf("cannot release gateway IP %s in pool %s", ip, poolName) } - // Release DHCP lease if this is a DHCP-sourced pool. clientID - // selection mirrors allocateFromPool so vpcd's Manager can find the - // lease it originally acquired. - if pool != nil && pool.IsDHCP() { - clientID := alloc.AllocationID - if clientID == "" { - clientID = alloc.ENIId - } - if clientID == "" { - clientID = alloc.InstanceId - } - if releaseErr := dhcp.RequestRelease(m.nc, clientID); releaseErr != nil { - slog.Warn("Failed to release DHCP lease", "pool", poolName, "ip", ip, "err", releaseErr) - } - } - delete(record.Allocated, ip) data, err := json.Marshal(record) @@ -455,16 +303,6 @@ func (m *ExternalIPAM) findPool(region, az string) *ExternalPoolConfig { return nil } -// findPoolByName returns the pool config by name. -func (m *ExternalIPAM) findPoolByName(name string) *ExternalPoolConfig { - for i := range m.pools { - if m.pools[i].Name == name { - return &m.pools[i] - } - } - return nil -} - func (m *ExternalIPAM) getRecord(poolName string) (*ExternalIPAMRecord, uint64, error) { entry, err := m.kv.Get(poolName) if err != nil { @@ -481,7 +319,7 @@ func (m *ExternalIPAM) getRecord(poolName string) (*ExternalIPAMRecord, uint64, // nextAvailableExternalIP finds the next unallocated IP in the pool's // range. Addresses inside [GwLrpRangeStart, GwLrpRangeEnd] are skipped — -// vpcd reserves them for OVN gateway LRPs (mulga-siv-36). +// vpcd reserves them for OVN gateway LRPs. func nextAvailableExternalIP(record *ExternalIPAMRecord) (string, error) { startIP := net.ParseIP(record.RangeStart).To4() endIP := net.ParseIP(record.RangeEnd).To4() @@ -529,44 +367,41 @@ func ValidatePoolConfig(pool ExternalPoolConfig) error { if pool.GatewayIP != "" && net.ParseIP(pool.GatewayIP) == nil { return fmt.Errorf("invalid gateway_ip: %q", pool.GatewayIP) } - // DHCP pools don't need range_start/range_end - if !pool.IsDHCP() { - startIP := net.ParseIP(pool.RangeStart) - if startIP == nil { - return fmt.Errorf("invalid range_start: %q", pool.RangeStart) + startIP := net.ParseIP(pool.RangeStart) + if startIP == nil { + return fmt.Errorf("invalid range_start: %q", pool.RangeStart) + } + endIP := net.ParseIP(pool.RangeEnd) + if endIP == nil { + return fmt.Errorf("invalid range_end: %q", pool.RangeEnd) + } + if ipToInt(startIP.To4()).Cmp(ipToInt(endIP.To4())) > 0 { + return fmt.Errorf("range_start %s is greater than range_end %s", pool.RangeStart, pool.RangeEnd) + } + // gw_lrp_range must be valid IPs and must NOT overlap range_start/end + // — otherwise vpcd's gateway LRP allocator and per-VM EIP allocator + // would fight over the same address. + if pool.GwLrpRangeStart != "" || pool.GwLrpRangeEnd != "" { + gwS := net.ParseIP(pool.GwLrpRangeStart) + gwE := net.ParseIP(pool.GwLrpRangeEnd) + if gwS == nil { + return fmt.Errorf("invalid gw_lrp_range_start: %q", pool.GwLrpRangeStart) } - endIP := net.ParseIP(pool.RangeEnd) - if endIP == nil { - return fmt.Errorf("invalid range_end: %q", pool.RangeEnd) + if gwE == nil { + return fmt.Errorf("invalid gw_lrp_range_end: %q", pool.GwLrpRangeEnd) } - if ipToInt(startIP.To4()).Cmp(ipToInt(endIP.To4())) > 0 { - return fmt.Errorf("range_start %s is greater than range_end %s", pool.RangeStart, pool.RangeEnd) + gwSi := ipToInt(gwS.To4()) + gwEi := ipToInt(gwE.To4()) + if gwSi.Cmp(gwEi) > 0 { + return fmt.Errorf("gw_lrp_range_start %s is greater than gw_lrp_range_end %s", + pool.GwLrpRangeStart, pool.GwLrpRangeEnd) } - // gw_lrp_range must be valid IPs and must NOT overlap range_start/end - // — otherwise vpcd's gateway LRP allocator and per-VM EIP allocator - // would fight over the same address (mulga-siv-36). - if pool.GwLrpRangeStart != "" || pool.GwLrpRangeEnd != "" { - gwS := net.ParseIP(pool.GwLrpRangeStart) - gwE := net.ParseIP(pool.GwLrpRangeEnd) - if gwS == nil { - return fmt.Errorf("invalid gw_lrp_range_start: %q", pool.GwLrpRangeStart) - } - if gwE == nil { - return fmt.Errorf("invalid gw_lrp_range_end: %q", pool.GwLrpRangeEnd) - } - gwSi := ipToInt(gwS.To4()) - gwEi := ipToInt(gwE.To4()) - if gwSi.Cmp(gwEi) > 0 { - return fmt.Errorf("gw_lrp_range_start %s is greater than gw_lrp_range_end %s", - pool.GwLrpRangeStart, pool.GwLrpRangeEnd) - } - rangeSi := ipToInt(startIP.To4()) - rangeEi := ipToInt(endIP.To4()) - // Overlap test: !(gwE < rangeS || gwS > rangeE) - if gwEi.Cmp(rangeSi) >= 0 && gwSi.Cmp(rangeEi) <= 0 { - return fmt.Errorf("gw_lrp_range %s-%s overlaps range %s-%s", - pool.GwLrpRangeStart, pool.GwLrpRangeEnd, pool.RangeStart, pool.RangeEnd) - } + rangeSi := ipToInt(startIP.To4()) + rangeEi := ipToInt(endIP.To4()) + // Overlap test: !(gwE < rangeS || gwS > rangeE) + if gwEi.Cmp(rangeSi) >= 0 && gwSi.Cmp(rangeEi) <= 0 { + return fmt.Errorf("gw_lrp_range %s-%s overlaps range %s-%s", + pool.GwLrpRangeStart, pool.GwLrpRangeEnd, pool.RangeStart, pool.RangeEnd) } } return nil diff --git a/spinifex/handlers/ec2/vpc/external_ipam_test.go b/spinifex/handlers/ec2/vpc/external_ipam_test.go index e3220124d..cd03fa4d1 100644 --- a/spinifex/handlers/ec2/vpc/external_ipam_test.go +++ b/spinifex/handlers/ec2/vpc/external_ipam_test.go @@ -1,24 +1,19 @@ package handlers_ec2_vpc import ( - "encoding/json" "sync" - "sync/atomic" "testing" - "time" - "github.com/mulgadc/spinifex/spinifex/services/vpcd/dhcp" "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/nats-io/nats.go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func setupTestExternalIPAM(t *testing.T, pools []ExternalPoolConfig) *ExternalIPAM { t.Helper() - _, nc, js := testutil.StartTestJetStream(t) + _, _, js := testutil.StartTestJetStream(t) - ipam, err := NewExternalIPAM(nc, js, pools) + ipam, err := NewExternalIPAM(js, pools) require.NoError(t, err) return ipam } @@ -38,16 +33,16 @@ func TestExternalIPAM_AllocateSequential(t *testing.T) { ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) // .150 is reserved for gateway, so first allocable is .151 - ip1, poolName, err := ipam.AllocateIP("", "", "auto_assign", "", "eni-1", "i-1") + ip1, poolName, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-1", "i-1") require.NoError(t, err) assert.Equal(t, "192.168.1.151", ip1) assert.Equal(t, "wan", poolName) - ip2, _, err := ipam.AllocateIP("", "", "auto_assign", "", "eni-2", "i-2") + ip2, _, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-2", "i-2") require.NoError(t, err) assert.Equal(t, "192.168.1.152", ip2) - ip3, _, err := ipam.AllocateIP("", "", "auto_assign", "", "eni-3", "i-3") + ip3, _, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-3", "i-3") require.NoError(t, err) assert.Equal(t, "192.168.1.153", ip3) } @@ -62,7 +57,7 @@ func TestExternalIPAM_GatewayIPReserved(t *testing.T) { // Gateway IP (.150) should be reserved alloc, ok := record.Allocated["192.168.1.150"] assert.True(t, ok, "gateway IP should be in allocated map") - assert.Equal(t, "gateway", alloc.Type) + assert.Equal(t, PurposeIGWLRP, alloc.Purpose) // Cannot release gateway IP err = ipam.ReleaseIP("wan", "192.168.1.150") @@ -84,7 +79,7 @@ func TestExternalIPAM_ExplicitGatewayIP(t *testing.T) { assert.False(t, ok) // First allocable is .150 since .155 is the gateway - ip1, _, err := ipam.AllocateIP("", "", "auto_assign", "", "eni-1", "i-1") + ip1, _, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-1", "i-1") require.NoError(t, err) assert.Equal(t, "192.168.1.150", ip1) } @@ -93,11 +88,11 @@ func TestExternalIPAM_Release(t *testing.T) { pool := testPool() ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) - ip1, _, err := ipam.AllocateIP("", "", "auto_assign", "", "eni-1", "i-1") + ip1, _, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-1", "i-1") require.NoError(t, err) assert.Equal(t, "192.168.1.151", ip1) - ip2, _, err := ipam.AllocateIP("", "", "auto_assign", "", "eni-2", "i-2") + ip2, _, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-2", "i-2") require.NoError(t, err) assert.Equal(t, "192.168.1.152", ip2) @@ -106,7 +101,7 @@ func TestExternalIPAM_Release(t *testing.T) { require.NoError(t, err) // Next allocation reuses released IP - ip3, _, err := ipam.AllocateIP("", "", "auto_assign", "", "eni-3", "i-3") + ip3, _, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-3", "i-3") require.NoError(t, err) assert.Equal(t, "192.168.1.151", ip3) } @@ -122,16 +117,16 @@ func TestExternalIPAM_Exhaustion(t *testing.T) { ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) // .1 reserved for gateway, .2 and .3 allocable - ip1, _, err := ipam.AllocateIP("", "", "auto_assign", "", "eni-1", "i-1") + ip1, _, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-1", "i-1") require.NoError(t, err) assert.Equal(t, "10.0.0.2", ip1) - ip2, _, err := ipam.AllocateIP("", "", "auto_assign", "", "eni-2", "i-2") + ip2, _, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-2", "i-2") require.NoError(t, err) assert.Equal(t, "10.0.0.3", ip2) // Pool exhausted - _, _, err = ipam.AllocateIP("", "", "auto_assign", "", "eni-3", "i-3") + _, _, err = ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-3", "i-3") assert.ErrorContains(t, err, "InsufficientAddressCapacity") } @@ -148,7 +143,7 @@ func TestExternalIPAM_CASConflict(t *testing.T) { wg.Add(1) go func(idx int) { defer wg.Done() - ip, _, err := ipam.AllocateIP("", "", "auto_assign", "", "eni-"+itoa(idx), "i-"+itoa(idx)) + ip, _, err := ipam.AllocateIP("", "", PurposeENIPublic, "", "eni-"+itoa(idx), "i-"+itoa(idx)) results[idx] = ip errs[idx] = err }(i) @@ -192,13 +187,13 @@ func TestExternalIPAM_MultiPool(t *testing.T) { ipam := setupTestExternalIPAM(t, pools) // Allocate from US pool - ip1, poolName1, err := ipam.AllocateIP("us-east-1", "us-east-1a", "auto_assign", "", "eni-1", "i-1") + ip1, poolName1, err := ipam.AllocateIP("us-east-1", "us-east-1a", PurposeENIPublic, "", "eni-1", "i-1") require.NoError(t, err) assert.Equal(t, "203.0.113.3", ip1) // .2 reserved for gateway assert.Equal(t, "us-east", poolName1) // Allocate from EU pool - ip2, poolName2, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", "auto_assign", "", "eni-2", "i-2") + ip2, poolName2, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", PurposeENIPublic, "", "eni-2", "i-2") require.NoError(t, err) assert.Equal(t, "198.51.100.3", ip2) assert.Equal(t, "eu-west", poolName2) @@ -234,19 +229,19 @@ func TestExternalIPAM_PoolFallback(t *testing.T) { ipam := setupTestExternalIPAM(t, pools) // AZ match: us-east-1a → az-pool - ip1, pool1, err := ipam.AllocateIP("us-east-1", "us-east-1a", "auto_assign", "", "eni-1", "i-1") + ip1, pool1, err := ipam.AllocateIP("us-east-1", "us-east-1a", PurposeENIPublic, "", "eni-1", "i-1") require.NoError(t, err) assert.Equal(t, "az-pool", pool1) assert.Equal(t, "10.0.0.2", ip1) // .1 reserved for gw // Region match (different AZ): us-east-1b → region-pool - ip2, pool2, err := ipam.AllocateIP("us-east-1", "us-east-1b", "auto_assign", "", "eni-2", "i-2") + ip2, pool2, err := ipam.AllocateIP("us-east-1", "us-east-1b", PurposeENIPublic, "", "eni-2", "i-2") require.NoError(t, err) assert.Equal(t, "region-pool", pool2) assert.Equal(t, "10.0.1.2", ip2) // No match: eu-west-1 → global-pool - ip3, pool3, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", "auto_assign", "", "eni-3", "i-3") + ip3, pool3, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", PurposeENIPublic, "", "eni-3", "i-3") require.NoError(t, err) assert.Equal(t, "global-pool", pool3) assert.Equal(t, "10.0.2.2", ip3) @@ -272,7 +267,7 @@ func TestExternalIPAM_SpecificPool(t *testing.T) { ipam := setupTestExternalIPAM(t, pools) // Allocate specifically from pool-b - ip, err := ipam.AllocateFromPool("pool-b", "elastic_ip", "eipalloc-test-b", "", "") + ip, err := ipam.AllocateFromPool("pool-b", PurposeEIP, "eipalloc-test-b", "", "") require.NoError(t, err) assert.Equal(t, "10.0.1.2", ip) // .1 reserved for gw } @@ -339,19 +334,19 @@ func TestExternalIPAM_RangeValidation(t *testing.T) { func TestExternalIPAM_InitFromConfig(t *testing.T) { pool := testPool() // Create IPAM twice — second init should be idempotent - _, nc, js := testutil.StartTestJetStream(t) + _, _, js := testutil.StartTestJetStream(t) // First init - ipam1, err := NewExternalIPAM(nc, js, []ExternalPoolConfig{pool}) + ipam1, err := NewExternalIPAM(js, []ExternalPoolConfig{pool}) require.NoError(t, err) // Allocate an IP - ip, _, err := ipam1.AllocateIP("", "", "auto_assign", "", "eni-1", "i-1") + ip, _, err := ipam1.AllocateIP("", "", PurposeENIPublic, "", "eni-1", "i-1") require.NoError(t, err) assert.Equal(t, "192.168.1.151", ip) // Second init (simulating restart) — should not lose allocation - ipam2, err := NewExternalIPAM(nc, js, []ExternalPoolConfig{pool}) + ipam2, err := NewExternalIPAM(js, []ExternalPoolConfig{pool}) require.NoError(t, err) record, err := ipam2.GetPoolRecord("wan") @@ -381,7 +376,7 @@ func TestExternalIPAM_NoPoolAvailable(t *testing.T) { ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) // No pool matches eu-west - _, _, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", "auto_assign", "", "eni-1", "i-1") + _, _, err := ipam.AllocateIP("eu-west-1", "eu-west-1a", PurposeENIPublic, "", "eni-1", "i-1") assert.ErrorContains(t, err, "InsufficientAddressCapacity") } @@ -389,247 +384,14 @@ func TestExternalIPAM_FindPoolByName_NotFound(t *testing.T) { pool := testPool() ipam := setupTestExternalIPAM(t, []ExternalPoolConfig{pool}) - // findPoolByName is private, but exercised via AllocateFromPool with unknown pool. - // The function returns nil when no pool matches, which means static allocation path. - // We verify by using NewExternalIPAMWithKV to directly check findPoolByName returns nil. + // AllocateFromPool with a non-existent pool name: the pool "nonexistent" + // has no KV record, so getRecord fails. kv := ipam.kv - ipam2 := NewExternalIPAMWithKV(nil, kv, []ExternalPoolConfig{pool}) - // AllocateFromPool with a non-existent pool name: findPoolByName returns nil, - // so pool.IsDHCP() is skipped and static allocation is used. - // The pool "nonexistent" has no KV record, so getRecord fails. - _, err := ipam2.AllocateFromPool("nonexistent", "auto_assign", "", "eni-1", "i-1") + ipam2 := NewExternalIPAMWithKV(kv, []ExternalPoolConfig{pool}) + _, err := ipam2.AllocateFromPool("nonexistent", PurposeENIPublic, "", "eni-1", "i-1") assert.Error(t, err) } -// dhcpStub captures acquire/release requests and replies with a -// deterministic lease so the handler-side IPAM code can be exercised -// without running vpcd. -type dhcpStub struct { - mu sync.Mutex - acquired []dhcp.AcquireRequestMsg - released []dhcp.ReleaseRequestMsg - nextIP string - acquireErr string - acquireCalls atomic.Int32 - releaseCalls atomic.Int32 - serverID string - leaseSeconds int64 - hwAddrOverride string - releaseErrOnce atomic.Bool -} - -func newDHCPStub(t *testing.T, nc *nats.Conn) *dhcpStub { - t.Helper() - s := &dhcpStub{ - nextIP: "192.168.1.151", - serverID: "192.168.1.1", - leaseSeconds: 3600, - } - acquire, err := nc.Subscribe(dhcp.TopicAcquire, func(msg *nats.Msg) { - s.acquireCalls.Add(1) - var req dhcp.AcquireRequestMsg - if err := json.Unmarshal(msg.Data, &req); err != nil { - respondTestJSON(t, msg, dhcp.AcquireReplyMsg{Error: "malformed request"}) - return - } - s.mu.Lock() - s.acquired = append(s.acquired, req) - ip := s.nextIP - errMsg := s.acquireErr - serverID := s.serverID - expires := time.Now().Add(time.Duration(s.leaseSeconds) * time.Second).Unix() - hw := s.hwAddrOverride - s.mu.Unlock() - - if errMsg != "" { - respondTestJSON(t, msg, dhcp.AcquireReplyMsg{Error: errMsg}) - return - } - if hw == "" { - hw = "02:00:00:aa:bb:cc" - } - respondTestJSON(t, msg, dhcp.AcquireReplyMsg{ - IP: ip, - SubnetMask: "255.255.255.0", - Routers: []string{"192.168.1.1"}, - DNS: []string{"192.168.1.1"}, - ServerID: serverID, - HWAddr: hw, - ExpiresUnix: expires, - }) - }) - require.NoError(t, err) - t.Cleanup(func() { _ = acquire.Unsubscribe() }) - - release, err := nc.Subscribe(dhcp.TopicRelease, func(msg *nats.Msg) { - s.releaseCalls.Add(1) - var req dhcp.ReleaseRequestMsg - _ = json.Unmarshal(msg.Data, &req) - s.mu.Lock() - s.released = append(s.released, req) - s.mu.Unlock() - reply := dhcp.ReleaseReplyMsg{} - if s.releaseErrOnce.CompareAndSwap(true, false) { - reply.Error = "first release fails" - } - respondTestJSON(t, msg, reply) - }) - require.NoError(t, err) - t.Cleanup(func() { _ = release.Unsubscribe() }) - return s -} - -func respondTestJSON(t *testing.T, msg *nats.Msg, v any) { - t.Helper() - data, err := json.Marshal(v) - require.NoError(t, err) - require.NoError(t, msg.Respond(data)) -} - -func TestExternalIPAM_DHCPAllocateAndRelease(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - stub := newDHCPStub(t, nc) - stub.nextIP = "192.168.1.200" - - pool := ExternalPoolConfig{ - Name: "wan-dhcp", - Source: "dhcp", - Gateway: "192.168.1.1", - GatewayIP: "192.168.1.1", // pre-set so initPool skips gateway DORA - PrefixLen: 24, - DhcpBindBridge: "br-wan", - } - ipam, err := NewExternalIPAM(nc, js, []ExternalPoolConfig{pool}) - require.NoError(t, err) - - ip, err := ipam.AllocateFromPool("wan-dhcp", "elastic_ip", "eipalloc-123", "eni-abc", "i-xyz") - require.NoError(t, err) - assert.Equal(t, "192.168.1.200", ip) - require.EqualValues(t, 1, stub.acquireCalls.Load()) - - stub.mu.Lock() - got := stub.acquired[0] - stub.mu.Unlock() - assert.Equal(t, "eipalloc-123", got.ClientID, "clientID precedence: allocID first") - assert.Equal(t, "eni-abc", got.Hostname, "hostname defaults to eniID") - assert.Equal(t, "i-xyz", got.VendorClass, "vendor class i-") - assert.Equal(t, "wan-dhcp", got.PoolName) - assert.Equal(t, "br-wan", got.Bridge) - - record, err := ipam.GetPoolRecord("wan-dhcp") - require.NoError(t, err) - alloc, ok := record.Allocated["192.168.1.200"] - require.True(t, ok) - assert.Equal(t, "192.168.1.1", alloc.DHCPServerID) - assert.NotZero(t, alloc.LeaseExpiresUnix) - assert.Equal(t, "02:00:00:aa:bb:cc", alloc.HWAddr) - - require.NoError(t, ipam.ReleaseIP("wan-dhcp", "192.168.1.200")) - require.EqualValues(t, 1, stub.releaseCalls.Load()) - stub.mu.Lock() - assert.Equal(t, "eipalloc-123", stub.released[0].ClientID) - stub.mu.Unlock() -} - -func TestExternalIPAM_DHCPAcquireErrorPropagates(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - stub := newDHCPStub(t, nc) - stub.acquireErr = "server unreachable" - - pool := ExternalPoolConfig{ - Name: "wan-dhcp-err", Source: "dhcp", Gateway: "192.168.1.1", - GatewayIP: "192.168.1.1", PrefixLen: 24, DhcpBindBridge: "br-wan", - } - ipam, err := NewExternalIPAM(nc, js, []ExternalPoolConfig{pool}) - require.NoError(t, err) - - _, err = ipam.AllocateFromPool("wan-dhcp-err", "elastic_ip", "eipalloc-err", "", "") - require.Error(t, err) - assert.Contains(t, err.Error(), "server unreachable") -} - -func TestExternalIPAM_DHCPGatewayFromLease(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - stub := newDHCPStub(t, nc) - stub.nextIP = "192.168.3.247" - - pool := ExternalPoolConfig{ - Name: "wan-dhcp-gw", Source: "dhcp", Gateway: "192.168.3.1", - PrefixLen: 24, DhcpBindBridge: "br-wan", - // GatewayIP intentionally empty — initPool must pull it from DHCP. - } - _, err := NewExternalIPAM(nc, js, []ExternalPoolConfig{pool}) - require.NoError(t, err) - assert.EqualValues(t, 1, stub.acquireCalls.Load()) - - stub.mu.Lock() - got := stub.acquired[0] - stub.mu.Unlock() - assert.Equal(t, "gateway-wan-dhcp-gw", got.ClientID) - assert.Equal(t, "mulga-spinifex-gw", got.VendorClass) -} - -func TestExternalIPAM_DHCPGatewayErrorPropagates(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - stub := newDHCPStub(t, nc) - stub.acquireErr = "no response from upstream" - - pool := ExternalPoolConfig{ - Name: "wan-gw-err", Source: "dhcp", Gateway: "192.168.1.1", - PrefixLen: 24, DhcpBindBridge: "br-wan", - } - _, err := NewExternalIPAM(nc, js, []ExternalPoolConfig{pool}) - require.Error(t, err) - assert.Contains(t, err.Error(), "no response from upstream") -} - -func TestExternalIPAM_DHCPReleaseErrorIsNonFatal(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - stub := newDHCPStub(t, nc) - stub.nextIP = "192.168.1.155" - stub.releaseErrOnce.Store(true) - - pool := ExternalPoolConfig{ - Name: "wan-rel-err", Source: "dhcp", Gateway: "192.168.1.1", - GatewayIP: "192.168.1.1", PrefixLen: 24, DhcpBindBridge: "br-wan", - } - ipam, err := NewExternalIPAM(nc, js, []ExternalPoolConfig{pool}) - require.NoError(t, err) - - ip, err := ipam.AllocateFromPool("wan-rel-err", "elastic_ip", "eipalloc-rel", "", "") - require.NoError(t, err) - - // Release reports vpcd failure via slog.Warn but ReleaseIP itself - // still succeeds — the allocation is removed from KV. - require.NoError(t, ipam.ReleaseIP("wan-rel-err", ip)) - rec, err := ipam.GetPoolRecord("wan-rel-err") - require.NoError(t, err) - _, stillThere := rec.Allocated[ip] - assert.False(t, stillThere) -} - -func TestDHCPIdentityOptions(t *testing.T) { - tests := []struct { - name string - eniID string - instanceID string - pool string - wantHost string - wantVendor string - }{ - {"eni+instance", "eni-1", "i-abcd", "wan", "eni-1", "i-abcd"}, - {"instance only", "", "i-xyz", "wan", "spinifex-i-xyz", "i-xyz"}, - {"neither", "", "", "wan", "spinifex-wan", "mulga-spinifex"}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - host, vendor := dhcpIdentityOptions(tc.eniID, tc.instanceID, tc.pool) - assert.Equal(t, tc.wantHost, host) - assert.Equal(t, tc.wantVendor, vendor) - }) - } -} - // TestNextAvailableExternalIP_SkipsGwLrpRange verifies the per-VM EIP allocator // honors gw_lrp_range by skipping IPs reserved for vpcd's gateway LRPs (siv-36). func TestNextAvailableExternalIP_SkipsGwLrpRange(t *testing.T) { diff --git a/spinifex/handlers/ec2/vpc/ipam.go b/spinifex/handlers/ec2/vpc/ipam.go index 92a40208a..b79d27f70 100644 --- a/spinifex/handlers/ec2/vpc/ipam.go +++ b/spinifex/handlers/ec2/vpc/ipam.go @@ -15,14 +15,22 @@ import ( const ( KVBucketIPAM = "spinifex-vpc-ipam" - KVBucketIPAMVersion = 1 + KVBucketIPAMVersion = 2 ) +// IPEntry tags one IP allocation with its Purpose + owner (eni-, eipalloc-, +// etc) so multi-VPC clusters can reclaim/audit by (owner, purpose). +type IPEntry struct { + IP string `json:"ip"` + Purpose string `json:"purpose"` // one of the Purpose* constants + OwnerID string `json:"owner_id,omitempty"` // ENI / EIP / NATGW / IGW resource ID +} + // IPAMRecord tracks allocated IPs for a subnet. type IPAMRecord struct { - SubnetId string `json:"subnet_id"` - CidrBlock string `json:"cidr_block"` - Allocated []string `json:"allocated"` // List of allocated IPs + SubnetId string `json:"subnet_id"` + CidrBlock string `json:"cidr_block"` + Allocated []IPEntry `json:"allocated"` } // IPAM manages IP address allocation for VPC subnets using NATS KV with CAS. @@ -36,7 +44,7 @@ func NewIPAM(js nats.JetStreamContext) (*IPAM, error) { if err != nil { return nil, fmt.Errorf("failed to create IPAM KV bucket: %w", err) } - if err := migrate.DefaultRegistry.RunKV(KVBucketIPAM, kv, KVBucketIPAMVersion); err != nil { + if err := migrate.DefaultRegistry.RunKVWithJetStream(KVBucketIPAM, kv, js, KVBucketIPAMVersion); err != nil { return nil, fmt.Errorf("migrate %s: %w", KVBucketIPAM, err) } return &IPAM{kv: kv}, nil @@ -50,8 +58,10 @@ func NewIPAMWithKV(kv nats.KeyValue) *IPAM { // AllocateIP allocates an IP address from the given subnet. // It reserves the first 4 and last IP per AWS convention: // .0=network, .1=gateway, .2=DNS, .3=reserved, .255=broadcast (for /24). -// Uses CAS for conflict-free allocation across nodes. -func (m *IPAM) AllocateIP(subnetId, cidrBlock string) (string, error) { +// Uses CAS for conflict-free allocation across nodes. purpose is one of the +// Purpose* constants in purpose.go; ownerID is the ENI / instance / EIP +// resource ID that owns this allocation. +func (m *IPAM) AllocateIP(subnetId, cidrBlock, purpose, ownerID string) (string, error) { for attempt := range 5 { record, revision, err := m.getRecord(subnetId) if err != nil && !errors.Is(err, nats.ErrKeyNotFound) { @@ -71,7 +81,7 @@ func (m *IPAM) AllocateIP(subnetId, cidrBlock string) (string, error) { return "", err } - record.Allocated = append(record.Allocated, ip) + record.Allocated = append(record.Allocated, IPEntry{IP: ip, Purpose: purpose, OwnerID: ownerID}) // CAS write data, err := json.Marshal(record) @@ -90,7 +100,7 @@ func (m *IPAM) AllocateIP(subnetId, cidrBlock string) (string, error) { continue // CAS conflict, retry } - slog.Info("IPAM allocated IP", "subnet", subnetId, "ip", ip) + slog.Info("IPAM allocated IP", "subnet", subnetId, "ip", ip, "purpose", purpose, "owner", ownerID) return ip, nil } @@ -109,8 +119,8 @@ func (m *IPAM) ReleaseIP(subnetId, ip string) error { } found := false - for i, allocated := range record.Allocated { - if allocated == ip { + for i, entry := range record.Allocated { + if entry.IP == ip { record.Allocated = append(record.Allocated[:i], record.Allocated[i+1:]...) found = true break @@ -137,8 +147,8 @@ func (m *IPAM) ReleaseIP(subnetId, ip string) error { return fmt.Errorf("IPAM release failed after CAS retries for subnet %s", subnetId) } -// AllocatedIPs returns the list of allocated IPs for a subnet. -func (m *IPAM) AllocatedIPs(subnetId string) ([]string, error) { +// AllocatedIPs returns the list of allocated IP entries for a subnet. +func (m *IPAM) AllocatedIPs(subnetId string) ([]IPEntry, error) { record, _, err := m.getRecord(subnetId) if err != nil { if errors.Is(err, nats.ErrKeyNotFound) { @@ -177,8 +187,8 @@ func (m *IPAM) nextAvailableIP(record *IPAMRecord) (string, error) { } allocated := make(map[string]bool, len(record.Allocated)) - for _, ip := range record.Allocated { - allocated[ip] = true + for _, entry := range record.Allocated { + allocated[entry.IP] = true } ones, bits := ipNet.Mask.Size() diff --git a/spinifex/handlers/ec2/vpc/ipam_test.go b/spinifex/handlers/ec2/vpc/ipam_test.go index 7006e0e23..ac7473d4c 100644 --- a/spinifex/handlers/ec2/vpc/ipam_test.go +++ b/spinifex/handlers/ec2/vpc/ipam_test.go @@ -22,7 +22,7 @@ func setupTestIPAM(t *testing.T) *IPAM { func TestIPAM_AllocateFirst(t *testing.T) { ipam := setupTestIPAM(t) - ip, err := ipam.AllocateIP("subnet-1", "10.0.1.0/24") + ip, err := ipam.AllocateIP("subnet-1", "10.0.1.0/24", PurposeENIPrimary, "eni-1") require.NoError(t, err) // First allocable IP is .4 (skip .0 network, .1 gateway, .2 DNS, .3 reserved) assert.Equal(t, "10.0.1.4", ip) @@ -31,15 +31,15 @@ func TestIPAM_AllocateFirst(t *testing.T) { func TestIPAM_AllocateSequential(t *testing.T) { ipam := setupTestIPAM(t) - ip1, err := ipam.AllocateIP("subnet-seq", "10.0.2.0/24") + ip1, err := ipam.AllocateIP("subnet-seq", "10.0.2.0/24", PurposeENIPrimary, "eni-seq") require.NoError(t, err) assert.Equal(t, "10.0.2.4", ip1) - ip2, err := ipam.AllocateIP("subnet-seq", "10.0.2.0/24") + ip2, err := ipam.AllocateIP("subnet-seq", "10.0.2.0/24", PurposeENIPrimary, "eni-seq") require.NoError(t, err) assert.Equal(t, "10.0.2.5", ip2) - ip3, err := ipam.AllocateIP("subnet-seq", "10.0.2.0/24") + ip3, err := ipam.AllocateIP("subnet-seq", "10.0.2.0/24", PurposeENIPrimary, "eni-seq") require.NoError(t, err) assert.Equal(t, "10.0.2.6", ip3) } @@ -47,11 +47,11 @@ func TestIPAM_AllocateSequential(t *testing.T) { func TestIPAM_Release(t *testing.T) { ipam := setupTestIPAM(t) - ip1, err := ipam.AllocateIP("subnet-rel", "10.0.3.0/24") + ip1, err := ipam.AllocateIP("subnet-rel", "10.0.3.0/24", PurposeENIPrimary, "eni-rel") require.NoError(t, err) assert.Equal(t, "10.0.3.4", ip1) - ip2, err := ipam.AllocateIP("subnet-rel", "10.0.3.0/24") + ip2, err := ipam.AllocateIP("subnet-rel", "10.0.3.0/24", PurposeENIPrimary, "eni-rel") require.NoError(t, err) assert.Equal(t, "10.0.3.5", ip2) @@ -60,7 +60,7 @@ func TestIPAM_Release(t *testing.T) { require.NoError(t, err) // Next allocation should reuse the released IP - ip3, err := ipam.AllocateIP("subnet-rel", "10.0.3.0/24") + ip3, err := ipam.AllocateIP("subnet-rel", "10.0.3.0/24", PurposeENIPrimary, "eni-rel") require.NoError(t, err) assert.Equal(t, "10.0.3.4", ip3) } @@ -68,7 +68,7 @@ func TestIPAM_Release(t *testing.T) { func TestIPAM_ReleaseNotAllocated(t *testing.T) { ipam := setupTestIPAM(t) - _, err := ipam.AllocateIP("subnet-rna", "10.0.4.0/24") + _, err := ipam.AllocateIP("subnet-rna", "10.0.4.0/24", PurposeENIPrimary, "eni-rna") require.NoError(t, err) err = ipam.ReleaseIP("subnet-rna", "10.0.4.99") @@ -91,7 +91,7 @@ func TestIPAM_Exhaustion(t *testing.T) { var allocated []string for i := range 11 { - ip, err := ipam.AllocateIP(subnetId, cidr) + ip, err := ipam.AllocateIP(subnetId, cidr, PurposeENIPrimary, "eni-test") require.NoError(t, err, "allocation %d should succeed", i) allocated = append(allocated, ip) } @@ -101,7 +101,7 @@ func TestIPAM_Exhaustion(t *testing.T) { assert.Equal(t, "10.0.5.14", allocated[len(allocated)-1]) // Next allocation should fail — subnet exhausted - _, err := ipam.AllocateIP(subnetId, cidr) + _, err := ipam.AllocateIP(subnetId, cidr, PurposeENIPrimary, "eni-test") assert.ErrorContains(t, err, "exhausted") } @@ -113,7 +113,7 @@ func TestIPAM_ReservedIPs(t *testing.T) { // First 4 allocations should be .4, .5, .6, .7 (skipping .0-.3) for i := 4; i <= 7; i++ { - ip, err := ipam.AllocateIP(subnetId, cidr) + ip, err := ipam.AllocateIP(subnetId, cidr, PurposeENIPrimary, "eni-test") require.NoError(t, err) expected := "10.0.6." + itoa(i) assert.Equal(t, expected, ip) @@ -129,29 +129,31 @@ func TestIPAM_AllocatedIPs(t *testing.T) { assert.Nil(t, ips) // Allocate some IPs - _, _ = ipam.AllocateIP("subnet-list", "10.0.7.0/24") - _, _ = ipam.AllocateIP("subnet-list", "10.0.7.0/24") + _, _ = ipam.AllocateIP("subnet-list", "10.0.7.0/24", PurposeENIPrimary, "eni-list") + _, _ = ipam.AllocateIP("subnet-list", "10.0.7.0/24", PurposeENIPrimary, "eni-list") ips, err = ipam.AllocatedIPs("subnet-list") require.NoError(t, err) assert.Len(t, ips, 2) - assert.Equal(t, "10.0.7.4", ips[0]) - assert.Equal(t, "10.0.7.5", ips[1]) + assert.Equal(t, "10.0.7.4", ips[0].IP) + assert.Equal(t, PurposeENIPrimary, ips[0].Purpose) + assert.Equal(t, "eni-list", ips[0].OwnerID) + assert.Equal(t, "10.0.7.5", ips[1].IP) } func TestIPAM_MultipleSubnets(t *testing.T) { ipam := setupTestIPAM(t) - ip1, err := ipam.AllocateIP("subnet-a", "10.0.10.0/24") + ip1, err := ipam.AllocateIP("subnet-a", "10.0.10.0/24", PurposeENIPrimary, "eni-a") require.NoError(t, err) assert.Equal(t, "10.0.10.4", ip1) - ip2, err := ipam.AllocateIP("subnet-b", "10.0.20.0/24") + ip2, err := ipam.AllocateIP("subnet-b", "10.0.20.0/24", PurposeENIPrimary, "eni-b") require.NoError(t, err) assert.Equal(t, "10.0.20.4", ip2) // Each subnet tracks independently - ip3, err := ipam.AllocateIP("subnet-a", "10.0.10.0/24") + ip3, err := ipam.AllocateIP("subnet-a", "10.0.10.0/24", PurposeENIPrimary, "eni-a") require.NoError(t, err) assert.Equal(t, "10.0.10.5", ip3) } @@ -160,7 +162,7 @@ func TestIPAM_LargerSubnet(t *testing.T) { ipam := setupTestIPAM(t) // /20 subnet — first allocable is still .4 - ip, err := ipam.AllocateIP("subnet-big", "172.16.0.0/20") + ip, err := ipam.AllocateIP("subnet-big", "172.16.0.0/20", PurposeENIPrimary, "eni-big") require.NoError(t, err) assert.Equal(t, "172.16.0.4", ip) } @@ -177,14 +179,14 @@ func TestIPAM_NewIPAMWithKV(t *testing.T) { require.NotNil(t, ipam) // Should work the same as regular IPAM - ip, err := ipam.AllocateIP("subnet-kv", "10.0.1.0/24") + ip, err := ipam.AllocateIP("subnet-kv", "10.0.1.0/24", PurposeENIPrimary, "eni-kv") require.NoError(t, err) assert.Equal(t, "10.0.1.4", ip) } func TestIPAM_InvalidCIDR(t *testing.T) { ipam := setupTestIPAM(t) - _, err := ipam.AllocateIP("subnet-bad", "not-a-cidr") + _, err := ipam.AllocateIP("subnet-bad", "not-a-cidr", PurposeENIPrimary, "eni-bad") assert.Error(t, err) assert.Contains(t, err.Error(), "parse CIDR") } diff --git a/spinifex/handlers/ec2/vpc/purpose.go b/spinifex/handlers/ec2/vpc/purpose.go new file mode 100644 index 000000000..9fa2070cc --- /dev/null +++ b/spinifex/handlers/ec2/vpc/purpose.go @@ -0,0 +1,31 @@ +package handlers_ec2_vpc + +// Purpose tags every IP allocation with its semantic role so multi-VPC +// clusters can reclaim / audit by (owner, purpose). The constants match the +// JSON values written to KV records. +const ( + PurposeENIPrimary = "eni-primary" + PurposeENISecondary = "eni-secondary" + PurposeENIPublic = "eni-public" // auto-assigned public IP + PurposeEIP = "eip" + PurposeNATGWExternal = "natgw-external" + PurposeIGWLRP = "igw-lrp" + // Future-reserved values; defined now so call sites and migrations can + // reference them without a follow-up rename. + PurposeEdgeLRExternal = "edge-lr-external" + PurposeNodeMgmt = "node-mgmt" // currently .env + PurposeSubnetRouter = "subnet-router" // .1 + PurposeSubnetDNS = "subnet-dns" // .2 + PurposeSubnetReserved = "subnet-reserved" // .0, .3, .last + // PurposeUnknown tags records the migration could not classify (e.g. + // orphan IPs with no owning ENIRecord). Surfaces in logs for triage. + PurposeUnknown = "unknown" +) + +// LegacyExternalTypeToPurpose maps the old ExternalIPAllocation.Type values +// onto the new Purpose enum. Used by the v1→v2 external-IPAM migration. +var LegacyExternalTypeToPurpose = map[string]string{ + "gateway": PurposeIGWLRP, + "auto_assign": PurposeENIPublic, + "elastic_ip": PurposeEIP, +} diff --git a/spinifex/handlers/ec2/vpc/security_group.go b/spinifex/handlers/ec2/vpc/security_group.go index 4664609bf..05354b628 100644 --- a/spinifex/handlers/ec2/vpc/security_group.go +++ b/spinifex/handlers/ec2/vpc/security_group.go @@ -1150,8 +1150,7 @@ func (s *VPCServiceImpl) createDefaultSecurityGroupInternal(accountID, vpcId str // deleteSecurityGroupInternal removes an SG record without the public-API // CannotDelete guard for default SGs. Used by DeleteVpc to cascade-delete the -// per-VPC default SG. Honors the Phase 7 sync contract so vpcd tear-down -// failures propagate. +// per-VPC default SG. Surfaces vpcd tear-down failures to the caller. func (s *VPCServiceImpl) deleteSecurityGroupInternal(accountID, groupId string) error { key := utils.AccountKey(accountID, groupId) entry, err := s.sgKV.Get(key) diff --git a/spinifex/handlers/ec2/vpc/service_impl.go b/spinifex/handlers/ec2/vpc/service_impl.go index 4790a0df5..b09d683e9 100644 --- a/spinifex/handlers/ec2/vpc/service_impl.go +++ b/spinifex/handlers/ec2/vpc/service_impl.go @@ -35,13 +35,18 @@ const ( KVBucketVNICounterVersion = 1 ) -// VPCRecord represents a stored VPC +// VPCRecord represents a stored VPC. AZ stamps the local availability zone +// at create time; the network/reconcile package filters its intent set by +// matching `vpc.AZ == localAZ` (with the legacy rule that empty matches +// every AZ for pre-Phase-2.2 records). New records always populate AZ from +// the daemon's config.AZ. type VPCRecord struct { VpcId string `json:"vpc_id"` CidrBlock string `json:"cidr_block"` State string `json:"state"` IsDefault bool `json:"is_default"` VNI int64 `json:"vni"` + AZ string `json:"az,omitempty"` EnableDnsHostnames bool `json:"enable_dns_hostnames"` EnableDnsSupport bool `json:"enable_dns_support"` EnableNetworkAddressUsageMetrics bool `json:"enable_network_address_usage_metrics"` @@ -86,6 +91,16 @@ func (s *VPCServiceImpl) SetExternalIPAM(ipam *ExternalIPAM, eipKV nats.KeyValue s.eipKV = eipKV } +// localAZ returns the node's local availability zone, sourced from +// s.config.AZ. Returns "" when no config is wired (mostly test paths), +// which the reconciler treats as a legacy record matching every AZ. +func (s *VPCServiceImpl) localAZ() string { + if s.config == nil { + return "" + } + return s.config.AZ +} + // NewVPCServiceImplWithNATS creates a VPC service with NATS JetStream for persistence func NewVPCServiceImplWithNATS(cfg *config.Config, natsConn *nats.Conn) (*VPCServiceImpl, error) { js, err := natsConn.JetStream() @@ -231,6 +246,7 @@ func (s *VPCServiceImpl) CreateVpc(input *ec2.CreateVpcInput, accountID string) State: "available", IsDefault: false, VNI: vni, + AZ: s.localAZ(), EnableDnsSupport: true, // AWS default EnableDnsHostnames: false, // AWS default Tags: utils.ExtractTags(input.TagSpecifications, "vpc"), @@ -1026,6 +1042,7 @@ func (s *VPCServiceImpl) EnsureDefaultVPC(accountID string, bootstrap ...Bootstr State: "available", IsDefault: true, VNI: vni, + AZ: s.localAZ(), EnableDnsSupport: true, // AWS default EnableDnsHostnames: true, // AWS default for default VPC Tags: map[string]string{"Name": "default"}, diff --git a/spinifex/migrate/002_external_ipam_purpose.go b/spinifex/migrate/002_external_ipam_purpose.go new file mode 100644 index 000000000..68baf0b5c --- /dev/null +++ b/spinifex/migrate/002_external_ipam_purpose.go @@ -0,0 +1,134 @@ +package migrate + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" +) + +// extIPAMBucket mirrors handlers/ec2/vpc.KVBucketExternalIPAM. Duplicated to +// keep the migrate package free of any handler dependency; if the bucket name +// changes, both constants must move together. +const extIPAMBucket = "spinifex-external-ipam" + +const extIPAMMaxRetries = 3 + +// extLegacyTypeToPurpose maps the v1 ExternalIPAllocation.Type values onto +// the v2 Purpose enum. Frozen copy of vpc.LegacyExternalTypeToPurpose to +// keep this migration self-contained. +var extLegacyTypeToPurpose = map[string]string{ + "gateway": "igw-lrp", + "auto_assign": "eni-public", + "elastic_ip": "eip", +} + +// extIPAllocV1V2 is a union of the v1 and v2 ExternalIPAllocation schemas. +// The migration reads the legacy Type field, writes the new Purpose field, +// and clears Type so the on-disk record matches the v2 struct exactly. +type extIPAllocV1V2 struct { + Type string `json:"type,omitempty"` + Purpose string `json:"purpose,omitempty"` + AllocationID string `json:"allocation_id,omitempty"` + Association string `json:"association,omitempty"` + ENIId string `json:"eni_id,omitempty"` + InstanceId string `json:"instance_id,omitempty"` + Note string `json:"note,omitempty"` +} + +type extIPAMRecordV1V2 struct { + PoolName string `json:"pool_name"` + RangeStart string `json:"range_start"` + RangeEnd string `json:"range_end"` + Gateway string `json:"gateway"` + GatewayIP string `json:"gateway_ip"` + PrefixLen int `json:"prefix_len"` + Region string `json:"region,omitempty"` + AZ string `json:"az,omitempty"` + GwLrpRangeStart string `json:"gw_lrp_range_start,omitempty"` + GwLrpRangeEnd string `json:"gw_lrp_range_end,omitempty"` + Allocated map[string]extIPAllocV1V2 `json:"allocated"` +} + +func init() { + DefaultRegistry.RegisterKV(extIPAMBucket, KVMigration{ + FromVersion: 1, + ToVersion: 2, + Description: "rename ExternalIPAllocation.Type → .Purpose with enum mapping", + Run: func(ctx KVContext) error { + keys, err := ctx.KV.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return nil + } + return fmt.Errorf("list keys: %w", err) + } + + for _, key := range keys { + if key == utils.VersionKey { + continue + } + if err := renameExternalIPAMType(ctx, key); err != nil { + return err + } + } + return nil + }, + }) +} + +func renameExternalIPAMType(ctx KVContext, key string) error { + var lastErr error + for attempt := range extIPAMMaxRetries { + entry, err := ctx.KV.Get(key) + if err != nil { + if errors.Is(err, nats.ErrKeyNotFound) { + return nil + } + return fmt.Errorf("get %s: %w", key, err) + } + + var record extIPAMRecordV1V2 + if err := json.Unmarshal(entry.Value(), &record); err != nil { + return fmt.Errorf("unmarshal %s: %w", key, err) + } + + changed := false + for ip, alloc := range record.Allocated { + if alloc.Purpose != "" { + continue + } + purpose := extLegacyTypeToPurpose[alloc.Type] + if purpose == "" { + ctx.Logger.Warn("external IPAM v1→v2 migration: unknown legacy type, tagging unknown", + "pool", key, "ip", ip, "legacy_type", alloc.Type) + purpose = "unknown" + } + alloc.Purpose = purpose + alloc.Type = "" + record.Allocated[ip] = alloc + changed = true + } + + if !changed { + return nil + } + + data, err := json.Marshal(record) + if err != nil { + return fmt.Errorf("marshal %s: %w", key, err) + } + if _, err := ctx.KV.Update(key, data, entry.Revision()); err != nil { + if errors.Is(err, nats.ErrKeyExists) { + ctx.Logger.Warn("external IPAM migration CAS conflict, retrying", "key", key, "attempt", attempt+1) + lastErr = err + continue + } + return fmt.Errorf("update %s: %w", key, err) + } + return nil + } + return fmt.Errorf("rename Type→Purpose for %s: exceeded %d CAS retries: %w", key, extIPAMMaxRetries, lastErr) +} diff --git a/spinifex/migrate/002_external_ipam_purpose_test.go b/spinifex/migrate/002_external_ipam_purpose_test.go new file mode 100644 index 000000000..2319807f0 --- /dev/null +++ b/spinifex/migrate/002_external_ipam_purpose_test.go @@ -0,0 +1,103 @@ +package migrate + +import ( + "encoding/json" + "log/slog" + "testing" + + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func extIPAMMigrationKV(t *testing.T) nats.KeyValue { + t.Helper() + _, nc := startTestNATS(t) + return createTestBucket(t, nc, "ext-ipam-migration-test") +} + +func runExtIPAMMigration(t *testing.T, kv nats.KeyValue) { + t.Helper() + for _, m := range DefaultRegistry.kvMigrations[extIPAMBucket] { + if m.FromVersion == 1 && m.ToVersion == 2 { + require.NoError(t, m.Run(KVContext{KV: kv, Logger: slog.Default()})) + return + } + } + t.Fatalf("v1→v2 migration not registered for %s", extIPAMBucket) +} + +func TestExtIPAMMigration_RenamesType(t *testing.T) { + kv := extIPAMMigrationKV(t) + + record := extIPAMRecordV1V2{ + PoolName: "wan", + RangeStart: "192.168.1.10", + RangeEnd: "192.168.1.250", + Gateway: "192.168.1.1", + PrefixLen: 24, + Allocated: map[string]extIPAllocV1V2{ + "192.168.1.10": {Type: "gateway", Note: "OVN router SNAT address"}, + "192.168.1.11": {Type: "auto_assign", ENIId: "eni-1", InstanceId: "i-1"}, + "192.168.1.12": {Type: "elastic_ip", AllocationID: "eipalloc-1"}, + "192.168.1.13": {Type: "junk-legacy", ENIId: "eni-zombie"}, + }, + } + data, err := json.Marshal(record) + require.NoError(t, err) + _, err = kv.Put("wan", data) + require.NoError(t, err) + + runExtIPAMMigration(t, kv) + + entry, err := kv.Get("wan") + require.NoError(t, err) + var got extIPAMRecordV1V2 + require.NoError(t, json.Unmarshal(entry.Value(), &got)) + + assert.Equal(t, "igw-lrp", got.Allocated["192.168.1.10"].Purpose) + assert.Equal(t, "eni-public", got.Allocated["192.168.1.11"].Purpose) + assert.Equal(t, "eip", got.Allocated["192.168.1.12"].Purpose) + assert.Equal(t, "unknown", got.Allocated["192.168.1.13"].Purpose, "unrecognized legacy types tagged unknown") + + for ip, alloc := range got.Allocated { + assert.Empty(t, alloc.Type, "legacy Type field cleared for %s", ip) + } + + assert.Equal(t, "eni-1", got.Allocated["192.168.1.11"].ENIId, "non-Type fields preserved") + assert.Equal(t, "eipalloc-1", got.Allocated["192.168.1.12"].AllocationID) +} + +func TestExtIPAMMigration_IdempotentOnAlreadyMigrated(t *testing.T) { + kv := extIPAMMigrationKV(t) + + record := extIPAMRecordV1V2{ + PoolName: "wan", + Allocated: map[string]extIPAllocV1V2{ + "192.168.1.10": {Purpose: "igw-lrp"}, + "192.168.1.11": {Purpose: "eni-public", ENIId: "eni-1"}, + }, + } + data, err := json.Marshal(record) + require.NoError(t, err) + _, err = kv.Put("wan", data) + require.NoError(t, err) + rev := getKVRevision(t, kv, "wan") + + runExtIPAMMigration(t, kv) + + newRev := getKVRevision(t, kv, "wan") + assert.Equal(t, rev, newRev, "no write when all records already have Purpose") +} + +func TestExtIPAMMigration_EmptyBucket(t *testing.T) { + kv := extIPAMMigrationKV(t) + runExtIPAMMigration(t, kv) +} + +func getKVRevision(t *testing.T, kv nats.KeyValue, key string) uint64 { + t.Helper() + entry, err := kv.Get(key) + require.NoError(t, err) + return entry.Revision() +} diff --git a/spinifex/migrate/003_ipam_purpose.go b/spinifex/migrate/003_ipam_purpose.go new file mode 100644 index 000000000..b1b98f838 --- /dev/null +++ b/spinifex/migrate/003_ipam_purpose.go @@ -0,0 +1,224 @@ +package migrate + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" +) + +// ipamBucket / eniBucket mirror handlers/ec2/vpc.KVBucketIPAM and +// .KVBucketENIs. Duplicated here so the migrate package stays handler-free. +const ( + ipamBucket = "spinifex-vpc-ipam" + eniBucket = "spinifex-vpc-enis" +) + +const ipamMigrationMaxRetries = 3 + +// ipEntryV2 mirrors handlers/ec2/vpc.IPEntry (frozen snapshot). +type ipEntryV2 struct { + IP string `json:"ip"` + Purpose string `json:"purpose"` + OwnerID string `json:"owner_id,omitempty"` +} + +// ipamRecordV1V2 is a union of v1 ([]string) and v2 ([]IPEntry) schemas. +// Allocated is left as raw JSON so we can detect which schema is on-disk +// and re-marshal as IPEntry. +type ipamRecordV1V2 struct { + SubnetId string `json:"subnet_id"` + CidrBlock string `json:"cidr_block"` + Allocated json.RawMessage `json:"allocated"` +} + +// ipamRecordV2 is the post-migration shape. +type ipamRecordV2 struct { + SubnetId string `json:"subnet_id"` + CidrBlock string `json:"cidr_block"` + Allocated []ipEntryV2 `json:"allocated"` +} + +// eniRecordSnapshot is the minimum ENIRecord shape needed to attribute IPAM +// entries to their owning ENI. Frozen copy keeps the migration self-contained. +type eniRecordSnapshot struct { + NetworkInterfaceId string `json:"network_interface_id"` + SubnetId string `json:"subnet_id"` + PrivateIpAddress string `json:"private_ip_address"` +} + +func init() { + DefaultRegistry.RegisterKV(ipamBucket, KVMigration{ + FromVersion: 1, + ToVersion: 2, + Description: "convert subnet IPAM Allocated []string → []IPEntry; backfill Purpose+OwnerID from ENIs", + Run: func(ctx KVContext) error { + ownerIndex, err := buildIPAMOwnerIndex(ctx.JetStream) + if err != nil { + return fmt.Errorf("build owner index: %w", err) + } + + keys, err := ctx.KV.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return nil + } + return fmt.Errorf("list keys: %w", err) + } + + for _, key := range keys { + if key == utils.VersionKey { + continue + } + if err := convertIPAMRecord(ctx, key, ownerIndex); err != nil { + return err + } + } + return nil + }, + }) +} + +// buildIPAMOwnerIndex returns map[subnetID]map[ip]eniID by walking the ENI +// bucket. Returns an empty index if the ENI bucket is missing +// (fresh-install case) so the migration still runs cleanly. +func buildIPAMOwnerIndex(js nats.JetStreamContext) (map[string]map[string]string, error) { + if js == nil { + // Called via the bare RunKV path (e.g. in tests with no JS). Skip + // owner attribution — all entries become Purpose=unknown so the + // schema-only conversion still succeeds. + return map[string]map[string]string{}, nil + } + eniKV, err := js.KeyValue(eniBucket) + if err != nil { + if errors.Is(err, nats.ErrBucketNotFound) { + return map[string]map[string]string{}, nil + } + return nil, fmt.Errorf("open ENI bucket: %w", err) + } + + keys, err := eniKV.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return map[string]map[string]string{}, nil + } + return nil, fmt.Errorf("list ENI keys: %w", err) + } + + index := make(map[string]map[string]string) + for _, key := range keys { + if key == utils.VersionKey { + continue + } + entry, err := eniKV.Get(key) + if err != nil { + if errors.Is(err, nats.ErrKeyNotFound) { + continue + } + return nil, fmt.Errorf("get ENI %s: %w", key, err) + } + var eni eniRecordSnapshot + if err := json.Unmarshal(entry.Value(), &eni); err != nil { + continue + } + if eni.SubnetId == "" || eni.PrivateIpAddress == "" { + continue + } + if _, ok := index[eni.SubnetId]; !ok { + index[eni.SubnetId] = make(map[string]string) + } + index[eni.SubnetId][eni.PrivateIpAddress] = eni.NetworkInterfaceId + } + return index, nil +} + +func convertIPAMRecord(ctx KVContext, key string, ownerIndex map[string]map[string]string) error { + var lastErr error + for attempt := range ipamMigrationMaxRetries { + entry, err := ctx.KV.Get(key) + if err != nil { + if errors.Is(err, nats.ErrKeyNotFound) { + return nil + } + return fmt.Errorf("get %s: %w", key, err) + } + + var raw ipamRecordV1V2 + if err := json.Unmarshal(entry.Value(), &raw); err != nil { + return fmt.Errorf("unmarshal %s: %w", key, err) + } + + // Detect schema: v2 records start with `[{`, v1 with `["`. + // Idempotent skip when already migrated. + if v2Entries, ok := tryDecodeV2(raw.Allocated); ok { + _ = v2Entries + return nil + } + + var legacy []string + if len(raw.Allocated) > 0 { + if err := json.Unmarshal(raw.Allocated, &legacy); err != nil { + return fmt.Errorf("unmarshal legacy []string for %s: %w", key, err) + } + } + + ownerByIP := ownerIndex[raw.SubnetId] + converted := make([]ipEntryV2, 0, len(legacy)) + for _, ip := range legacy { + ownerID, found := ownerByIP[ip] + purpose := "eni-primary" + if !found { + ownerID = "" + purpose = "unknown" + ctx.Logger.Warn("internal IPAM v1→v2 migration: no owning ENI, tagging unknown", + "subnet", raw.SubnetId, "ip", ip) + } + converted = append(converted, ipEntryV2{IP: ip, Purpose: purpose, OwnerID: ownerID}) + } + + newRecord := ipamRecordV2{ + SubnetId: raw.SubnetId, + CidrBlock: raw.CidrBlock, + Allocated: converted, + } + data, err := json.Marshal(newRecord) + if err != nil { + return fmt.Errorf("marshal %s: %w", key, err) + } + if _, err := ctx.KV.Update(key, data, entry.Revision()); err != nil { + if errors.Is(err, nats.ErrKeyExists) { + ctx.Logger.Warn("internal IPAM migration CAS conflict, retrying", "key", key, "attempt", attempt+1) + lastErr = err + continue + } + return fmt.Errorf("update %s: %w", key, err) + } + return nil + } + return fmt.Errorf("convert IPAM record %s: exceeded %d CAS retries: %w", key, ipamMigrationMaxRetries, lastErr) +} + +// tryDecodeV2 returns true when raw decodes as []ipEntryV2 with at least one +// entry whose IP field is set — the cheapest reliable v2 detector. Empty +// arrays decode as either schema; we treat them as already migrated. +func tryDecodeV2(raw json.RawMessage) ([]ipEntryV2, bool) { + if len(raw) == 0 { + return nil, true + } + var entries []ipEntryV2 + if err := json.Unmarshal(raw, &entries); err != nil { + return nil, false + } + for _, e := range entries { + if e.IP != "" { + return entries, true + } + } + // Zero-length array — already v2-shaped. + if len(entries) == 0 { + return entries, true + } + return nil, false +} diff --git a/spinifex/migrate/003_ipam_purpose_test.go b/spinifex/migrate/003_ipam_purpose_test.go new file mode 100644 index 000000000..b479b0856 --- /dev/null +++ b/spinifex/migrate/003_ipam_purpose_test.go @@ -0,0 +1,158 @@ +package migrate + +import ( + "encoding/json" + "log/slog" + "testing" + + "github.com/nats-io/nats.go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func setupIPAMMigrationKV(t *testing.T) (nats.KeyValue, nats.JetStreamContext) { + t.Helper() + _, nc := startTestNATS(t) + js, err := nc.JetStream() + require.NoError(t, err) + kv := createTestBucket(t, nc, ipamBucket) + return kv, js +} + +func runIPAMMigration(t *testing.T, kv nats.KeyValue, js nats.JetStreamContext) { + t.Helper() + for _, m := range DefaultRegistry.kvMigrations[ipamBucket] { + if m.FromVersion == 1 && m.ToVersion == 2 { + require.NoError(t, m.Run(KVContext{KV: kv, JetStream: js, Logger: slog.Default()})) + return + } + } + t.Fatalf("v1→v2 migration not registered for %s", ipamBucket) +} + +func seedENIBucket(t *testing.T, js nats.JetStreamContext, enis []eniRecordSnapshot) { + t.Helper() + kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: eniBucket, History: 1}) + require.NoError(t, err) + for _, eni := range enis { + data, err := json.Marshal(eni) + require.NoError(t, err) + _, err = kv.Put(eni.NetworkInterfaceId, data) + require.NoError(t, err) + } +} + +func TestIPAMMigration_ConvertsV1ToV2WithOwnerBackfill(t *testing.T) { + kv, js := setupIPAMMigrationKV(t) + seedENIBucket(t, js, []eniRecordSnapshot{ + {NetworkInterfaceId: "eni-aaa", SubnetId: "subnet-1", PrivateIpAddress: "10.0.1.4"}, + {NetworkInterfaceId: "eni-bbb", SubnetId: "subnet-1", PrivateIpAddress: "10.0.1.5"}, + {NetworkInterfaceId: "eni-ccc", SubnetId: "subnet-2", PrivateIpAddress: "10.0.2.4"}, + }) + + type legacyRecord struct { + SubnetId string `json:"subnet_id"` + CidrBlock string `json:"cidr_block"` + Allocated []string `json:"allocated"` + } + subnet1 := legacyRecord{ + SubnetId: "subnet-1", + CidrBlock: "10.0.1.0/24", + Allocated: []string{"10.0.1.4", "10.0.1.5", "10.0.1.99"}, // .99 has no owning ENI + } + data, err := json.Marshal(subnet1) + require.NoError(t, err) + _, err = kv.Put("subnet-1", data) + require.NoError(t, err) + + subnet2 := legacyRecord{ + SubnetId: "subnet-2", + CidrBlock: "10.0.2.0/24", + Allocated: []string{"10.0.2.4"}, + } + data, err = json.Marshal(subnet2) + require.NoError(t, err) + _, err = kv.Put("subnet-2", data) + require.NoError(t, err) + + runIPAMMigration(t, kv, js) + + entry, err := kv.Get("subnet-1") + require.NoError(t, err) + var got ipamRecordV2 + require.NoError(t, json.Unmarshal(entry.Value(), &got)) + + require.Len(t, got.Allocated, 3) + assert.Equal(t, "10.0.1.4", got.Allocated[0].IP) + assert.Equal(t, "eni-primary", got.Allocated[0].Purpose) + assert.Equal(t, "eni-aaa", got.Allocated[0].OwnerID) + assert.Equal(t, "10.0.1.5", got.Allocated[1].IP) + assert.Equal(t, "eni-bbb", got.Allocated[1].OwnerID) + assert.Equal(t, "10.0.1.99", got.Allocated[2].IP) + assert.Equal(t, "unknown", got.Allocated[2].Purpose, "orphan IP tagged unknown") + assert.Empty(t, got.Allocated[2].OwnerID) + + entry, err = kv.Get("subnet-2") + require.NoError(t, err) + require.NoError(t, json.Unmarshal(entry.Value(), &got)) + require.Len(t, got.Allocated, 1) + assert.Equal(t, "eni-ccc", got.Allocated[0].OwnerID) +} + +func TestIPAMMigration_IdempotentOnAlreadyV2(t *testing.T) { + kv, js := setupIPAMMigrationKV(t) + + v2 := ipamRecordV2{ + SubnetId: "subnet-1", + CidrBlock: "10.0.1.0/24", + Allocated: []ipEntryV2{ + {IP: "10.0.1.4", Purpose: "eni-primary", OwnerID: "eni-aaa"}, + }, + } + data, err := json.Marshal(v2) + require.NoError(t, err) + _, err = kv.Put("subnet-1", data) + require.NoError(t, err) + rev := getKVRevision(t, kv, "subnet-1") + + runIPAMMigration(t, kv, js) + + newRev := getKVRevision(t, kv, "subnet-1") + assert.Equal(t, rev, newRev, "no write when record is already v2") +} + +func TestIPAMMigration_NoJetStream_TagsUnknown(t *testing.T) { + _, nc := startTestNATS(t) + kv := createTestBucket(t, nc, ipamBucket) + + type legacyRecord struct { + SubnetId string `json:"subnet_id"` + CidrBlock string `json:"cidr_block"` + Allocated []string `json:"allocated"` + } + rec := legacyRecord{ + SubnetId: "subnet-1", + CidrBlock: "10.0.1.0/24", + Allocated: []string{"10.0.1.4"}, + } + data, err := json.Marshal(rec) + require.NoError(t, err) + _, err = kv.Put("subnet-1", data) + require.NoError(t, err) + + // Run migration with nil JetStream — schema conversion still runs but + // every entry becomes Purpose=unknown. + runIPAMMigration(t, kv, nil) + + entry, err := kv.Get("subnet-1") + require.NoError(t, err) + var got ipamRecordV2 + require.NoError(t, json.Unmarshal(entry.Value(), &got)) + require.Len(t, got.Allocated, 1) + assert.Equal(t, "unknown", got.Allocated[0].Purpose) +} + +func TestIPAMMigration_EmptyBucket(t *testing.T) { + kv, js := setupIPAMMigrationKV(t) + runIPAMMigration(t, kv, js) +} diff --git a/spinifex/migrate/migrate.go b/spinifex/migrate/migrate.go index 4a2f3d712..41f695352 100644 --- a/spinifex/migrate/migrate.go +++ b/spinifex/migrate/migrate.go @@ -21,9 +21,12 @@ type KVMigration struct { } // KVContext provides KV migration functions access to the bucket being migrated. +// JetStream is non-nil only when the caller used RunKVWithJetStream — used by +// migrations that need to read sibling buckets (e.g. owner-attribution lookups). type KVContext struct { - KV nats.KeyValue - Logger *slog.Logger + KV nats.KeyValue + JetStream nats.JetStreamContext + Logger *slog.Logger } // ConfigMigration represents a versioned transformation of on-disk config files. @@ -97,6 +100,13 @@ func (r *Registry) RegisterConfig(target string, m ConfigMigration) { }) } +// RunKVWithJetStream is RunKV with a JetStream handle attached to each +// migration's KVContext, enabling cross-bucket reads (e.g. owner-attribution +// during a backfill). Prefer plain RunKV when the migration is self-contained. +func (r *Registry) RunKVWithJetStream(bucket string, kv nats.KeyValue, js nats.JetStreamContext, targetVersion int) error { + return r.runKV(bucket, kv, js, targetVersion) +} + // RunKV executes all pending migrations for a KV bucket up to targetVersion. // It reads the current version via utils.ReadVersion, runs each migration in // order, and stamps the version via utils.WriteVersion after each step. @@ -109,6 +119,10 @@ func (r *Registry) RegisterConfig(target string, m ConfigMigration) { // an error — migration authors must register migrations before bumping version // constants. func (r *Registry) RunKV(bucket string, kv nats.KeyValue, targetVersion int) error { + return r.runKV(bucket, kv, nil, targetVersion) +} + +func (r *Registry) runKV(bucket string, kv nats.KeyValue, js nats.JetStreamContext, targetVersion int) error { current, err := utils.ReadVersion(kv) if err != nil { return fmt.Errorf("read version for %s: %w", bucket, err) @@ -168,7 +182,7 @@ func (r *Registry) RunKV(bucket string, kv nats.KeyValue, targetVersion int) err logger := slog.Default() for _, m := range pending { logger.Info("Running KV migration", "bucket", bucket, "from", m.FromVersion, "to", m.ToVersion, "description", m.Description) - ctx := KVContext{KV: kv, Logger: logger} + ctx := KVContext{KV: kv, JetStream: js, Logger: logger} if err := m.Run(ctx); err != nil { return fmt.Errorf("KV migration %s %d→%d failed: %w", bucket, m.FromVersion, m.ToVersion, err) } diff --git a/spinifex/network/external/eip.go b/spinifex/network/external/eip.go new file mode 100644 index 000000000..03e0fc8f5 --- /dev/null +++ b/spinifex/network/external/eip.go @@ -0,0 +1,49 @@ +package external + +import ( + "context" + "errors" + "fmt" + + "github.com/mulgadc/spinifex/spinifex/network/policy" +) + +// EIPManager attaches/detaches EIP NAT rules. Thin L5 wrapper over policy.NATManager +// that adds the flows-barrier so callers see the EIP reachable on return. +type EIPManager interface { + AttachEIP(ctx context.Context, eip policy.EIPSpec) error + DetachEIP(ctx context.Context, vpcID, logicalIP string) error +} + +type eipManager struct { + nat policy.NATManager + barrier FlowsBarrier +} + +var _ EIPManager = (*eipManager)(nil) + +// NewEIPManager constructs an EIPManager. barrier may be nil (no wait). +func NewEIPManager(nat policy.NATManager, barrier FlowsBarrier) (EIPManager, error) { + if nat == nil { + return nil, errors.New("EIPManager: NATManager required") + } + if barrier == nil { + barrier = func() error { return nil } + } + return &eipManager{nat: nat, barrier: barrier}, nil +} + +func (m *eipManager) AttachEIP(ctx context.Context, eip policy.EIPSpec) error { + if eip.VPCID == "" || eip.ExternalIP == "" || eip.LogicalIP == "" { + return fmt.Errorf("AttachEIP: VPCID, ExternalIP, LogicalIP all required") + } + if err := m.nat.AddEIP(ctx, eip); err != nil { + return err + } + _ = m.barrier() + return nil +} + +func (m *eipManager) DetachEIP(ctx context.Context, vpcID, logicalIP string) error { + return m.nat.DeleteEIP(ctx, vpcID, logicalIP) +} diff --git a/spinifex/network/external/eip_test.go b/spinifex/network/external/eip_test.go new file mode 100644 index 000000000..a838953be --- /dev/null +++ b/spinifex/network/external/eip_test.go @@ -0,0 +1,63 @@ +package external + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mulgadc/spinifex/spinifex/network/ovn/mock" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +func TestNewEIPManager_RejectsNilNAT(t *testing.T) { + _, err := NewEIPManager(nil, nil) + require.Error(t, err) +} + +func TestEIPManager_AttachAndDetach(t *testing.T) { + ctx := context.Background() + m := mock.New() + require.NoError(t, m.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: topology.VPCRouter("vpc-1")})) + nm, _ := policy.NewNATManager(m, policy.NATModeDistributed) + barrierCalls := 0 + mgr, err := NewEIPManager(nm, func() error { barrierCalls++; return nil }) + require.NoError(t, err) + + require.NoError(t, mgr.AttachEIP(ctx, policy.EIPSpec{ + VPCID: "vpc-1", ExternalIP: "1.2.3.4", LogicalIP: "10.0.0.5", + PortName: "port-a", MAC: "aa:bb:cc:dd:ee:ff", + })) + + var found *nbdb.NAT + for _, n := range m.NATs { + if n.LogicalIP == "10.0.0.5" { + found = n + } + } + require.NotNil(t, found) + assert.Equal(t, 1, barrierCalls) + + require.NoError(t, mgr.DetachEIP(ctx, "vpc-1", "10.0.0.5")) + for _, n := range m.NATs { + assert.NotEqual(t, "10.0.0.5", n.LogicalIP) + } +} + +func TestEIPManager_AttachRejectsEmptyFields(t *testing.T) { + nm, _ := policy.NewNATManager(mock.New(), policy.NATModeDistributed) + mgr, _ := NewEIPManager(nm, nil) + require.Error(t, mgr.AttachEIP(context.Background(), policy.EIPSpec{})) +} + +func TestEIPManager_DetachIdempotent(t *testing.T) { + ctx := context.Background() + m := mock.New() + require.NoError(t, m.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: topology.VPCRouter("vpc-1")})) + nm, _ := policy.NewNATManager(m, policy.NATModeDistributed) + mgr, _ := NewEIPManager(nm, nil) + require.NoError(t, mgr.DetachEIP(ctx, "vpc-1", "10.0.0.99")) +} diff --git a/spinifex/network/external/external.go b/spinifex/network/external/external.go new file mode 100644 index 000000000..0ba64cbe9 --- /dev/null +++ b/spinifex/network/external/external.go @@ -0,0 +1,4 @@ +// Package external is L5 of the spinifex network stack: VPC-facing external +// connectivity (IGW, EIP, NAT Gateway). Uses L1 (ovn), L3 (policy), and L0 +// (host) uplink CIDR resolved at startup. +package external diff --git a/spinifex/network/external/helpers.go b/spinifex/network/external/helpers.go new file mode 100644 index 000000000..0f1e9b344 --- /dev/null +++ b/spinifex/network/external/helpers.go @@ -0,0 +1,14 @@ +package external + +import ( + "net" +) + +func ipv4ToUint32(ip net.IP) uint32 { + v := ip.To4() + return uint32(v[0])<<24 | uint32(v[1])<<16 | uint32(v[2])<<8 | uint32(v[3]) +} + +func uint32ToIPv4(n uint32) net.IP { + return net.IPv4(byte(n>>24&0xff), byte(n>>16&0xff), byte(n>>8&0xff), byte(n&0xff)).To4() +} diff --git a/spinifex/network/external/igw.go b/spinifex/network/external/igw.go new file mode 100644 index 000000000..333bc7ca5 --- /dev/null +++ b/spinifex/network/external/igw.go @@ -0,0 +1,274 @@ +package external + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/mulgadc/spinifex/spinifex/network/ovn" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/topology" + "github.com/mulgadc/spinifex/spinifex/utils" +) + +// FlowsBarrier blocks until ovn-northd compiles NB→SB and chassis install flows. +// Production wraps `ovn-nbctl --wait=hv sync`; tests stub. +type FlowsBarrier func() error + +// IGWManager attaches/detaches Internet Gateways to VPCs. Idempotent. +type IGWManager interface { + AttachIGW(ctx context.Context, spec IGWSpec) error + DetachIGW(ctx context.Context, vpcID string) error +} + +// IGWManagerConfig is the construction-time bag for igwManager. +// FlowsBarrier defaults to a no-op when nil. +type IGWManagerConfig struct { + OVN ovn.Client + Routes policy.RouteManager + NAT policy.NATManager + Pool *ExternalPoolConfig + Allocator GatewayIPAllocator + Chassis []string + NATMode policy.NATMode + FlowsBarrier FlowsBarrier +} + +type igwManager struct { + ovn ovn.Client + routes policy.RouteManager + nat policy.NATManager + pool *ExternalPoolConfig + allocator GatewayIPAllocator + chassis []string + natMode policy.NATMode + barrier FlowsBarrier +} + +var _ IGWManager = (*igwManager)(nil) + +// NewIGWManager constructs an IGWManager from cfg. +func NewIGWManager(cfg IGWManagerConfig) (IGWManager, error) { + switch { + case cfg.OVN == nil: + return nil, errors.New("IGWManager: OVN client required") + case cfg.Routes == nil: + return nil, errors.New("IGWManager: RouteManager required") + case cfg.NAT == nil: + return nil, errors.New("IGWManager: NATManager required") + case cfg.Allocator == nil: + return nil, errors.New("IGWManager: GatewayIPAllocator required") + case cfg.NATMode == policy.NATModeUnknown: + return nil, errors.New("IGWManager: NATMode unknown; resolve from host.Wiring.UplinkMode()") + } + barrier := cfg.FlowsBarrier + if barrier == nil { + barrier = func() error { return nil } + } + return &igwManager{ + ovn: cfg.OVN, + routes: cfg.Routes, + nat: cfg.NAT, + pool: cfg.Pool, + allocator: cfg.Allocator, + chassis: cfg.Chassis, + natMode: cfg.NATMode, + barrier: barrier, + }, nil +} + +// AttachIGW wires external connectivity for spec.VPCID: external switch + +// localnet + gateway LRP + default route + chassis + flows barrier. +// Idempotent: returns nil if the external switch already exists. +func (m *igwManager) AttachIGW(ctx context.Context, spec IGWSpec) error { + if spec.VPCID == "" { + return errors.New("AttachIGW: VPCID required") + } + + extSwitchName := topology.ExternalSwitch(spec.VPCID) + extPortName := topology.ExternalLocalnetPort(spec.VPCID) + gwPortName := topology.GatewayRouterPort(spec.VPCID) + switchGWPortName := topology.GatewaySwitchPort(spec.VPCID) + routerName := topology.VPCRouter(spec.VPCID) + + extSwitch := &nbdb.LogicalSwitch{ + Name: extSwitchName, + ExternalIDs: map[string]string{ + "spinifex:vpc_id": spec.VPCID, + "spinifex:igw_id": spec.InternetGatewayID, + "spinifex:role": "external", + }, + } + existingSwitch, err := m.ovn.EnsureLogicalSwitch(ctx, extSwitch) + if err != nil { + return fmt.Errorf("ensure external switch %s: %w", extSwitchName, err) + } + if existingSwitch.UUID != extSwitch.UUID { + slog.Debug("external: IGW topology already exists, skipping", + "vpc_id", spec.VPCID, "ext_switch", extSwitchName) + return nil + } + + localnetOpts := map[string]string{"network_name": "external"} + if m.natMode == policy.NATModeCentralized { + localnetOpts["nat-addresses"] = "router" + } + if err := m.ovn.CreateLogicalSwitchPort(ctx, extSwitchName, &nbdb.LogicalSwitchPort{ + Name: extPortName, + Type: "localnet", + Addresses: []string{"unknown"}, + Options: localnetOpts, + ExternalIDs: map[string]string{ + "spinifex:vpc_id": spec.VPCID, + "spinifex:igw_id": spec.InternetGatewayID, + }, + }); err != nil { + _ = m.ovn.DeleteLogicalSwitch(ctx, extSwitchName) + return fmt.Errorf("create localnet port %s: %w", extPortName, err) + } + + gwNetwork, wanNexthop, gwLrpIP, err := m.resolveGatewayNetwork(ctx, spec.VPCID) + if err != nil { + _ = m.ovn.DeleteLogicalSwitchPort(ctx, extSwitchName, extPortName) + _ = m.ovn.DeleteLogicalSwitch(ctx, extSwitchName) + return err + } + + lrpExtIDs := map[string]string{ + "spinifex:vpc_id": spec.VPCID, + "spinifex:igw_id": spec.InternetGatewayID, + "spinifex:role": "gateway", + } + if gwLrpIP != "" { + lrpExtIDs[gatewayIPExtIDKey] = gwLrpIP + } + if err := m.ovn.CreateLogicalRouterPort(ctx, routerName, &nbdb.LogicalRouterPort{ + Name: gwPortName, + MAC: utils.HashMAC(gwPortName), + Networks: []string{gwNetwork}, + ExternalIDs: lrpExtIDs, + }); err != nil { + _ = m.ovn.DeleteLogicalSwitchPort(ctx, extSwitchName, extPortName) + _ = m.ovn.DeleteLogicalSwitch(ctx, extSwitchName) + return fmt.Errorf("create gateway router port %s: %w", gwPortName, err) + } + + if err := m.ovn.CreateLogicalSwitchPort(ctx, extSwitchName, &nbdb.LogicalSwitchPort{ + Name: switchGWPortName, + Type: "router", + Addresses: []string{"router"}, + Options: map[string]string{"router-port": gwPortName}, + ExternalIDs: map[string]string{ + "spinifex:vpc_id": spec.VPCID, + "spinifex:igw_id": spec.InternetGatewayID, + }, + }); err != nil { + _ = m.ovn.DeleteLogicalRouterPort(ctx, routerName, gwPortName) + _ = m.ovn.DeleteLogicalSwitchPort(ctx, extSwitchName, extPortName) + _ = m.ovn.DeleteLogicalSwitch(ctx, extSwitchName) + return fmt.Errorf("create switch gateway port %s: %w", switchGWPortName, err) + } + + if err := m.routes.AddDefaultRoute(ctx, spec.VPCID, wanNexthop, gwPortName); err != nil { + return fmt.Errorf("add default route on %s: %w", routerName, err) + } + + if len(m.chassis) == 0 { + slog.Warn("external: no chassis configured — gateway port has no chassis binding, external traffic will not flow", + "vpc_id", spec.VPCID, "gw_port", gwPortName) + } + for i, chassis := range m.chassis { + priority := max(20-(i*5), 1) + if err := m.ovn.SetGatewayChassis(ctx, gwPortName, chassis, priority); err != nil { + slog.Warn("external: failed to set gateway chassis", + "gw_port", gwPortName, "chassis", chassis, "priority", priority, "err", err) + } + } + + slog.Info("external: attached internet gateway", + "vpc_id", spec.VPCID, + "igw_id", spec.InternetGatewayID, + "ext_switch", extSwitchName, + "gw_port", gwPortName, + "lrp_network", gwNetwork, + "wan_nexthop", wanNexthop, + "chassis_count", len(m.chassis), + ) + + _ = m.barrier() + return nil +} + +// DetachIGW reverses AttachIGW. Idempotent. +func (m *igwManager) DetachIGW(ctx context.Context, vpcID string) error { + if vpcID == "" { + return errors.New("DetachIGW: vpcID required") + } + + extSwitchName := topology.ExternalSwitch(vpcID) + extPortName := topology.ExternalLocalnetPort(vpcID) + gwPortName := topology.GatewayRouterPort(vpcID) + switchGWPortName := topology.GatewaySwitchPort(vpcID) + routerName := topology.VPCRouter(vpcID) + + if err := m.routes.DeleteDefaultRoute(ctx, vpcID); err != nil { + slog.Warn("external: delete default route failed", "router", routerName, "err", err) + } + + if router, err := m.ovn.GetLogicalRouter(ctx, routerName); err == nil { + vpcCIDR := router.ExternalIDs["spinifex:cidr"] + if vpcCIDR != "" { + if err := m.nat.DeleteSNAT(ctx, vpcID, vpcCIDR); err != nil { + slog.Warn("external: delete IGW SNAT failed", "router", routerName, "cidr", vpcCIDR, "err", err) + } + } + } else { + slog.Warn("external: get router for NAT cleanup failed", "router", routerName, "err", err) + } + + if err := m.ovn.DeleteLogicalSwitchPort(ctx, extSwitchName, switchGWPortName); err != nil { + slog.Warn("external: delete switch gateway port failed", "port", switchGWPortName, "err", err) + } + if err := m.ovn.DeleteLogicalRouterPort(ctx, routerName, gwPortName); err != nil { + slog.Warn("external: delete gateway router port failed", "port", gwPortName, "err", err) + } + if err := m.ovn.DeleteLogicalSwitchPort(ctx, extSwitchName, extPortName); err != nil { + slog.Warn("external: delete localnet port failed", "port", extPortName, "err", err) + } + if err := m.ovn.DeleteLogicalSwitch(ctx, extSwitchName); err != nil { + return fmt.Errorf("delete external switch %s: %w", extSwitchName, err) + } + + if err := m.allocator.Release(ctx, vpcID); err != nil { + slog.Warn("external: allocator release failed", "vpc_id", vpcID, "err", err) + } + + slog.Info("external: detached internet gateway", "vpc_id", vpcID) + return nil +} + +// resolveGatewayNetwork picks LRP Networks/nexthop/IP. Distributed: link-local. +// Centralised: WAN-subnet IP (LRP is on-wire). +func (m *igwManager) resolveGatewayNetwork(ctx context.Context, vpcID string) (network, nexthop, gwIP string, err error) { + network = linkLocalGatewayNetwork + nexthop = linkLocalGatewayNexthop + + if m.pool != nil && m.pool.Gateway != "" { + nexthop = m.pool.Gateway + } + + if m.natMode != policy.NATModeCentralized || m.pool == nil { + return network, nexthop, "", nil + } + + ip, prefix, ok, allocErr := m.allocator.Allocate(ctx, vpcID, m.pool) + if allocErr != nil { + return "", "", "", fmt.Errorf("allocate gateway LRP IP: %w", allocErr) + } + if !ok { + return network, nexthop, "", nil + } + return fmt.Sprintf("%s/%d", ip, prefix), nexthop, ip, nil +} diff --git a/spinifex/network/external/igw_test.go b/spinifex/network/external/igw_test.go new file mode 100644 index 000000000..4ae72f3ed --- /dev/null +++ b/spinifex/network/external/igw_test.go @@ -0,0 +1,211 @@ +package external + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mulgadc/spinifex/spinifex/network/ovn/mock" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +func seedVPCRouter(t *testing.T, m *mock.Client, vpcID, cidr string) { + t.Helper() + extIDs := map[string]string{"spinifex:vpc_id": vpcID} + if cidr != "" { + extIDs["spinifex:cidr"] = cidr + } + require.NoError(t, m.CreateLogicalRouter(context.Background(), &nbdb.LogicalRouter{ + Name: topology.VPCRouter(vpcID), + ExternalIDs: extIDs, + })) +} + +func newTestIGWManager(t *testing.T, m *mock.Client, mode policy.NATMode, pool *ExternalPoolConfig, allocator GatewayIPAllocator, chassis []string) (IGWManager, *int) { + t.Helper() + nm, err := policy.NewNATManager(m, mode) + require.NoError(t, err) + barrierCalls := 0 + mgr, err := NewIGWManager(IGWManagerConfig{ + OVN: m, + Routes: policy.NewRouteManager(m), + NAT: nm, + Pool: pool, + Allocator: allocator, + Chassis: chassis, + NATMode: mode, + FlowsBarrier: func() error { + barrierCalls++ + return nil + }, + }) + require.NoError(t, err) + return mgr, &barrierCalls +} + +func TestNewIGWManager_RejectsMissingDeps(t *testing.T) { + _, err := NewIGWManager(IGWManagerConfig{NATMode: policy.NATModeDistributed}) + require.Error(t, err) + + nm, _ := policy.NewNATManager(mock.New(), policy.NATModeDistributed) + _, err = NewIGWManager(IGWManagerConfig{ + OVN: mock.New(), Routes: policy.NewRouteManager(mock.New()), NAT: nm, + Allocator: LinkLocalAllocator{}, + }) + require.Error(t, err, "NATMode unknown must be rejected") +} + +func TestAttachIGW_Distributed_LinkLocalLRP(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + pool := &ExternalPoolConfig{Name: "p", Gateway: "192.168.1.1", PrefixLen: 24} + mgr, calls := newTestIGWManager(t, m, policy.NATModeDistributed, pool, LinkLocalAllocator{}, []string{"chassis-a", "chassis-b"}) + + require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + + // External switch + ports created. + _, err := m.GetLogicalSwitch(ctx, topology.ExternalSwitch("vpc-1")) + require.NoError(t, err) + localnet, err := m.GetLogicalSwitchPort(ctx, topology.ExternalLocalnetPort("vpc-1")) + require.NoError(t, err) + assert.Equal(t, "external", localnet.Options["network_name"]) + _, hasNat := localnet.Options["nat-addresses"] + assert.False(t, hasNat, "distributed mode must NOT set nat-addresses") + + // Gateway LRP exists with link-local network. + lrp, err := m.GetLogicalRouterPort(ctx, topology.GatewayRouterPort("vpc-1")) + require.NoError(t, err) + assert.Equal(t, []string{linkLocalGatewayNetwork}, lrp.Networks) + _, hasGwIP := lrp.ExternalIDs[gatewayIPExtIDKey] + assert.False(t, hasGwIP, "link-local LRP must not record a gateway IP") + + // Default route points at pool.Gateway, OutputPort pinned. + route, err := m.FindStaticRoute(ctx, topology.VPCRouter("vpc-1"), "0.0.0.0/0") + require.NoError(t, err) + require.NotNil(t, route) + assert.Equal(t, pool.Gateway, route.Nexthop) + require.NotNil(t, route.OutputPort) + assert.Equal(t, topology.GatewayRouterPort("vpc-1"), *route.OutputPort) + + // Gateway chassis set once per chassis. + assert.Equal(t, 2, m.SetGatewayChassisCalls) + + // Flows barrier ran. + assert.Equal(t, 1, *calls) +} + +func TestAttachIGW_Centralized_AllocatesGwLrpIP(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + pool := &ExternalPoolConfig{ + Name: "p", Gateway: "192.168.1.1", PrefixLen: 24, + GwLrpRangeStart: "192.168.1.240", GwLrpRangeEnd: "192.168.1.243", + } + mgr, _ := newTestIGWManager(t, m, policy.NATModeCentralized, pool, NewStaticRangeAllocator(m), []string{"chassis-a"}) + + require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + + localnet, err := m.GetLogicalSwitchPort(ctx, topology.ExternalLocalnetPort("vpc-1")) + require.NoError(t, err) + assert.Equal(t, "router", localnet.Options["nat-addresses"], "centralized mode must set nat-addresses=router") + + lrp, err := m.GetLogicalRouterPort(ctx, topology.GatewayRouterPort("vpc-1")) + require.NoError(t, err) + assert.Equal(t, []string{"192.168.1.240/24"}, lrp.Networks) + assert.Equal(t, "192.168.1.240", lrp.ExternalIDs[gatewayIPExtIDKey]) +} + +func TestAttachIGW_IsIdempotent(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + mgr, _ := newTestIGWManager(t, m, policy.NATModeDistributed, nil, LinkLocalAllocator{}, nil) + + require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + + // Still only one LRP. + lrps, err := m.ListLogicalRouterPorts(ctx) + require.NoError(t, err) + gwCount := 0 + for _, lrp := range lrps { + if lrp.Name == topology.GatewayRouterPort("vpc-1") { + gwCount++ + } + } + assert.Equal(t, 1, gwCount) +} + +func TestAttachIGW_NoPoolNoChassis_UsesLinkLocalFallbacks(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + mgr, _ := newTestIGWManager(t, m, policy.NATModeDistributed, nil, LinkLocalAllocator{}, nil) + + require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + + route, err := m.FindStaticRoute(ctx, topology.VPCRouter("vpc-1"), "0.0.0.0/0") + require.NoError(t, err) + require.NotNil(t, route) + assert.Equal(t, linkLocalGatewayNexthop, route.Nexthop) + assert.Equal(t, 0, m.SetGatewayChassisCalls) +} + +func TestDetachIGW_RemovesEverything(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + pool := &ExternalPoolConfig{Name: "p", Gateway: "192.168.1.1", PrefixLen: 24} + mgr, _ := newTestIGWManager(t, m, policy.NATModeDistributed, pool, LinkLocalAllocator{}, []string{"chassis-a"}) + + require.NoError(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + require.NoError(t, mgr.DetachIGW(ctx, "vpc-1")) + + _, err := m.GetLogicalSwitch(ctx, topology.ExternalSwitch("vpc-1")) + assert.Error(t, err) + _, err = m.GetLogicalRouterPort(ctx, topology.GatewayRouterPort("vpc-1")) + assert.Error(t, err) + route, err := m.FindStaticRoute(ctx, topology.VPCRouter("vpc-1"), "0.0.0.0/0") + require.NoError(t, err) + assert.Nil(t, route) +} + +func TestDetachIGW_IdempotentOnAbsent(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + mgr, _ := newTestIGWManager(t, m, policy.NATModeDistributed, nil, LinkLocalAllocator{}, nil) + + err := mgr.DetachIGW(ctx, "vpc-1") + require.Error(t, err) // external switch absent +} + +type failingAllocator struct{} + +var _ GatewayIPAllocator = failingAllocator{} + +func (failingAllocator) Allocate(_ context.Context, _ string, _ *ExternalPoolConfig) (string, int, bool, error) { + return "", 0, false, errors.New("boom") +} + +func (failingAllocator) Release(_ context.Context, _ string) error { return nil } + +func TestAttachIGW_AllocatorFailureUnwindsExtSwitch(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedVPCRouter(t, m, "vpc-1", "10.0.0.0/16") + pool := &ExternalPoolConfig{Name: "p", Gateway: "192.168.1.1", PrefixLen: 24} + mgr, _ := newTestIGWManager(t, m, policy.NATModeCentralized, pool, failingAllocator{}, nil) + + require.Error(t, mgr.AttachIGW(ctx, IGWSpec{VPCID: "vpc-1", InternetGatewayID: "igw-1"})) + + _, err := m.GetLogicalSwitch(ctx, topology.ExternalSwitch("vpc-1")) + assert.Error(t, err, "external switch must be unwound on allocator failure") +} diff --git a/spinifex/network/external/natgw.go b/spinifex/network/external/natgw.go new file mode 100644 index 000000000..edeb398ed --- /dev/null +++ b/spinifex/network/external/natgw.go @@ -0,0 +1,41 @@ +package external + +import ( + "context" + "errors" + "fmt" + + "github.com/mulgadc/spinifex/spinifex/network/policy" +) + +// NATGWManager attaches/detaches NAT Gateway SNAT rules. Thin L5 wrapper +// over policy.NATManager. OVN type is snat (vs dnat_and_snat for EIP). +type NATGWManager interface { + AttachNATGateway(ctx context.Context, gw policy.NATGWSpec) error + DetachNATGateway(ctx context.Context, vpcID, subnetCIDR string) error +} + +type natGWManager struct { + nat policy.NATManager +} + +var _ NATGWManager = (*natGWManager)(nil) + +// NewNATGWManager constructs a NATGWManager. +func NewNATGWManager(nat policy.NATManager) (NATGWManager, error) { + if nat == nil { + return nil, errors.New("NATGWManager: NATManager required") + } + return &natGWManager{nat: nat}, nil +} + +func (m *natGWManager) AttachNATGateway(ctx context.Context, gw policy.NATGWSpec) error { + if gw.VPCID == "" || gw.PublicIP == "" || gw.SubnetCIDR == "" { + return fmt.Errorf("AttachNATGateway: VPCID, PublicIP, SubnetCIDR all required") + } + return m.nat.AddNATGateway(ctx, gw) +} + +func (m *natGWManager) DetachNATGateway(ctx context.Context, vpcID, subnetCIDR string) error { + return m.nat.DeleteNATGateway(ctx, vpcID, subnetCIDR) +} diff --git a/spinifex/network/external/natgw_test.go b/spinifex/network/external/natgw_test.go new file mode 100644 index 000000000..2f9582c2e --- /dev/null +++ b/spinifex/network/external/natgw_test.go @@ -0,0 +1,51 @@ +package external + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mulgadc/spinifex/spinifex/network/ovn/mock" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +func TestNewNATGWManager_RejectsNilNAT(t *testing.T) { + _, err := NewNATGWManager(nil) + require.Error(t, err) +} + +func TestNATGWManager_AttachAndDetach(t *testing.T) { + ctx := context.Background() + m := mock.New() + require.NoError(t, m.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: topology.VPCRouter("vpc-1")})) + nm, _ := policy.NewNATManager(m, policy.NATModeCentralized) + mgr, err := NewNATGWManager(nm) + require.NoError(t, err) + + require.NoError(t, mgr.AttachNATGateway(ctx, policy.NATGWSpec{ + VPCID: "vpc-1", NATGatewayID: "nat-xyz", + PublicIP: "9.9.9.9", SubnetCIDR: "10.0.1.0/24", + })) + + var found *nbdb.NAT + for _, n := range m.NATs { + if n.Type == "snat" && n.LogicalIP == "10.0.1.0/24" { + found = n + } + } + require.NotNil(t, found) + assert.Equal(t, "9.9.9.9", found.ExternalIP) + + require.NoError(t, mgr.DetachNATGateway(ctx, "vpc-1", "10.0.1.0/24")) + require.NoError(t, mgr.DetachNATGateway(ctx, "vpc-1", "10.0.1.0/24")) // idempotent +} + +func TestNATGWManager_AttachRejectsEmptyFields(t *testing.T) { + nm, _ := policy.NewNATManager(mock.New(), policy.NATModeCentralized) + mgr, _ := NewNATGWManager(nm) + require.Error(t, mgr.AttachNATGateway(context.Background(), policy.NATGWSpec{})) +} diff --git a/spinifex/network/external/pool.go b/spinifex/network/external/pool.go new file mode 100644 index 000000000..a81d1cfd1 --- /dev/null +++ b/spinifex/network/external/pool.go @@ -0,0 +1,199 @@ +package external + +import ( + "context" + "fmt" + "log/slog" + "net" + + "github.com/mulgadc/spinifex/spinifex/network/ovn" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +// linkLocalGatewayNetwork is the link-local /30 the IGW gateway LRP carries +// in distributed-NAT mode. LRP IP never goes on the wire — per-VM dnat_and_snat +// with external_mac/logical_port handles upstream ARP per chassis. +// Centralised NAT requires a WAN-subnet IP from gw_lrp_range instead. +const linkLocalGatewayNetwork = "169.254.0.1/30" + +const linkLocalGatewayNexthop = "169.254.0.2" + +// gatewayIPExtIDKey is the LRP external_ids key for the allocated gateway IP. +const gatewayIPExtIDKey = "spinifex:gateway_ip" + +// FindPool returns the first matching pool: AZ-scoped → region-scoped → unscoped. +func FindPool(pools []ExternalPoolConfig, region, az string) *ExternalPoolConfig { + for i := range pools { + p := &pools[i] + if p.AZ != "" && p.AZ == az && p.Region == region { + return p + } + } + for i := range pools { + p := &pools[i] + if p.AZ == "" && p.Region != "" && p.Region == region { + return p + } + } + for i := range pools { + p := &pools[i] + if p.Region == "" && p.AZ == "" { + return p + } + } + return nil +} + +// GatewayIPAllocator resolves the gateway LRP IP for a VPC. +// ok=false means caller falls back to link-local; error means abort. +type GatewayIPAllocator interface { + Allocate(ctx context.Context, vpcID string, pool *ExternalPoolConfig) (ip string, prefixLen int, ok bool, err error) + Release(ctx context.Context, vpcID string) error +} + +// LinkLocalAllocator always returns ok=false (distributed-NAT, link-local LRP). +type LinkLocalAllocator struct{} + +var _ GatewayIPAllocator = LinkLocalAllocator{} + +func (LinkLocalAllocator) Allocate(_ context.Context, _ string, _ *ExternalPoolConfig) (string, int, bool, error) { + return "", 0, false, nil +} + +func (LinkLocalAllocator) Release(_ context.Context, _ string) error { return nil } + +// StaticRangeAllocator picks the next free IP in pool.gw_lrp_range. +// Read-only: persistence is via IGWManager's LRP external_ids. +type StaticRangeAllocator struct { + OVN ovn.Client +} + +var _ GatewayIPAllocator = (*StaticRangeAllocator)(nil) + +func NewStaticRangeAllocator(client ovn.Client) *StaticRangeAllocator { + return &StaticRangeAllocator{OVN: client} +} + +// Allocate returns the next free gateway LRP IP for vpcID. Idempotent: if the +// LRP already exists with a recorded allocation, returns the existing IP. +func (a *StaticRangeAllocator) Allocate(ctx context.Context, vpcID string, pool *ExternalPoolConfig) (string, int, bool, error) { + start, end, prefix, ok := gwLrpRange(pool) + if !ok { + return "", 0, false, nil + } + + gwPortName := topology.GatewayRouterPort(vpcID) + lrps, err := a.OVN.ListLogicalRouterPorts(ctx) + if err != nil { + return "", 0, false, fmt.Errorf("list LRPs for gw IP allocation: %w", err) + } + used := make(map[uint32]struct{}, len(lrps)) + for _, lrp := range lrps { + existing := lrp.ExternalIDs[gatewayIPExtIDKey] + if existing == "" { + continue + } + if lrp.Name == gwPortName { + return existing, prefix, true, nil + } + if v := net.ParseIP(existing).To4(); v != nil { + used[ipv4ToUint32(v)] = struct{}{} + } + } + + startU := ipv4ToUint32(start) + endU := ipv4ToUint32(end) + for n := startU; n <= endU; n++ { + if _, taken := used[n]; taken { + continue + } + return uint32ToIPv4(n).String(), prefix, true, nil + } + return "", 0, false, fmt.Errorf("gw_lrp_range exhausted for pool %q (%s-%s)", pool.Name, pool.GwLrpRangeStart, pool.GwLrpRangeEnd) +} + +// Release is a no-op — LRP external_id is source of truth, cleared on detach. +func (a *StaticRangeAllocator) Release(_ context.Context, _ string) error { return nil } + +// gwLrpRange returns the gateway LRP IP range: explicit pool.GwLrpRange* if set, +// else last 16 host IPs of the WAN subnet (shifted below RangeStart on overlap). +func gwLrpRange(pool *ExternalPoolConfig) (start, end net.IP, prefix int, ok bool) { + if pool == nil { + return nil, nil, 0, false + } + prefix = pool.PrefixLen + if prefix <= 0 || prefix > 32 { + prefix = 24 + } + + if pool.GwLrpRangeStart != "" || pool.GwLrpRangeEnd != "" { + s := net.ParseIP(pool.GwLrpRangeStart).To4() + e := net.ParseIP(pool.GwLrpRangeEnd).To4() + if s != nil && e != nil && ipv4ToUint32(s) <= ipv4ToUint32(e) { + return s, e, prefix, true + } + slog.Warn("external: invalid explicit gw_lrp_range, attempting auto-derive", + "pool", pool.Name, "start", pool.GwLrpRangeStart, "end", pool.GwLrpRangeEnd) + } + + gw := net.ParseIP(pool.Gateway).To4() + if gw == nil { + return nil, nil, 0, false + } + mask := net.CIDRMask(prefix, 32) + network := gw.Mask(mask) + bcast := make(net.IP, 4) + for i := range 4 { + bcast[i] = network[i] | ^mask[i] + } + bcastU := ipv4ToUint32(bcast) + if bcastU < 17 { + return nil, nil, 0, false + } + autoEndU := bcastU - 1 + autoStartU := bcastU - 16 + + if pool.RangeStart != "" && pool.RangeEnd != "" { + rs := net.ParseIP(pool.RangeStart).To4() + re := net.ParseIP(pool.RangeEnd).To4() + if rs != nil && re != nil { + rsU := ipv4ToUint32(rs) + reU := ipv4ToUint32(re) + if autoEndU >= rsU && autoStartU <= reU { + if rsU < 17 { + return nil, nil, 0, false + } + autoEndU = rsU - 1 + autoStartU = rsU - 16 + } + } + } + + netU := ipv4ToUint32(network) + if autoStartU <= netU { + autoStartU = netU + 1 + } + if autoEndU >= bcastU { + autoEndU = bcastU - 1 + } + if autoStartU > autoEndU { + return nil, nil, 0, false + } + + gwU := ipv4ToUint32(gw) + if gwU >= autoStartU && gwU <= autoEndU { + switch gwU { + case autoStartU: + autoStartU++ + case autoEndU: + autoEndU-- + default: + autoStartU = gwU + 1 + } + if autoStartU > autoEndU { + return nil, nil, 0, false + } + } + + return uint32ToIPv4(autoStartU), uint32ToIPv4(autoEndU), prefix, true +} diff --git a/spinifex/network/external/pool_test.go b/spinifex/network/external/pool_test.go new file mode 100644 index 000000000..918035b23 --- /dev/null +++ b/spinifex/network/external/pool_test.go @@ -0,0 +1,182 @@ +package external + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mulgadc/spinifex/spinifex/network/ovn/mock" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +func TestFindPool_AZBeatsRegionBeatsUnscoped(t *testing.T) { + pools := []ExternalPoolConfig{ + {Name: "default"}, + {Name: "region-us", Region: "us"}, + {Name: "az-us-1a", Region: "us", AZ: "us-1a"}, + } + + assert.Equal(t, "az-us-1a", FindPool(pools, "us", "us-1a").Name) + assert.Equal(t, "region-us", FindPool(pools, "us", "us-1b").Name) + assert.Equal(t, "default", FindPool(pools, "eu", "eu-1a").Name) +} + +func TestFindPool_NoMatchReturnsNil(t *testing.T) { + pools := []ExternalPoolConfig{{Name: "region-us", Region: "us"}} + assert.Nil(t, FindPool(pools, "eu", "eu-1a")) +} + +func TestLinkLocalAllocator_AlwaysReturnsNotOK(t *testing.T) { + a := LinkLocalAllocator{} + ip, prefix, ok, err := a.Allocate(context.Background(), "vpc-1", &ExternalPoolConfig{Gateway: "192.168.1.1", PrefixLen: 24}) + require.NoError(t, err) + assert.False(t, ok) + assert.Empty(t, ip) + assert.Zero(t, prefix) + require.NoError(t, a.Release(context.Background(), "vpc-1")) +} + +func TestStaticRangeAllocator_ExplicitRange_AllocatesFirstFree(t *testing.T) { + ctx := context.Background() + m := mock.New() + a := NewStaticRangeAllocator(m) + + pool := &ExternalPoolConfig{ + Name: "p", + Gateway: "192.168.1.1", + PrefixLen: 24, + GwLrpRangeStart: "192.168.1.240", + GwLrpRangeEnd: "192.168.1.243", + } + + ip, prefix, ok, err := a.Allocate(ctx, "vpc-1", pool) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, "192.168.1.240", ip) + assert.Equal(t, 24, prefix) +} + +func TestStaticRangeAllocator_SkipsUsedIPs(t *testing.T) { + ctx := context.Background() + m := mock.New() + // Seed an existing LRP on a sibling VPC already holding .240. + require.NoError(t, m.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: topology.VPCRouter("vpc-other")})) + require.NoError(t, m.CreateLogicalRouterPort(ctx, topology.VPCRouter("vpc-other"), &nbdb.LogicalRouterPort{ + Name: topology.GatewayRouterPort("vpc-other"), + ExternalIDs: map[string]string{gatewayIPExtIDKey: "192.168.1.240"}, + })) + a := NewStaticRangeAllocator(m) + + pool := &ExternalPoolConfig{ + Gateway: "192.168.1.1", PrefixLen: 24, + GwLrpRangeStart: "192.168.1.240", GwLrpRangeEnd: "192.168.1.243", + } + + ip, _, ok, err := a.Allocate(ctx, "vpc-new", pool) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, "192.168.1.241", ip) +} + +func TestStaticRangeAllocator_ReturnsExistingForSameVPC(t *testing.T) { + ctx := context.Background() + m := mock.New() + require.NoError(t, m.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: topology.VPCRouter("vpc-1")})) + require.NoError(t, m.CreateLogicalRouterPort(ctx, topology.VPCRouter("vpc-1"), &nbdb.LogicalRouterPort{ + Name: topology.GatewayRouterPort("vpc-1"), + ExternalIDs: map[string]string{gatewayIPExtIDKey: "192.168.1.242"}, + })) + a := NewStaticRangeAllocator(m) + + pool := &ExternalPoolConfig{ + Gateway: "192.168.1.1", PrefixLen: 24, + GwLrpRangeStart: "192.168.1.240", GwLrpRangeEnd: "192.168.1.243", + } + + ip, _, ok, err := a.Allocate(ctx, "vpc-1", pool) + require.NoError(t, err) + require.True(t, ok) + assert.Equal(t, "192.168.1.242", ip) +} + +func TestStaticRangeAllocator_RangeExhaustedReturnsError(t *testing.T) { + ctx := context.Background() + m := mock.New() + require.NoError(t, m.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: topology.VPCRouter("vpc-other")})) + require.NoError(t, m.CreateLogicalRouterPort(ctx, topology.VPCRouter("vpc-other"), &nbdb.LogicalRouterPort{ + Name: topology.GatewayRouterPort("vpc-other"), + ExternalIDs: map[string]string{gatewayIPExtIDKey: "192.168.1.240"}, + })) + a := NewStaticRangeAllocator(m) + pool := &ExternalPoolConfig{ + Gateway: "192.168.1.1", PrefixLen: 24, + GwLrpRangeStart: "192.168.1.240", GwLrpRangeEnd: "192.168.1.240", + } + + _, _, ok, err := a.Allocate(ctx, "vpc-new", pool) + require.Error(t, err) + assert.False(t, ok) +} + +func TestStaticRangeAllocator_NoRangeReturnsNotOK(t *testing.T) { + a := NewStaticRangeAllocator(mock.New()) + _, _, ok, err := a.Allocate(context.Background(), "vpc-1", &ExternalPoolConfig{}) + require.NoError(t, err) + assert.False(t, ok) +} + +func TestGwLrpRange_AutoDeriveTopOfSubnet(t *testing.T) { + pool := &ExternalPoolConfig{Gateway: "192.168.1.1", PrefixLen: 24} + start, end, prefix, ok := gwLrpRange(pool) + require.True(t, ok) + assert.Equal(t, "192.168.1.239", start.String()) + assert.Equal(t, "192.168.1.254", end.String()) + assert.Equal(t, 24, prefix) +} + +func TestGwLrpRange_ShiftsBelowEIPRangeOnOverlap(t *testing.T) { + pool := &ExternalPoolConfig{ + Gateway: "192.168.1.1", + PrefixLen: 24, + RangeStart: "192.168.1.240", + RangeEnd: "192.168.1.250", + } + start, end, _, ok := gwLrpRange(pool) + require.True(t, ok) + assert.Equal(t, "192.168.1.224", start.String()) + assert.Equal(t, "192.168.1.239", end.String()) +} + +func TestGwLrpRange_ExplicitTakesPriority(t *testing.T) { + pool := &ExternalPoolConfig{ + Gateway: "192.168.1.1", PrefixLen: 24, + GwLrpRangeStart: "192.168.1.10", + GwLrpRangeEnd: "192.168.1.20", + } + start, end, _, ok := gwLrpRange(pool) + require.True(t, ok) + assert.Equal(t, "192.168.1.10", start.String()) + assert.Equal(t, "192.168.1.20", end.String()) +} + +func TestGwLrpRange_NilPoolReturnsNotOK(t *testing.T) { + _, _, _, ok := gwLrpRange(nil) + assert.False(t, ok) +} + +func TestGwLrpRange_GatewayInsideAutoRangeIsExcluded(t *testing.T) { + // Tiny /29 forces the auto-range to overlap the gateway IP. + pool := &ExternalPoolConfig{Gateway: "192.168.1.1", PrefixLen: 29} + start, end, _, ok := gwLrpRange(pool) + require.True(t, ok) + startU := ipv4ToUint32(start) + endU := ipv4ToUint32(end) + gwU := ipv4ToUint32(net.ParseIP(pool.Gateway).To4()) + for n := startU; n <= endU; n++ { + assert.NotEqual(t, gwU, n, "gateway IP must not appear in auto-range") + } +} diff --git a/spinifex/network/external/types.go b/spinifex/network/external/types.go new file mode 100644 index 000000000..5602102bd --- /dev/null +++ b/spinifex/network/external/types.go @@ -0,0 +1,24 @@ +package external + +// ExternalPoolConfig describes one external IP pool wired to a VPC. +// Mirrors [external_pools] in spinifex.toml. +type ExternalPoolConfig struct { + Name string + RangeStart string + RangeEnd string + Gateway string + GatewayIP string + PrefixLen int + DNSServers []string + Region string + AZ string + GwLrpRangeStart string + GwLrpRangeEnd string +} + +// IGWSpec is the L5 input for IGWManager.AttachIGW / DetachIGW. +// InternetGatewayID is propagated into OVN external_ids for reconcile correlation. +type IGWSpec struct { + VPCID string + InternetGatewayID string +} diff --git a/spinifex/network/host/bridge_ip.go b/spinifex/network/host/bridge_ip.go new file mode 100644 index 000000000..51ec826e1 --- /dev/null +++ b/spinifex/network/host/bridge_ip.go @@ -0,0 +1,33 @@ +package host + +import ( + "fmt" + "net" + "strings" +) + +// GetBridgeIPv4 returns the first IPv4 address on the named Linux bridge. +// Returns "", nil if the bridge does not exist. +func GetBridgeIPv4(bridgeName string) (string, error) { + iface, err := net.InterfaceByName(bridgeName) + if err != nil { + if strings.Contains(err.Error(), "no such network interface") { + return "", nil + } + return "", fmt.Errorf("lookup %s: %w", bridgeName, err) + } + addrs, err := iface.Addrs() + if err != nil { + return "", fmt.Errorf("list addrs on %s: %w", bridgeName, err) + } + for _, addr := range addrs { + ipNet, ok := addr.(*net.IPNet) + if !ok { + continue + } + if ipNet.IP.To4() != nil { + return ipNet.IP.String(), nil + } + } + return "", fmt.Errorf("no IPv4 address on %s", bridgeName) +} diff --git a/spinifex/network/host/bridge_ip_test.go b/spinifex/network/host/bridge_ip_test.go new file mode 100644 index 000000000..584c739ea --- /dev/null +++ b/spinifex/network/host/bridge_ip_test.go @@ -0,0 +1,24 @@ +package host + +import "testing" + +func TestGetBridgeIPv4_Loopback(t *testing.T) { + // "lo" is always present and has 127.0.0.1 + ip, err := GetBridgeIPv4("lo") + if err != nil { + t.Fatalf("GetBridgeIPv4(lo): %v", err) + } + if ip != "127.0.0.1" { + t.Errorf("GetBridgeIPv4(lo) = %q, want 127.0.0.1", ip) + } +} + +func TestGetBridgeIPv4_NonexistentBridge(t *testing.T) { + ip, err := GetBridgeIPv4("br-nonexistent-test-xyz") + if err != nil { + t.Fatalf("expected nil error for absent bridge, got: %v", err) + } + if ip != "" { + t.Errorf("expected empty IP for absent bridge, got %q", ip) + } +} diff --git a/spinifex/network/host/bridges.go b/spinifex/network/host/bridges.go new file mode 100644 index 000000000..9b11c7cd2 --- /dev/null +++ b/spinifex/network/host/bridges.go @@ -0,0 +1,34 @@ +package host + +import ( + "context" + "fmt" +) + +// ensureBridges idempotently creates br-int and uplinkBridge with OVN settings. +// br-int needs fail-mode=secure + disable-in-band (canonical OVN integration bridge). +// Shared by Physical and Veth — only uplink port wiring differs. +func ensureBridges(ctx context.Context, r Runner, uplinkBridge string) error { + if uplinkBridge == "" { + return fmt.Errorf("host: ensureBridges UplinkBridge required") + } + if _, err := r.Run(ctx, "ovs-vsctl", "--may-exist", "add-br", "br-int"); err != nil { + return fmt.Errorf("create br-int: %w", err) + } + if _, err := r.Run(ctx, "ovs-vsctl", "set", "Bridge", "br-int", "fail-mode=secure"); err != nil { + return fmt.Errorf("set br-int fail-mode: %w", err) + } + if _, err := r.Run(ctx, "ovs-vsctl", "set", "Bridge", "br-int", "other-config:disable-in-band=true"); err != nil { + return fmt.Errorf("set br-int disable-in-band: %w", err) + } + if _, err := r.Run(ctx, "ip", "link", "set", "br-int", "up"); err != nil { + return fmt.Errorf("bring up br-int: %w", err) + } + if _, err := r.Run(ctx, "ovs-vsctl", "--may-exist", "add-br", uplinkBridge); err != nil { + return fmt.Errorf("create %s: %w", uplinkBridge, err) + } + if _, err := r.Run(ctx, "ip", "link", "set", uplinkBridge, "up"); err != nil { + return fmt.Errorf("bring up %s: %w", uplinkBridge, err) + } + return nil +} diff --git a/spinifex/network/host/health.go b/spinifex/network/host/health.go new file mode 100644 index 000000000..31488395c --- /dev/null +++ b/spinifex/network/host/health.go @@ -0,0 +1,48 @@ +package host + +import ( + "os" + "strings" + + "github.com/mulgadc/spinifex/spinifex/utils" +) + +// OVNHealth reports the readiness of OVN networking on this compute node. +type OVNHealth struct { + BrIntExists bool `json:"br_int_exists"` + OVNControllerUp bool `json:"ovn_controller_up"` + ChassisID string `json:"chassis_id,omitempty"` + EncapIP string `json:"encap_ip,omitempty"` + OVNRemote string `json:"ovn_remote,omitempty"` +} + +// HealthStatus probes local OVS/OVN state to determine network readiness. +func HealthStatus() OVNHealth { + status := OVNHealth{} + + if err := utils.SudoCommand("ovs-vsctl", "br-exists", "br-int").Run(); err == nil { + status.BrIntExists = true + } + + // Trixie OVN moved the pid file to /var/run/ovn/; older installs use + // /var/run/openvswitch/. Probe at runtime so the same binary works on both. + ovnTarget := "ovn-controller" + if _, err := os.Stat("/var/run/ovn/ovn-controller.pid"); err == nil { + ovnTarget = "/var/run/ovn/ovn-controller" + } + if out, err := utils.SudoCommand("ovs-appctl", "-t", ovnTarget, "version").CombinedOutput(); err == nil && len(out) > 0 { + status.OVNControllerUp = true + } + + if out, err := utils.SudoCommand("ovs-vsctl", "get", "Open_vSwitch", ".", "external_ids:system-id").CombinedOutput(); err == nil { + status.ChassisID = strings.Trim(strings.TrimSpace(string(out)), "\"") + } + if out, err := utils.SudoCommand("ovs-vsctl", "get", "Open_vSwitch", ".", "external_ids:ovn-encap-ip").CombinedOutput(); err == nil { + status.EncapIP = strings.Trim(strings.TrimSpace(string(out)), "\"") + } + if out, err := utils.SudoCommand("ovs-vsctl", "get", "Open_vSwitch", ".", "external_ids:ovn-remote").CombinedOutput(); err == nil { + status.OVNRemote = strings.Trim(strings.TrimSpace(string(out)), "\"") + } + + return status +} diff --git a/spinifex/network/host/host.go b/spinifex/network/host/host.go new file mode 100644 index 000000000..def2527db --- /dev/null +++ b/spinifex/network/host/host.go @@ -0,0 +1,77 @@ +// Package host is L0 of the spinifex network stack: kernel/OVS host wiring. +// Owns OVS bridges (br-int, br-ext), the uplink port attaching br-ext to WAN, +// and the external CIDR. Bridge mode (physical vs veth) is resolved here and +// invisible above L0. See ADR-0006 (S2, S3). +package host + +import ( + "context" + "fmt" + "net" + "net/netip" +) + +// UplinkMode names the physical wiring strategy attaching br-ext to the WAN. +type UplinkMode int + +const ( + UplinkModeUnknown UplinkMode = iota + + // UplinkModePhysical enslaves the WAN NIC directly to br-ext (distributed NAT). + UplinkModePhysical + + // UplinkModeVeth bridges a Linux bridge to br-ext via a veth pair (centralised NAT). + UplinkModeVeth +) + +func (m UplinkMode) String() string { + switch m { + case UplinkModePhysical: + return "physical" + case UplinkModeVeth: + return "veth" + default: + return "unknown" + } +} + +// Wiring owns the host's L0 network state. Methods are idempotent. +type Wiring interface { + // EnsureBridges ensures br-int and br-ext exist with the OVS settings + // ovn-controller requires (br-int: fail-mode=secure, disable-in-band). + EnsureBridges(ctx context.Context) error + + // EnsureUplinkPort attaches the uplink port to br-ext and returns its MAC. + // L2 programs the gateway LRP with this MAC so the LRP shares L2 identity + // with the uplink (upstream switch MAC learning). + EnsureUplinkPort(ctx context.Context) (net.HardwareAddr, error) + + // UplinkMode returns the configured mode. L3 reads once at startup + // (ADR-0006 S3); never re-reads at runtime. + UplinkMode() UplinkMode + + // ExternalCIDR returns the IPv4 prefix on this node's uplink bridge. + // Only valid after EnsureUplinkPort succeeds (absorbs boot-race vs DHCP). + ExternalCIDR(ctx context.Context) (netip.Prefix, error) +} + +// Runner executes a privileged host command and returns combined stdout/stderr. +// Stubbable for tests; production impl is execRunner in run.go. +type Runner interface { + Run(ctx context.Context, name string, args ...string) ([]byte, error) +} + +// InterfaceReader observes kernel network state via net.InterfaceByName / /sys/class/net. +type InterfaceReader interface { + // BridgeCIDR returns the first IPv4 prefix on the named link, or error if none. + BridgeCIDR(name string) (netip.Prefix, error) + + // LinkMAC returns the hardware address of the named link. + LinkMAC(name string) (net.HardwareAddr, error) + + // LinkMaster returns the master link name, or "" when none. + LinkMaster(name string) (string, error) +} + +// ErrNoUplinkAddr is returned by ExternalCIDR when the WAN bridge has no IPv4 yet. +var ErrNoUplinkAddr = fmt.Errorf("uplink bridge has no IPv4 address") diff --git a/spinifex/network/host/host_test.go b/spinifex/network/host/host_test.go new file mode 100644 index 000000000..ee6def29c --- /dev/null +++ b/spinifex/network/host/host_test.go @@ -0,0 +1,289 @@ +package host + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "slices" + "strings" + "sync" + "testing" +) + +// stubRunner records commands and returns scripted output by prefix. +type stubRunner struct { + mu sync.Mutex + calls []string + resp map[string]stubResp +} + +type stubResp struct { + out []byte + err error +} + +func newStubRunner() *stubRunner { return &stubRunner{resp: map[string]stubResp{}} } + +func (s *stubRunner) expect(prefix string, out []byte, err error) { + s.resp[prefix] = stubResp{out: out, err: err} +} + +func (s *stubRunner) Run(_ context.Context, name string, args ...string) ([]byte, error) { + s.mu.Lock() + defer s.mu.Unlock() + joined := strings.Join(append([]string{name}, args...), " ") + s.calls = append(s.calls, joined) + for prefix, r := range s.resp { + if strings.HasPrefix(joined, prefix) { + return r.out, r.err + } + } + return nil, fmt.Errorf("stubRunner: unexpected command %q", joined) +} + +func (s *stubRunner) called(prefix string) bool { + s.mu.Lock() + defer s.mu.Unlock() + for _, c := range s.calls { + if strings.HasPrefix(c, prefix) { + return true + } + } + return false +} + +type stubReader struct { + cidrs map[string]netip.Prefix + macs map[string]net.HardwareAddr + master map[string]string + errs map[string]error +} + +func newStubReader() *stubReader { + return &stubReader{ + cidrs: map[string]netip.Prefix{}, + macs: map[string]net.HardwareAddr{}, + master: map[string]string{}, + errs: map[string]error{}, + } +} + +func (r *stubReader) BridgeCIDR(name string) (netip.Prefix, error) { + if err, ok := r.errs["cidr:"+name]; ok { + return netip.Prefix{}, err + } + if p, ok := r.cidrs[name]; ok { + return p, nil + } + return netip.Prefix{}, fmt.Errorf("%w: %q", ErrNoUplinkAddr, name) +} + +func (r *stubReader) LinkMAC(name string) (net.HardwareAddr, error) { + if err, ok := r.errs["mac:"+name]; ok { + return nil, err + } + if m, ok := r.macs[name]; ok { + return m, nil + } + return nil, fmt.Errorf("no MAC for %q", name) +} + +func (r *stubReader) LinkMaster(name string) (string, error) { + if err, ok := r.errs["master:"+name]; ok { + return "", err + } + return r.master[name], nil +} + +func mustMAC(t *testing.T, s string) net.HardwareAddr { + t.Helper() + mac, err := net.ParseMAC(s) + if err != nil { + t.Fatalf("parse MAC %q: %v", s, err) + } + return mac +} + +func TestUplinkModeString(t *testing.T) { + cases := []struct { + m UplinkMode + want string + }{ + {UplinkModePhysical, "physical"}, + {UplinkModeVeth, "veth"}, + {UplinkModeUnknown, "unknown"}, + {UplinkMode(99), "unknown"}, + } + for _, c := range cases { + if got := c.m.String(); got != c.want { + t.Errorf("UplinkMode(%d).String() = %q, want %q", c.m, got, c.want) + } + } +} + +func TestPhysical_EnsureUplinkPort(t *testing.T) { + r := newStubRunner() + r.expect("ovs-vsctl port-to-br enp0s3", []byte("br-ext\n"), nil) + rd := newStubReader() + rd.macs["enp0s3"] = mustMAC(t, "aa:bb:cc:dd:ee:ff") + + p := &Physical{ + ExternalInterface: "enp0s3", + UplinkBridge: "br-ext", + Runner: r, + Reader: rd, + } + mac, err := p.EnsureUplinkPort(context.Background()) + if err != nil { + t.Fatalf("EnsureUplinkPort: %v", err) + } + if mac.String() != "aa:bb:cc:dd:ee:ff" { + t.Errorf("got MAC %q, want aa:bb:cc:dd:ee:ff", mac) + } + if p.UplinkMode() != UplinkModePhysical { + t.Errorf("UplinkMode = %v, want physical", p.UplinkMode()) + } +} + +func TestPhysical_EnsureUplinkPort_WrongBridge(t *testing.T) { + r := newStubRunner() + r.expect("ovs-vsctl port-to-br enp0s3", []byte("br-other\n"), nil) + p := &Physical{ + ExternalInterface: "enp0s3", + UplinkBridge: "br-ext", + Runner: r, + Reader: newStubReader(), + } + _, err := p.EnsureUplinkPort(context.Background()) + if err == nil || !strings.Contains(err.Error(), "br-other") { + t.Fatalf("expected bridge mismatch error, got: %v", err) + } +} + +func TestPhysical_ExternalCIDR(t *testing.T) { + rd := newStubReader() + want := netip.MustParsePrefix("192.0.2.5/24") + rd.cidrs["br-ext"] = want + p := &Physical{UplinkBridge: "br-ext", Reader: rd} + got, err := p.ExternalCIDR(context.Background()) + if err != nil { + t.Fatalf("ExternalCIDR: %v", err) + } + if got != want { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestPhysical_ExternalCIDR_NoAddr(t *testing.T) { + p := &Physical{UplinkBridge: "br-ext", Reader: newStubReader()} + _, err := p.ExternalCIDR(context.Background()) + if !errors.Is(err, ErrNoUplinkAddr) { + t.Fatalf("expected ErrNoUplinkAddr, got: %v", err) + } +} + +func TestVeth_EnsureUplinkPort(t *testing.T) { + r := newStubRunner() + r.expect("ovs-vsctl port-to-br veth-wan-ovs", []byte("br-ext\n"), nil) + rd := newStubReader() + rd.macs[VethOVSEnd] = mustMAC(t, "02:11:22:33:44:55") + rd.master[VethLinuxEnd] = "br-wan" + + v := &Veth{ + LinuxBridge: "br-wan", + UplinkBridge: "br-ext", + Runner: r, + Reader: rd, + } + mac, err := v.EnsureUplinkPort(context.Background()) + if err != nil { + t.Fatalf("EnsureUplinkPort: %v", err) + } + if mac.String() != "02:11:22:33:44:55" { + t.Errorf("got MAC %q", mac) + } + if v.UplinkMode() != UplinkModeVeth { + t.Errorf("UplinkMode = %v, want veth", v.UplinkMode()) + } +} + +func TestVeth_EnsureUplinkPort_WrongMaster(t *testing.T) { + r := newStubRunner() + r.expect("ovs-vsctl port-to-br veth-wan-ovs", []byte("br-ext\n"), nil) + rd := newStubReader() + rd.macs[VethOVSEnd] = mustMAC(t, "02:11:22:33:44:55") + rd.master[VethLinuxEnd] = "br-other" + + v := &Veth{LinuxBridge: "br-wan", UplinkBridge: "br-ext", Runner: r, Reader: rd} + _, err := v.EnsureUplinkPort(context.Background()) + if err == nil || !strings.Contains(err.Error(), "br-other") { + t.Fatalf("expected master mismatch error, got: %v", err) + } +} + +func TestVeth_ExternalCIDR(t *testing.T) { + rd := newStubReader() + want := netip.MustParsePrefix("10.0.0.42/24") + rd.cidrs["br-wan"] = want + v := &Veth{LinuxBridge: "br-wan", UplinkBridge: "br-ext", Reader: rd} + got, err := v.ExternalCIDR(context.Background()) + if err != nil { + t.Fatalf("ExternalCIDR: %v", err) + } + if got != want { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestEnsureBridges_RunsExpectedOps(t *testing.T) { + r := newStubRunner() + for _, prefix := range []string{ + "ovs-vsctl --may-exist add-br br-int", + "ovs-vsctl set Bridge br-int fail-mode=secure", + "ovs-vsctl set Bridge br-int other-config:disable-in-band=true", + "ip link set br-int up", + "ovs-vsctl --may-exist add-br br-ext", + "ip link set br-ext up", + } { + r.expect(prefix, nil, nil) + } + p := &Physical{ExternalInterface: "enp0s3", UplinkBridge: "br-ext", Runner: r, Reader: newStubReader()} + if err := p.EnsureBridges(context.Background()); err != nil { + t.Fatalf("EnsureBridges: %v", err) + } + for _, want := range []string{ + "ovs-vsctl --may-exist add-br br-int", + "ovs-vsctl set Bridge br-int fail-mode=secure", + "ovs-vsctl --may-exist add-br br-ext", + } { + if !r.called(want) { + t.Errorf("missing call %q; got %v", want, r.calls) + } + } +} + +func TestEnsureBridges_FailureSurfacesContext(t *testing.T) { + r := newStubRunner() + r.expect("ovs-vsctl --may-exist add-br br-int", []byte("permission denied"), errors.New("exit 1")) + p := &Physical{ExternalInterface: "enp0s3", UplinkBridge: "br-ext", Runner: r} + err := p.EnsureBridges(context.Background()) + if err == nil || !strings.Contains(err.Error(), "create br-int") { + t.Fatalf("expected create br-int wrap, got: %v", err) + } +} + +var ( + _ Wiring = (*Physical)(nil) + _ Wiring = (*Veth)(nil) + _ Runner = execRunner{} + _ InterfaceReader = kernelReader{} +) + +func TestUplinkModes_ConstantSet(t *testing.T) { + modes := []UplinkMode{UplinkModeUnknown, UplinkModePhysical, UplinkModeVeth} + if !slices.Contains(modes, UplinkModePhysical) || !slices.Contains(modes, UplinkModeVeth) { + t.Fatal("expected physical and veth in supported mode set") + } +} diff --git a/spinifex/network/host/ipsec.go b/spinifex/network/host/ipsec.go new file mode 100644 index 000000000..1fbf24e78 --- /dev/null +++ b/spinifex/network/host/ipsec.go @@ -0,0 +1,53 @@ +package host + +import ( + "fmt" + "strings" + + "github.com/mulgadc/spinifex/spinifex/utils" +) + +// SetIPSecCertPaths writes the local IPsec peer cert pointers into the OVS +// Open_vSwitch table. ovs-monitor-ipsec reads these to materialise strongSwan +// configs for every Geneve tunnel ovn-controller programs. +func SetIPSecCertPaths(certPath, keyPath, caCertPath string) error { + out, err := utils.SudoCommand("ovs-vsctl", "set", "Open_vSwitch", ".", + fmt.Sprintf("other_config:certificate=%s", certPath), + fmt.Sprintf("other_config:private_key=%s", keyPath), + fmt.Sprintf("other_config:ca_cert=%s", caCertPath), + ).CombinedOutput() + if err != nil { + return fmt.Errorf("set IPsec cert pointers: %s: %w", strings.TrimSpace(string(out)), err) + } + return nil +} + +// EnableIPSecEncapsulation flips ipsec_encapsulation=true on the local +// Open_vSwitch row. Caller must first verify ovs-monitor-ipsec is active — +// flipping without a live daemon creates a silent-drop trap. +func EnableIPSecEncapsulation() error { + out, err := utils.SudoCommand("ovs-vsctl", "set", "Open_vSwitch", ".", + "other_config:ipsec_encapsulation=true", + ).CombinedOutput() + if err != nil { + return fmt.Errorf("enable ipsec_encapsulation: %s: %w", strings.TrimSpace(string(out)), err) + } + return nil +} + +// SetNBGlobalIPSec writes NB_Global.ipsec on the local OVN NB DB. Triggers +// ovn-controller on every chassis to add options:remote_name to Geneve tunnel +// ports, which is what ovs-monitor-ipsec keys off to materialise strongSwan +// connections. Only the management node has a local NB socket; callers gate +// the call on socket presence. +func SetNBGlobalIPSec(enable bool) error { + val := "false" + if enable { + val = "true" + } + out, err := utils.SudoCommand("ovn-nbctl", "set", "NB_Global", ".", "ipsec="+val).CombinedOutput() + if err != nil { + return fmt.Errorf("set NB_Global ipsec=%s: %s: %w", val, strings.TrimSpace(string(out)), err) + } + return nil +} diff --git a/spinifex/network/host/ipsec_enable.go b/spinifex/network/host/ipsec_enable.go new file mode 100644 index 000000000..2e2c525a2 --- /dev/null +++ b/spinifex/network/host/ipsec_enable.go @@ -0,0 +1,86 @@ +package host + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + "time" + + "github.com/mulgadc/spinifex/spinifex/admin" + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/utils" +) + +// systemctlActiveTimeout bounds the wait for openvswitch-ipsec.service to become active. +var systemctlActiveTimeout = 5 * time.Second + +// ovnNBSocketPath gates the NB_Global ipsec write to the management node. +var ovnNBSocketPath = "/run/ovn/ovnnb_db.sock" + +// EnableOVNIPSec wires the local IPsec peer cert and flips ipsec_encapsulation=true. +// Idempotent. Single-node clusters short-circuit (no Geneve tunnels to encrypt). +// Lives in L0 per ADR-0006 S8 (IPSec is OVN-native only; SA lifecycle invisible above L0). +func EnableOVNIPSec(configPath string, clusterConfig *config.ClusterConfig) error { + if configPath == "" { + return fmt.Errorf("config path unset") + } + if clusterConfig != nil && len(clusterConfig.Nodes) <= 1 { + slog.Info("ipsec: single-node cluster, skipping enable (no peers)") + return nil + } + configDir := filepath.Dir(configPath) + certPath, keyPath := admin.IPSecCertPaths(configDir) + caCertPath := filepath.Join(configDir, "ca.pem") + + for _, p := range []string{certPath, keyPath, caCertPath} { + if _, err := os.Stat(p); err != nil { + return fmt.Errorf("missing IPsec credential %s: %w", p, err) + } + } + + if err := ensureOVSMonitorIPSecActive(); err != nil { + return fmt.Errorf("ovs-monitor-ipsec: %w", err) + } + + if err := SetIPSecCertPaths(certPath, keyPath, caCertPath); err != nil { + return err + } + + if err := EnableIPSecEncapsulation(); err != nil { + return err + } + + // NB_Global.ipsec is cluster-wide; only the management node has a local NB socket. + // Without this flag, ovn-controller skips adding options:remote_name to Geneve + // tunnels and ovs-monitor-ipsec never materialises strongSwan connections. + if _, err := os.Stat(ovnNBSocketPath); err == nil { + if err := SetNBGlobalIPSec(true); err != nil { + return err + } + } + + slog.Info("OVN native IPsec enabled on intra-AZ Geneve", + "cert", certPath, + "key", keyPath, + "ca", caCertPath, + ) + return nil +} + +// ensureOVSMonitorIPSecActive polls openvswitch-ipsec.service for "active". +// If inactive, refuse to flip ipsec_encapsulation=true (would silently drop traffic). +func ensureOVSMonitorIPSecActive() error { + deadline := time.Now().Add(systemctlActiveTimeout) + var lastOut string + for time.Now().Before(deadline) { + out, _ := utils.SudoCommand("systemctl", "is-active", "openvswitch-ipsec.service").CombinedOutput() + lastOut = strings.TrimSpace(string(out)) + if lastOut == "active" { + return nil + } + time.Sleep(250 * time.Millisecond) + } + return fmt.Errorf("not active after %s: %s (provision via scripts/setup-ovn.sh)", systemctlActiveTimeout, lastOut) +} diff --git a/spinifex/network/host/ipsec_enable_test.go b/spinifex/network/host/ipsec_enable_test.go new file mode 100644 index 000000000..80bb850d7 --- /dev/null +++ b/spinifex/network/host/ipsec_enable_test.go @@ -0,0 +1,169 @@ +package host + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/config" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func multiNodeClusterConfig() *config.ClusterConfig { + return &config.ClusterConfig{ + Node: "node1", + Nodes: map[string]config.Config{ + "node1": {}, + "node2": {}, + }, + } +} + +type recordingSudo struct { + runs [][]string + activeOutput string +} + +func (r *recordingSudo) stub(name string, args ...string) *exec.Cmd { + r.runs = append(r.runs, append([]string{name}, args...)) + if name == "systemctl" && len(args) >= 1 && args[0] == "is-active" { + out := r.activeOutput + if out == "" { + out = "active\n" + } + return exec.Command("printf", "%s", out) + } + return exec.Command("true") +} + +func TestEnableOVNIPSec(t *testing.T) { + recorder := &recordingSudo{} + t.Cleanup(utils.SetSudoCommandForTest(recorder.stub)) + + configDir := t.TempDir() + configPath := filepath.Join(configDir, "spinifex.toml") + require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600)) + + for _, rel := range []string{"ca.pem", "ipsec/peer.pem", "ipsec/peer.key"} { + full := filepath.Join(configDir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0750)) + require.NoError(t, os.WriteFile(full, []byte("x"), 0600)) + } + + // Worker path: no local NB socket → no ovn-nbctl call. + origNBSock := ovnNBSocketPath + ovnNBSocketPath = filepath.Join(configDir, "no-such-socket") + t.Cleanup(func() { ovnNBSocketPath = origNBSock }) + + require.NoError(t, EnableOVNIPSec(configPath, multiNodeClusterConfig())) + + require.Len(t, recorder.runs, 3) + assert.Equal(t, []string{"systemctl", "is-active", "openvswitch-ipsec.service"}, recorder.runs[0]) + for _, run := range recorder.runs[1:] { + assert.Equal(t, "ovs-vsctl", run[0]) + assert.Equal(t, "set", run[1]) + assert.Equal(t, "Open_vSwitch", run[2]) + } + joined := strings.Join(recorder.runs[1], " ") + assert.Contains(t, joined, "other_config:certificate="+filepath.Join(configDir, "ipsec", "peer.pem")) + assert.Contains(t, joined, "other_config:private_key="+filepath.Join(configDir, "ipsec", "peer.key")) + assert.Contains(t, joined, "other_config:ca_cert="+filepath.Join(configDir, "ca.pem")) + assert.Contains(t, strings.Join(recorder.runs[2], " "), "other_config:ipsec_encapsulation=true") +} + +func TestEnableOVNIPSec_Management(t *testing.T) { + recorder := &recordingSudo{} + t.Cleanup(utils.SetSudoCommandForTest(recorder.stub)) + + configDir := t.TempDir() + configPath := filepath.Join(configDir, "spinifex.toml") + require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600)) + for _, rel := range []string{"ca.pem", "ipsec/peer.pem", "ipsec/peer.key"} { + full := filepath.Join(configDir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0750)) + require.NoError(t, os.WriteFile(full, []byte("x"), 0600)) + } + + sockPath := filepath.Join(configDir, "ovnnb_db.sock") + require.NoError(t, os.WriteFile(sockPath, []byte{}, 0600)) + origNBSock := ovnNBSocketPath + ovnNBSocketPath = sockPath + t.Cleanup(func() { ovnNBSocketPath = origNBSock }) + + require.NoError(t, EnableOVNIPSec(configPath, multiNodeClusterConfig())) + + require.Len(t, recorder.runs, 4) + assert.Equal(t, []string{"ovn-nbctl", "set", "NB_Global", ".", "ipsec=true"}, recorder.runs[3]) +} + +func TestEnableOVNIPSec_SingleNodeSkip(t *testing.T) { + t.Cleanup(utils.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd { + t.Fatalf("utils.SudoCommand must not run on single-node short-circuit; got %s %v", name, args) + return exec.Command("true") + })) + + configDir := t.TempDir() + configPath := filepath.Join(configDir, "spinifex.toml") + require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600)) + + cfg := &config.ClusterConfig{ + Node: "node1", + Nodes: map[string]config.Config{"node1": {}}, + } + require.NoError(t, EnableOVNIPSec(configPath, cfg)) +} + +func TestEnableOVNIPSec_MonitorIPSecInactive(t *testing.T) { + recorder := &recordingSudo{activeOutput: "inactive\n"} + t.Cleanup(utils.SetSudoCommandForTest(recorder.stub)) + + origTimeout := systemctlActiveTimeout + systemctlActiveTimeout = 100 * time.Millisecond + t.Cleanup(func() { systemctlActiveTimeout = origTimeout }) + + configDir := t.TempDir() + configPath := filepath.Join(configDir, "spinifex.toml") + require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600)) + for _, rel := range []string{"ca.pem", "ipsec/peer.pem", "ipsec/peer.key"} { + full := filepath.Join(configDir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0750)) + require.NoError(t, os.WriteFile(full, []byte("x"), 0600)) + } + + err := EnableOVNIPSec(configPath, multiNodeClusterConfig()) + require.Error(t, err) + assert.Contains(t, err.Error(), "ovs-monitor-ipsec") + assert.Contains(t, err.Error(), "not active") + + // ovs-vsctl must NOT run — flip without live daemon is the silent-drop trap. + for _, run := range recorder.runs { + assert.NotEqual(t, "ovs-vsctl", run[0], "ovs-vsctl invoked despite dead daemon: %v", run) + } +} + +func TestEnableOVNIPSec_MissingCert(t *testing.T) { + t.Cleanup(utils.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd { + t.Fatalf("utils.SudoCommand must not run when cert files are absent") + return exec.Command("true") + })) + + configDir := t.TempDir() + configPath := filepath.Join(configDir, "spinifex.toml") + require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600)) + require.NoError(t, os.WriteFile(filepath.Join(configDir, "ca.pem"), []byte("x"), 0600)) + + err := EnableOVNIPSec(configPath, multiNodeClusterConfig()) + require.Error(t, err) + assert.Contains(t, err.Error(), "missing IPsec credential") +} + +func TestEnableOVNIPSec_NoConfigPath(t *testing.T) { + err := EnableOVNIPSec("", nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "config path unset") +} diff --git a/spinifex/network/host/physical.go b/spinifex/network/host/physical.go new file mode 100644 index 000000000..21cea3477 --- /dev/null +++ b/spinifex/network/host/physical.go @@ -0,0 +1,83 @@ +package host + +import ( + "context" + "fmt" + "log/slog" + "net" + "net/netip" + "strings" +) + +// Physical implements Wiring for the "WAN NIC enslaved to br-ext" model. +// Each chassis owns its uplink; NAT can be distributed. +type Physical struct { + // ExternalInterface is the WAN NIC (e.g. "enp0s3") on UplinkBridge. + ExternalInterface string + + // UplinkBridge holds the WAN NIC and IPv4 uplink (conventionally "br-ext"). + UplinkBridge string + + Runner Runner + Reader InterfaceReader +} + +var _ Wiring = (*Physical)(nil) + +func (p *Physical) runner() Runner { + if p.Runner != nil { + return p.Runner + } + return NewExecRunner() +} + +func (p *Physical) reader() InterfaceReader { + if p.Reader != nil { + return p.Reader + } + return NewKernelReader() +} + +// EnsureBridges creates br-int and br-ext idempotently with OVN settings. +func (p *Physical) EnsureBridges(ctx context.Context) error { + return ensureBridges(ctx, p.runner(), p.UplinkBridge) +} + +// EnsureUplinkPort verifies the WAN NIC is on UplinkBridge and returns its MAC. +// The gateway LRP shares this MAC so upstream switch CAM stays consistent. +func (p *Physical) EnsureUplinkPort(ctx context.Context) (net.HardwareAddr, error) { + if p.ExternalInterface == "" { + return nil, fmt.Errorf("host.Physical: ExternalInterface required") + } + if p.UplinkBridge == "" { + return nil, fmt.Errorf("host.Physical: UplinkBridge required") + } + + out, err := p.runner().Run(ctx, "ovs-vsctl", "port-to-br", p.ExternalInterface) + if err != nil { + return nil, fmt.Errorf("port-to-br %q: %w", p.ExternalInterface, err) + } + br := strings.TrimSpace(string(out)) + if br != p.UplinkBridge { + return nil, fmt.Errorf("host.Physical: %q on bridge %q, expected %q", p.ExternalInterface, br, p.UplinkBridge) + } + + mac, err := p.reader().LinkMAC(p.ExternalInterface) + if err != nil { + return nil, fmt.Errorf("read MAC for %q: %w", p.ExternalInterface, err) + } + slog.Info("host.Physical: uplink verified", "iface", p.ExternalInterface, "bridge", br, "mac", mac.String()) + return mac, nil +} + +// UplinkMode returns UplinkModePhysical (distributed NAT eligible). +func (p *Physical) UplinkMode() UplinkMode { return UplinkModePhysical } + +// ExternalCIDR returns the IPv4 prefix on UplinkBridge. +// ErrNoUplinkAddr means the OS hasn't assigned an IP yet — caller should retry. +func (p *Physical) ExternalCIDR(_ context.Context) (netip.Prefix, error) { + if p.UplinkBridge == "" { + return netip.Prefix{}, fmt.Errorf("host.Physical: UplinkBridge required") + } + return p.reader().BridgeCIDR(p.UplinkBridge) +} diff --git a/spinifex/network/host/plumber.go b/spinifex/network/host/plumber.go new file mode 100644 index 000000000..51e365696 --- /dev/null +++ b/spinifex/network/host/plumber.go @@ -0,0 +1,108 @@ +package host + +import ( + "fmt" + "log/slog" + "os" + "sort" + "strconv" + "strings" + + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/mulgadc/spinifex/spinifex/vm" +) + +// OVSPlumber implements vm.NetworkPlumber using system commands +// (ip, ovs-vsctl); tests use a mock satisfying the same interface. +type OVSPlumber struct{} + +var _ vm.NetworkPlumber = (*OVSPlumber)(nil) + +// NewOVSPlumber returns the default plumber wired to utils.SudoCommand. +func NewOVSPlumber() *OVSPlumber { return &OVSPlumber{} } + +// SetupTap creates the kernel tap, brings it up, and attaches it to spec.Bridge. +// The pre-create del-port is unconditional because OVS conf.db survives reboot +// while kernel taps don't, so a /sys/class/net check would miss orphan ports. +// Rejected alternative: `--may-exist add-port` — it would silently keep stale +// external_ids from a prior launch with a different ENI/MAC. +func (p *OVSPlumber) SetupTap(spec vm.TapSpec) error { + if err := utils.SudoCommand("ovs-vsctl", "--if-exists", "del-port", spec.Bridge, spec.Name).Run(); err != nil { + slog.Warn("Pre-create del-port failed (continuing)", "tap", spec.Name, "bridge", spec.Bridge, "err", err) + } + if _, err := os.Stat("/sys/class/net/" + spec.Name); err == nil { + if err := utils.SudoCommand("ip", "tuntap", "del", "dev", spec.Name, "mode", "tap").Run(); err != nil { + slog.Warn("Pre-create tap del failed (continuing)", "tap", spec.Name, "err", err) + } + } + + // When the daemon runs non-root (production: User=spinifex-daemon), child + // QEMU processes inherit that uid without CAP_NET_ADMIN. The kernel's tun + // attach check then requires the tap to be owned by the calling euid + // (drivers/net/tun.c: tun_chr_open + TUNSETIFF perm check). Without the + // `user` flag the tap defaults to root:root and qemu's TUNSETIFF fails + // with EPERM ("could not configure /dev/net/tun: Operation not permitted"). + // Pass numeric uid/gid directly — user.Current() can fail silently under + // hardened systemd units / static builds, dropping the flags and leaving + // the tap root-owned (LB microvm in cell-17/tofu-examples). Numeric values + // always work and skip the NSS lookup. Root daemons skip this — they have + // CAP_NET_ADMIN ambient. + addArgs := []string{"tuntap", "add", "dev", spec.Name, "mode", "tap"} + if uid := os.Geteuid(); uid != 0 { + addArgs = append(addArgs, "user", strconv.Itoa(uid), "group", strconv.Itoa(os.Getegid())) + } + if out, err := utils.SudoCommand("ip", addArgs...).CombinedOutput(); err != nil { + return fmt.Errorf("create tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err) + } + + if out, err := utils.SudoCommand("ip", "link", "set", spec.Name, "up").CombinedOutput(); err != nil { + if cleanErr := utils.SudoCommand("ip", "tuntap", "del", "dev", spec.Name, "mode", "tap").Run(); cleanErr != nil { + slog.Warn("Failed to clean up tap after bring-up failure", "tap", spec.Name, "err", cleanErr) + } + return fmt.Errorf("bring up tap %s: %s: %w", spec.Name, strings.TrimSpace(string(out)), err) + } + + addPortArgs := []string{"add-port", spec.Bridge, spec.Name} + if len(spec.ExternalIDs) > 0 { + addPortArgs = append(addPortArgs, "--", "set", "Interface", spec.Name) + // Sort keys for deterministic command construction (test assertions, log readability). + keys := make([]string, 0, len(spec.ExternalIDs)) + for k := range spec.ExternalIDs { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + addPortArgs = append(addPortArgs, fmt.Sprintf("external_ids:%s=%s", k, spec.ExternalIDs[k])) + } + } + if out, err := utils.SudoCommand("ovs-vsctl", addPortArgs...).CombinedOutput(); err != nil { + if cleanErr := utils.SudoCommand("ip", "tuntap", "del", "dev", spec.Name, "mode", "tap").Run(); cleanErr != nil { + slog.Warn("Failed to clean up tap after OVS failure", "tap", spec.Name, "err", cleanErr) + } + return fmt.Errorf("add tap %s to %s: %s: %w", spec.Name, spec.Bridge, strings.TrimSpace(string(out)), err) + } + + slog.Info("Tap plumbing complete", "tap", spec.Name, "bridge", spec.Bridge, "external_ids", spec.ExternalIDs) + return nil +} + +// CleanupTap removes the named tap from OVS (any bridge) and the kernel. +// Idempotent: callers may invoke for an instance that never reached SetupTap +// (e.g. terminate that races mid-launch). +func (p *OVSPlumber) CleanupTap(name string) error { + if out, err := utils.SudoCommand("ovs-vsctl", "--if-exists", "del-port", name).CombinedOutput(); err != nil { + slog.Warn("Failed to remove tap from OVS", "tap", name, "err", err, "out", strings.TrimSpace(string(out))) + } + + if _, err := os.Stat("/sys/class/net/" + name); os.IsNotExist(err) { + slog.Info("Tap already absent", "tap", name) + return nil + } + + if out, err := utils.SudoCommand("ip", "tuntap", "del", "dev", name, "mode", "tap").CombinedOutput(); err != nil { + return fmt.Errorf("delete tap %s: %s: %w", name, strings.TrimSpace(string(out)), err) + } + + slog.Info("Tap cleanup complete", "tap", name) + return nil +} diff --git a/spinifex/network/host/plumber_test.go b/spinifex/network/host/plumber_test.go new file mode 100644 index 000000000..1ecfb0e21 --- /dev/null +++ b/spinifex/network/host/plumber_test.go @@ -0,0 +1,207 @@ +package host + +import ( + "os/exec" + "slices" + "strings" + "testing" + + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/mulgadc/spinifex/spinifex/vm" +) + +func TestOVSPlumber_SetupTap_AddPortArgs(t *testing.T) { + cases := []struct { + name string + spec vm.TapSpec + wantAddPort []string // expected tail of ovs-vsctl args (after add-port ) + }{ + { + name: "vpc style with external_ids", + spec: vm.TapSpec{ + Name: "tapeni-test", + Bridge: "br-int", + ExternalIDs: map[string]string{ + "iface-id": "port-eni-test", + "attached-mac": "02:00:00:aa:bb:cc", + }, + }, + wantAddPort: []string{ + "add-port", "br-int", "tapeni-test", + "--", "set", "Interface", "tapeni-test", + "external_ids:attached-mac=02:00:00:aa:bb:cc", + "external_ids:iface-id=port-eni-test", + }, + }, + { + name: "mgmt style without external_ids", + spec: vm.TapSpec{ + Name: "mg-test", + Bridge: "br-mgmt", + }, + wantAddPort: []string{"add-port", "br-mgmt", "mg-test"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var calls [][]string + t.Cleanup(utils.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd { + call := append([]string{name}, args...) + calls = append(calls, call) + return exec.Command("/bin/true") + })) + + p := NewOVSPlumber() + if err := p.SetupTap(tc.spec); err != nil { + t.Fatalf("SetupTap: %v", err) + } + + // Find the add-port invocation (last ovs-vsctl call). + var addPort []string + for _, c := range calls { + if len(c) >= 2 && c[0] == "ovs-vsctl" && c[1] != "--if-exists" { + addPort = c[1:] + } + } + if addPort == nil { + t.Fatalf("no add-port invocation captured; calls=%v", calls) + } + if !slices.Equal(addPort, tc.wantAddPort) { + t.Errorf("add-port args = %v, want %v", addPort, tc.wantAddPort) + } + }) + } +} + +// "lo" is always present so os.Stat hits the pre-create cleanup branch. +func TestOVSPlumber_SetupTap_PreExistingKernelTap(t *testing.T) { + failTuntapDel := true + t.Cleanup(utils.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd { + if name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "del" && failTuntapDel { + failTuntapDel = false + return exec.Command("/bin/false") + } + return exec.Command("/bin/true") + })) + + p := NewOVSPlumber() + if err := p.SetupTap(vm.TapSpec{Name: "lo", Bridge: "br-int"}); err != nil { + t.Fatalf("SetupTap with stubs should succeed: %v", err) + } +} + +func TestOVSPlumber_SetupTap_ErrorBranches(t *testing.T) { + cases := []struct { + name string + failMatch func(name string, args []string) bool + wantErr string + }{ + { + name: "ovs del-port failure is logged-warn (continues)", + failMatch: func(name string, args []string) bool { + return name == "ovs-vsctl" && len(args) >= 1 && args[0] == "--if-exists" + }, + wantErr: "", // pre-clear failure is swallowed; SetupTap returns nil + }, + { + name: "tuntap add failure surfaces", + failMatch: func(name string, args []string) bool { + return name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "add" + }, + wantErr: "create tap", + }, + { + name: "ip link set up failure surfaces and triggers tap cleanup", + failMatch: func(name string, args []string) bool { + return name == "ip" && len(args) >= 1 && args[0] == "link" + }, + wantErr: "bring up tap", + }, + { + name: "ovs add-port failure surfaces and triggers tap cleanup", + failMatch: func(name string, args []string) bool { + return name == "ovs-vsctl" && len(args) >= 1 && args[0] == "add-port" + }, + wantErr: "add tap", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Cleanup(utils.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd { + if tc.failMatch(name, args) { + return exec.Command("/bin/false") + } + return exec.Command("/bin/true") + })) + + p := NewOVSPlumber() + err := p.SetupTap(vm.TapSpec{Name: "tap-test-noexist", Bridge: "br-int"}) + if tc.wantErr == "" { + if err != nil { + t.Fatalf("expected nil, got %v", err) + } + return + } + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("expected error containing %q, got %v", tc.wantErr, err) + } + }) + } +} + +func TestOVSPlumber_CleanupTap_PresentKernelTap(t *testing.T) { + var sawTuntapDel bool + t.Cleanup(utils.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd { + if name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "del" { + sawTuntapDel = true + } + return exec.Command("/bin/true") + })) + + p := NewOVSPlumber() + if err := p.CleanupTap("lo"); err != nil { + t.Fatalf("CleanupTap: %v", err) + } + if !sawTuntapDel { + t.Errorf("expected ip tuntap del to be invoked, got no call") + } +} + +func TestOVSPlumber_CleanupTap_DelPortLoggedWarn(t *testing.T) { + t.Cleanup(utils.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd { + if name == "ovs-vsctl" { + return exec.Command("/bin/false") + } + return exec.Command("/bin/true") + })) + + p := NewOVSPlumber() + if err := p.CleanupTap("tap-test-noexist"); err != nil { + t.Fatalf("CleanupTap: %v", err) + } +} + +func TestOVSPlumber_CleanupTap_TuntapDelFailure(t *testing.T) { + t.Cleanup(utils.SetSudoCommandForTest(func(name string, args ...string) *exec.Cmd { + if name == "ip" && len(args) >= 2 && args[0] == "tuntap" && args[1] == "del" { + return exec.Command("/bin/false") + } + return exec.Command("/bin/true") + })) + + p := NewOVSPlumber() + err := p.CleanupTap("lo") + if err == nil || !strings.Contains(err.Error(), "delete tap") { + t.Fatalf("expected 'delete tap' error, got %v", err) + } +} + +// CleanupTap on a name with neither OVS port nor kernel tap must return nil. +func TestOVSPlumber_CleanupTap_MissingKernelTap(t *testing.T) { + p := NewOVSPlumber() + if err := p.CleanupTap("mg-test-noexist"); err != nil { + t.Fatalf("expected nil for missing kernel tap, got: %v", err) + } +} diff --git a/spinifex/network/host/run.go b/spinifex/network/host/run.go new file mode 100644 index 000000000..10bf8f06c --- /dev/null +++ b/spinifex/network/host/run.go @@ -0,0 +1,89 @@ +package host + +import ( + "context" + "fmt" + "net" + "net/netip" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// execRunner runs the command directly when uid 0, otherwise via sudo. +type execRunner struct{} + +var _ Runner = execRunner{} + +// NewExecRunner returns the default Runner. +func NewExecRunner() Runner { return execRunner{} } + +func (execRunner) Run(ctx context.Context, name string, args ...string) ([]byte, error) { + var cmd *exec.Cmd + if os.Getuid() == 0 { + cmd = exec.CommandContext(ctx, name, args...) + } else { + cmd = exec.CommandContext(ctx, "sudo", append([]string{name}, args...)...) + } + out, err := cmd.CombinedOutput() + if err != nil { + return out, fmt.Errorf("%s %s: %s: %w", name, strings.Join(args, " "), strings.TrimSpace(string(out)), err) + } + return out, nil +} + +// kernelReader satisfies InterfaceReader via the live kernel. +type kernelReader struct{} + +var _ InterfaceReader = kernelReader{} + +// NewKernelReader returns the default InterfaceReader. +func NewKernelReader() InterfaceReader { return kernelReader{} } + +func (kernelReader) BridgeCIDR(name string) (netip.Prefix, error) { + iface, err := net.InterfaceByName(name) + if err != nil { + return netip.Prefix{}, fmt.Errorf("interface %q: %w", name, err) + } + addrs, err := iface.Addrs() + if err != nil { + return netip.Prefix{}, fmt.Errorf("addrs %q: %w", name, err) + } + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok { + continue + } + v4 := ipnet.IP.To4() + if v4 == nil { + continue + } + ones, _ := ipnet.Mask.Size() + addr, _ := netip.AddrFromSlice(v4) + return netip.PrefixFrom(addr, ones), nil + } + return netip.Prefix{}, fmt.Errorf("%w: %q", ErrNoUplinkAddr, name) +} + +func (kernelReader) LinkMAC(name string) (net.HardwareAddr, error) { + iface, err := net.InterfaceByName(name) + if err != nil { + return nil, fmt.Errorf("interface %q: %w", name, err) + } + if len(iface.HardwareAddr) == 0 { + return nil, fmt.Errorf("interface %q: no hardware address", name) + } + return iface.HardwareAddr, nil +} + +func (kernelReader) LinkMaster(name string) (string, error) { + target, err := os.Readlink(filepath.Join("/sys/class/net", name, "master")) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", err + } + return filepath.Base(target), nil +} diff --git a/spinifex/network/host/veth.go b/spinifex/network/host/veth.go new file mode 100644 index 000000000..b13843c8f --- /dev/null +++ b/spinifex/network/host/veth.go @@ -0,0 +1,100 @@ +package host + +import ( + "context" + "fmt" + "log/slog" + "net" + "net/netip" + "strings" +) + +// veth-wan-ovs is the OVS-side endpoint; veth-wan-br is the Linux-bridge-side endpoint. +const ( + VethOVSEnd = "veth-wan-ovs" + VethLinuxEnd = "veth-wan-br" +) + +// Veth implements Wiring for the "Linux bridge bridged into OVS via veth" model. +// OS-managed Linux bridge holds the IPv4 uplink; NAT is centralised (one chassis owns the IP). +type Veth struct { + // LinuxBridge holds the OS-assigned WAN IPv4 (e.g. "br-wan"). + LinuxBridge string + + // UplinkBridge is the OVS bridge ovn-bridge-mappings targets (conventionally "br-ext"). + UplinkBridge string + + Runner Runner + Reader InterfaceReader +} + +var _ Wiring = (*Veth)(nil) + +func (v *Veth) runner() Runner { + if v.Runner != nil { + return v.Runner + } + return NewExecRunner() +} + +func (v *Veth) reader() InterfaceReader { + if v.Reader != nil { + return v.Reader + } + return NewKernelReader() +} + +// EnsureBridges ensures br-int and UplinkBridge exist. The veth pair is +// provisioned by spinifex-veth-wan.service. +func (v *Veth) EnsureBridges(ctx context.Context) error { + return ensureBridges(ctx, v.runner(), v.UplinkBridge) +} + +// EnsureUplinkPort verifies the veth pair wiring and returns the OVS-side MAC. +// The gateway LRP uses this MAC so OVS-egress frames carry the L2 src the +// upstream router learned. +func (v *Veth) EnsureUplinkPort(ctx context.Context) (net.HardwareAddr, error) { + if v.LinuxBridge == "" { + return nil, fmt.Errorf("host.Veth: LinuxBridge required") + } + if v.UplinkBridge == "" { + return nil, fmt.Errorf("host.Veth: UplinkBridge required") + } + + out, err := v.runner().Run(ctx, "ovs-vsctl", "port-to-br", VethOVSEnd) + if err != nil { + return nil, fmt.Errorf("port-to-br %q: not on OVS — is spinifex-veth-wan.service running? %w", VethOVSEnd, err) + } + br := strings.TrimSpace(string(out)) + if br != v.UplinkBridge { + return nil, fmt.Errorf("host.Veth: %q on bridge %q, expected %q", VethOVSEnd, br, v.UplinkBridge) + } + + master, err := v.reader().LinkMaster(VethLinuxEnd) + if err != nil { + return nil, fmt.Errorf("host.Veth: read master for %q: %w", VethLinuxEnd, err) + } + if master != v.LinuxBridge { + return nil, fmt.Errorf("host.Veth: %q master is %q, expected %q", VethLinuxEnd, master, v.LinuxBridge) + } + + mac, err := v.reader().LinkMAC(VethOVSEnd) + if err != nil { + return nil, fmt.Errorf("read MAC for %q: %w", VethOVSEnd, err) + } + slog.Info("host.Veth: uplink verified", + "ovs_end", VethOVSEnd, "linux_end", VethLinuxEnd, + "linux_bridge", v.LinuxBridge, "uplink_bridge", v.UplinkBridge, "mac", mac.String()) + return mac, nil +} + +// UplinkMode returns UplinkModeVeth (centralised NAT). +func (v *Veth) UplinkMode() UplinkMode { return UplinkModeVeth } + +// ExternalCIDR returns the IPv4 prefix on LinuxBridge (OS-assigned). +func (v *Veth) ExternalCIDR(_ context.Context) (netip.Prefix, error) { + if v.LinuxBridge == "" { + return netip.Prefix{}, fmt.Errorf("host.Veth: LinuxBridge required") + } + return v.reader().BridgeCIDR(v.LinuxBridge) +} diff --git a/spinifex/network/invariants/doc.go b/spinifex/network/invariants/doc.go new file mode 100644 index 000000000..21dd91b10 --- /dev/null +++ b/spinifex/network/invariants/doc.go @@ -0,0 +1,12 @@ +// Package invariants holds executable tests that pin the spinifex network +// layer tree to the safety and liveness invariants declared in ADR-0006 +// (docs/development/proposals/az-local-architecture/0006-spinifex-network-layer-contract.md). +// +// Each test name encodes its governing ADR clause ID (TestS1_…, TestS4_…, +// TestL1_…). On failure the test quotes the ADR clause verbatim so the fix +// path leads back to the contract, not to the test author. +// +// The package contains no production code; it exists solely so `go test +// ./spinifex/network/invariants/...` runs the suite without polluting any +// layer's own package. +package invariants diff --git a/spinifex/network/invariants/federation_isolation_test.go b/spinifex/network/invariants/federation_isolation_test.go new file mode 100644 index 000000000..38ad1a32c --- /dev/null +++ b/spinifex/network/invariants/federation_isolation_test.go @@ -0,0 +1,160 @@ +package invariants + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "sort" + "strings" + "testing" +) + +// TestS6_FederationIsolated enforces ADR-0006 clause S6. +// +// "L4 is the only layer aware of other AZs. No layer below L4 references +// remote AZ identifiers, OVN-IC transit switches, or inter-AZ link state. +// Cross-AZ federation is always mediated by L4." +func TestS6_FederationIsolated(t *testing.T) { + const clause = `ADR-0006 S6: "L4 is the only layer aware of other AZs. ` + + `No layer below L4 references remote AZ identifiers, OVN-IC transit ` + + `switches, or inter-AZ link state. Cross-AZ federation is always ` + + `mediated by L4."` + + forbiddenIdents := map[string]struct{}{ + "TransitSwitch": {}, + "TransitRouterPort": {}, + } + forbiddenLiterals := []string{ + "ovn-ic", + "ovn-icnbctl", + "ovn-icsbctl", + } + + roots := []string{ + filepath.Join(repoRoot(t), "spinifex", "network"), + filepath.Join(repoRoot(t), "spinifex", "daemon"), + filepath.Join(repoRoot(t), "spinifex", "handlers"), + } + type hit struct { + file string + line int + sym string + } + var hits []hit + + for _, root := range roots { + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if isVendoredOrGenerated(d.Name()) { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + if strings.HasSuffix(path, "_test.go") { + return nil + } + if isS6Exempt(path) { + return nil + } + fset := token.NewFileSet() + file, perr := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) + if perr != nil { + t.Fatalf("parse %s: %v", path, perr) + } + ast.Inspect(file, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.SelectorExpr: + if _, bad := forbiddenIdents[x.Sel.Name]; bad { + pos := fset.Position(x.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, x.Sel.Name}) + } + case *ast.Ident: + if _, bad := forbiddenIdents[x.Name]; bad { + pos := fset.Position(x.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, x.Name}) + } + case *ast.BasicLit: + if x.Kind != token.STRING { + return true + } + lit, ok := stringLit(x) + if !ok { + return true + } + for _, sub := range forbiddenLiterals { + if strings.Contains(lit, sub) { + pos := fset.Position(x.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, "\"" + lit + "\""}) + break + } + } + } + return true + }) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } + + if len(hits) == 0 { + return + } + + sort.Slice(hits, func(i, j int) bool { + if hits[i].file != hits[j].file { + return hits[i].file < hits[j].file + } + return hits[i].line < hits[j].line + }) + + rootForRel := repoRoot(t) + var b strings.Builder + b.WriteString(clause) + b.WriteString("\n") + limit := 5 + for i, h := range hits { + if i >= limit { + b.WriteString(" …\n") + break + } + b.WriteString(" ") + b.WriteString(relTo(h.file, rootForRel)) + b.WriteString(":") + b.WriteString(itoa(h.line)) + b.WriteString(": federation symbol ") + b.WriteString(h.sym) + b.WriteString(" outside L4\n") + } + b.WriteString(" Fix: route the inter-AZ operation through network/federation/. ") + b.WriteString("Only L4 may reference remote-AZ identifiers, OVN-IC objects, ") + b.WriteString("or inter-AZ link state.\n") + if len(hits) > limit { + b.WriteString(" ") + b.WriteString(itoa(len(hits) - limit)) + b.WriteString(" further violations suppressed.\n") + } + t.Fatalf("%s", b.String()) +} + +func isS6Exempt(path string) bool { + clean := filepath.ToSlash(path) + switch { + case strings.HasSuffix(clean, "/network/topology/names.go"): + return true + case strings.Contains(clean, "/network/federation/"): + return true + case strings.Contains(clean, "/network/invariants/"): + return true + } + return false +} diff --git a/spinifex/network/invariants/host_containment_test.go b/spinifex/network/invariants/host_containment_test.go new file mode 100644 index 000000000..ef2374997 --- /dev/null +++ b/spinifex/network/invariants/host_containment_test.go @@ -0,0 +1,134 @@ +package invariants + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "sort" + "strings" + "testing" +) + +// TestS2_BridgeModeContainedInL0 enforces ADR-0006 clause S2. +// +// "Bridge mode is contained in L0. No layer above L0 reads uplink type, +// bridge mode, or physical NIC state directly. L0's UplinkMode() is the +// only resolver; all layers above receive a typed enum at init time." +func TestS2_BridgeModeContainedInL0(t *testing.T) { + const clause = `ADR-0006 S2: "Bridge mode is contained in L0. No layer ` + + `above L0 reads uplink type, bridge mode, or physical NIC state ` + + `directly. L0's UplinkMode() is the only resolver; all layers ` + + `above receive a typed enum at init time."` + + hostTools := map[string]struct{}{ + "ovs-vsctl": {}, + "ovs-ofctl": {}, + "ovn-nbctl": {}, + "ovn-sbctl": {}, + } + + roots := []string{ + filepath.Join(repoRoot(t), "spinifex", "network"), + filepath.Join(repoRoot(t), "spinifex", "daemon"), + filepath.Join(repoRoot(t), "spinifex", "vm"), + } + type hit struct { + file string + line int + tool string + } + var hits []hit + + for _, root := range roots { + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if isVendoredOrGenerated(d.Name()) { + return filepath.SkipDir + } + // network/host/ is L0; only layer that may shell out. + if strings.HasSuffix(filepath.ToSlash(path), "/network/host") { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + if strings.HasSuffix(path, "_test.go") { + return nil + } + fset := token.NewFileSet() + file, perr := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) + if perr != nil { + t.Fatalf("parse %s: %v", path, perr) + } + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + if len(call.Args) < 1 { + return true + } + lit, ok := stringLit(call.Args[0]) + if !ok { + return true + } + if _, banned := hostTools[lit]; !banned { + return true + } + pos := fset.Position(call.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, lit}) + return true + }) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } + + if len(hits) == 0 { + return + } + + sort.Slice(hits, func(i, j int) bool { + if hits[i].file != hits[j].file { + return hits[i].file < hits[j].file + } + return hits[i].line < hits[j].line + }) + + rootForRel := repoRoot(t) + var b strings.Builder + b.WriteString(clause) + b.WriteString("\n") + limit := 5 + for i, h := range hits { + if i >= limit { + b.WriteString(" …\n") + break + } + b.WriteString(" ") + b.WriteString(relTo(h.file, rootForRel)) + b.WriteString(":") + b.WriteString(itoa(h.line)) + b.WriteString(": invokes ") + b.WriteString(h.tool) + b.WriteString(" outside network/host/\n") + } + b.WriteString(" Fix: move the bridge/NIC operation into network/host/ ") + b.WriteString("and expose a typed method on host.Wiring. Layers above ") + b.WriteString("L0 may not shell out to host tools directly.\n") + if len(hits) > limit { + b.WriteString(" ") + b.WriteString(itoa(len(hits) - limit)) + b.WriteString(" further violations suppressed.\n") + } + t.Fatalf("%s", b.String()) +} diff --git a/spinifex/network/invariants/ipsec_test.go b/spinifex/network/invariants/ipsec_test.go new file mode 100644 index 000000000..76b9128a1 --- /dev/null +++ b/spinifex/network/invariants/ipsec_test.go @@ -0,0 +1,88 @@ +package invariants + +import ( + "encoding/json" + "os/exec" + "sort" + "strings" + "testing" +) + +// TestS8_IPSecOVNNativeOnly enforces ADR-0006 clause S8. +// +// "IPSec is OVN-native only. No layer implements custom IKEv2, XFRM +// rules, or direct strongSwan management. IPSec SA lifecycle is +// delegated entirely to OVN native IPSec and is invisible above L0." +func TestS8_IPSecOVNNativeOnly(t *testing.T) { + const clause = `ADR-0006 S8: "IPSec is OVN-native only. No layer ` + + `implements custom IKEv2, XFRM rules, or direct strongSwan ` + + `management. IPSec SA lifecycle is delegated entirely to OVN ` + + `native IPSec and is invisible above L0."` + + deniedSubstrings := []string{ + "strongswan", + "strongSwan", + "/xfrm", + "netlink/nl/xfrm", + "ikev2", + "IKEv2", + "libreswan", + "openswan", + } + + cmd := exec.Command("go", "list", "-json", "./...") + cmd.Dir = repoRoot(t) + out, err := cmd.Output() + if err != nil { + t.Fatalf("go list: %v", err) + } + + type hit struct { + pkg string + imp string + } + var hits []hit + dec := json.NewDecoder(strings.NewReader(string(out))) + for dec.More() { + var p goListPackage + if err := dec.Decode(&p); err != nil { + t.Fatalf("decode go list: %v", err) + } + if !strings.HasPrefix(p.ImportPath, networkRoot) { + continue + } + for _, imp := range p.Imports { + for _, sub := range deniedSubstrings { + if strings.Contains(imp, sub) { + hits = append(hits, hit{p.ImportPath, imp}) + } + } + } + } + + if len(hits) == 0 { + return + } + + sort.Slice(hits, func(i, j int) bool { + if hits[i].pkg != hits[j].pkg { + return hits[i].pkg < hits[j].pkg + } + return hits[i].imp < hits[j].imp + }) + + var b strings.Builder + b.WriteString(clause) + b.WriteString("\n") + for _, h := range hits { + b.WriteString(" ") + b.WriteString(shortPkg(h.pkg)) + b.WriteString(" imports ") + b.WriteString(h.imp) + b.WriteString("\n") + } + b.WriteString(" Fix: remove the import. IPSec SA lifecycle is handled ") + b.WriteString("by OVN native IPSec (strongSwan-as-OVN-internal); do not ") + b.WriteString("manage IKEv2 / XFRM / strongSwan directly from Go.\n") + t.Fatalf("%s", b.String()) +} diff --git a/spinifex/network/invariants/layers_classify_test.go b/spinifex/network/invariants/layers_classify_test.go new file mode 100644 index 000000000..dadf55f89 --- /dev/null +++ b/spinifex/network/invariants/layers_classify_test.go @@ -0,0 +1,27 @@ +package invariants + +import "testing" + +func TestS1_ClassifyTable(t *testing.T) { + cases := []struct { + name string + from, to packageKind + wantBad bool + }{ + {"L3 imports L0 (downward, permitted)", kindL3Policy, kindL0Host, false}, + {"L5 imports L1 (downward skip, permitted per ADR L5 description)", kindL5External, kindL1OVN, false}, + {"L0 imports L1 (upward, forbidden)", kindL0Host, kindL1OVN, true}, + {"L2 imports L3 (upward, forbidden)", kindL2Topology, kindL3Policy, true}, + {"reconcile imports L1 (orchestrator, permitted)", kindCrossCutter, kindL1OVN, false}, + {"L2 imports reconcile (layer pulling orchestrator, forbidden)", kindL2Topology, kindCrossCutter, true}, + {"L1 imports L1 (same layer, permitted)", kindL1OVN, kindL1OVN, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := classify(tc.from, tc.to) + if (got != "") != tc.wantBad { + t.Fatalf("classify(%v, %v) = %q; wantBad=%v", tc.from, tc.to, got, tc.wantBad) + } + }) + } +} diff --git a/spinifex/network/invariants/layers_test.go b/spinifex/network/invariants/layers_test.go new file mode 100644 index 000000000..ca314a7f7 --- /dev/null +++ b/spinifex/network/invariants/layers_test.go @@ -0,0 +1,277 @@ +package invariants + +import ( + "encoding/json" + "os/exec" + "sort" + "strings" + "testing" +) + +// TestS1_LayerSkipProhibited enforces ADR-0006 clause S1. +// +// "No code above layer Lk calls through to a layer below Lk's immediate +// interface. Every cross-layer call passes through the typed interface +// of the immediate lower neighbor." +func TestS1_LayerSkipProhibited(t *testing.T) { + const clause = `ADR-0006 S1: "No code above layer Lk calls through to a ` + + `layer below Lk's immediate interface. Every cross-layer call ` + + `passes through the typed interface of the immediate lower neighbor."` + + pkgs := listNetworkPackages(t) + + type violation struct { + from string + fromKind packageKind + to string + toKind packageKind + reason string + } + var bad []violation + + for _, p := range pkgs { + fromKind := layerOf(p.ImportPath) + for _, imp := range p.Imports { + if !strings.HasPrefix(imp, networkRoot) { + continue + } + toKind := layerOf(imp) + reason := classify(fromKind, toKind) + if reason == "" { + continue + } + bad = append(bad, violation{ + from: p.ImportPath, fromKind: fromKind, + to: imp, toKind: toKind, + reason: reason, + }) + } + } + + if len(bad) == 0 { + return + } + + sort.Slice(bad, func(i, j int) bool { + if bad[i].from != bad[j].from { + return bad[i].from < bad[j].from + } + return bad[i].to < bad[j].to + }) + + var b strings.Builder + b.WriteString(clause) + b.WriteString("\n") + limit := 5 + for i, v := range bad { + if i >= limit { + b.WriteString(" …\n") + break + } + b.WriteString(" ") + b.WriteString(shortPkg(v.from)) + b.WriteString(" (") + b.WriteString(v.fromKind.String()) + b.WriteString(") -> ") + b.WriteString(shortPkg(v.to)) + b.WriteString(" (") + b.WriteString(v.toKind.String()) + b.WriteString("): ") + b.WriteString(v.reason) + b.WriteString("\n") + } + b.WriteString(" Fix: route the call through the immediate lower ") + b.WriteString("layer's typed interface, or move the offending code to ") + b.WriteString("an orchestrator (reconcile / subscribers).\n") + if len(bad) > limit { + b.WriteString(" ") + b.WriteString(itoa(len(bad) - limit)) + b.WriteString(" further violations suppressed.\n") + } + t.Fatalf("%s", b.String()) +} + +// packageKind tags a package by its position in the ADR-0006 layer model. +type packageKind int + +const ( + kindUnknown packageKind = iota + kindL0Host + kindL1OVN + kindL2Topology + kindL3Policy + kindL4Federation + kindL5External + kindCrossCutter + kindInvariants +) + +func (k packageKind) String() string { + switch k { + case kindL0Host: + return "L0/host" + case kindL1OVN: + return "L1/ovn" + case kindL2Topology: + return "L2/topology" + case kindL3Policy: + return "L3/policy" + case kindL4Federation: + return "L4/federation" + case kindL5External: + return "L5/external" + case kindCrossCutter: + return "cross-cutter" + case kindInvariants: + return "invariants" + default: + return "unknown" + } +} + +// rank returns the ADR layer number; non-layered packages return -1. +func (k packageKind) rank() int { + switch k { + case kindL0Host: + return 0 + case kindL1OVN: + return 1 + case kindL2Topology: + return 2 + case kindL3Policy: + return 3 + case kindL4Federation: + return 4 + case kindL5External: + return 5 + } + return -1 +} + +const networkRoot = "github.com/mulgadc/spinifex/spinifex/network" + +func layerOf(importPath string) packageKind { + suffix := strings.TrimPrefix(importPath, networkRoot) + suffix = strings.TrimPrefix(suffix, "/") + switch { + case suffix == "host" || strings.HasPrefix(suffix, "host/"): + return kindL0Host + case suffix == "ovn" || strings.HasPrefix(suffix, "ovn/"): + return kindL1OVN + case suffix == "topology" || strings.HasPrefix(suffix, "topology/"): + return kindL2Topology + case suffix == "policy" || strings.HasPrefix(suffix, "policy/"): + return kindL3Policy + case suffix == "federation" || strings.HasPrefix(suffix, "federation/"): + return kindL4Federation + case suffix == "external" || strings.HasPrefix(suffix, "external/"): + return kindL5External + case suffix == "reconcile" || strings.HasPrefix(suffix, "reconcile/"): + return kindCrossCutter + case suffix == "subscribers" || strings.HasPrefix(suffix, "subscribers/"): + return kindCrossCutter + case suffix == "invariants" || strings.HasPrefix(suffix, "invariants/"): + return kindInvariants + } + return kindUnknown +} + +// classify returns a non-empty reason iff the edge from→to violates S1. +func classify(from, to packageKind) string { + if from == to { + return "" + } + if from == kindInvariants || to == kindInvariants { + return "" + } + // Layer importing a cross-cutter would re-tangle the tree. + if to == kindCrossCutter && from != kindCrossCutter { + return "layer importing a cross-cutter (reconcile/subscribers)" + } + if from == kindCrossCutter { + return "" + } + fr, tr := from.rank(), to.rank() + if fr < 0 || tr < 0 { + return "" + } + if tr > fr { + return "upward layer import (L" + itoa(fr) + " -> L" + itoa(tr) + ")" + } + return "" +} + +type goListPackage struct { + ImportPath string `json:"ImportPath"` + Imports []string `json:"Imports"` +} + +func listNetworkPackages(t *testing.T) []goListPackage { + t.Helper() + cmd := exec.Command("go", "list", "-json", "./...") + cmd.Dir = repoRoot(t) + out, err := cmd.Output() + if err != nil { + t.Fatalf("go list: %v", err) + } + var pkgs []goListPackage + dec := json.NewDecoder(strings.NewReader(string(out))) + for dec.More() { + var p goListPackage + if err := dec.Decode(&p); err != nil { + t.Fatalf("decode go list output: %v", err) + } + if !strings.HasPrefix(p.ImportPath, networkRoot) { + continue + } + if layerOf(p.ImportPath) == kindInvariants { + continue + } + pkgs = append(pkgs, p) + } + if len(pkgs) == 0 { + t.Fatalf("go list returned no packages under %s", networkRoot) + } + return pkgs +} + +// repoRoot returns the spinifex module root via `go env GOMOD`. +func repoRoot(t *testing.T) string { + t.Helper() + cmd := exec.Command("go", "env", "GOMOD") + out, err := cmd.Output() + if err != nil { + t.Fatalf("go env GOMOD: %v", err) + } + gomod := strings.TrimSpace(string(out)) + if gomod == "" || gomod == "/dev/null" { + t.Fatalf("not in a go module (GOMOD=%q)", gomod) + } + return strings.TrimSuffix(gomod, "/go.mod") +} + +func shortPkg(p string) string { + return strings.TrimPrefix(p, networkRoot+"/") +} + +func itoa(n int) string { + if n == 0 { + return "0" + } + neg := n < 0 + if neg { + n = -n + } + var buf [20]byte + i := len(buf) + for n > 0 { + i-- + buf[i] = byte('0' + n%10) + n /= 10 + } + if neg { + i-- + buf[i] = '-' + } + return string(buf[i:]) +} diff --git a/spinifex/network/invariants/names_test.go b/spinifex/network/invariants/names_test.go new file mode 100644 index 000000000..29549c2b4 --- /dev/null +++ b/spinifex/network/invariants/names_test.go @@ -0,0 +1,262 @@ +package invariants + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" +) + +// TestS4_OVNNamesL2Owned enforces ADR-0006 clause S4. +// +// "All OVN NB DB object names are constructed exclusively by L2 using +// the naming contract table. No layer above or below L2 constructs OVN +// object names independently." +func TestS4_OVNNamesL2Owned(t *testing.T) { + const clause = `ADR-0006 S4: "All OVN NB DB object names are constructed ` + + `exclusively by L2 using the naming contract table. No layer above ` + + `or below L2 constructs OVN object names independently."` + + roots := s4ScanRoots(t) + type hit struct { + file string + line int + col int + lit string + via string // "concat", "sprintf", "join" + } + var hits []hit + + for _, root := range roots { + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if isVendoredOrGenerated(d.Name()) { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + if strings.HasSuffix(path, "_test.go") { + return nil + } + if isS4Exempt(path) { + return nil + } + fset := token.NewFileSet() + file, perr := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) + if perr != nil { + t.Fatalf("parse %s: %v", path, perr) + } + ast.Inspect(file, func(n ast.Node) bool { + switch x := n.(type) { + case *ast.BinaryExpr: + if x.Op != token.ADD { + return true + } + lit, ok := stringLit(x.X) + if !ok { + return true + } + if !hasOVNPrefix(lit) { + return true + } + pos := fset.Position(x.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, pos.Column, lit, "concat"}) + case *ast.CallExpr: + name := calleeName(x.Fun) + switch name { + case "fmt.Sprintf": + if len(x.Args) < 1 { + return true + } + lit, ok := stringLit(x.Args[0]) + if !ok { + return true + } + if !hasOVNPrefix(lit) { + return true + } + pos := fset.Position(x.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, pos.Column, lit, "sprintf"}) + case "strings.Join": + if len(x.Args) < 1 { + return true + } + comp, ok := x.Args[0].(*ast.CompositeLit) + if !ok { + return true + } + for _, e := range comp.Elts { + lit, ok := stringLit(e) + if !ok { + continue + } + if !hasOVNPrefix(lit) { + continue + } + pos := fset.Position(e.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, pos.Column, lit, "join"}) + } + } + } + return true + }) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } + + if len(hits) == 0 { + return + } + + sort.Slice(hits, func(i, j int) bool { + if hits[i].file != hits[j].file { + return hits[i].file < hits[j].file + } + return hits[i].line < hits[j].line + }) + + rootForRel := repoRoot(t) + var b strings.Builder + b.WriteString(clause) + b.WriteString("\n") + limit := 5 + for i, h := range hits { + if i >= limit { + b.WriteString(" …\n") + break + } + b.WriteString(" ") + b.WriteString(relTo(h.file, rootForRel)) + b.WriteString(":") + b.WriteString(itoa(h.line)) + b.WriteString(":") + b.WriteString(itoa(h.col)) + b.WriteString(": ") + b.WriteString(h.via) + b.WriteString(" of ") + b.WriteString(strconv.Quote(h.lit)) + b.WriteString("\n") + } + b.WriteString(" Fix: add a helper to spinifex/network/topology/names.go ") + b.WriteString("and call it from the offending site. Do not construct OVN ") + b.WriteString("object names outside L2.\n") + if len(hits) > limit { + b.WriteString(" ") + b.WriteString(itoa(len(hits) - limit)) + b.WriteString(" further violations suppressed.\n") + } + t.Fatalf("%s", b.String()) +} + +// ovnNamePrefixes is the ADR-0006 contract table; keep in lockstep with topology/names.go. +var ovnNamePrefixes = []string{ + "vpc-", + "subnet-", + "port-", + "rtr-", + "rtr-port-", + "gw-", + "gw-port-", + "ext-", + "ext-port-", + "ts-", + "trp-", +} + +// awsIDPrefixes are AWS resource-IDs that share leading bytes with ovnNamePrefixes; excluded from hasOVNPrefix. +var awsIDPrefixes = []string{ + "vpc-cidr-assoc-", +} + +func hasOVNPrefix(s string) bool { + for _, a := range awsIDPrefixes { + if strings.HasPrefix(s, a) { + return false + } + } + for _, p := range ovnNamePrefixes { + if len(s) > len(p) && strings.HasPrefix(s, p) { + return true + } + } + return false +} + +func stringLit(e ast.Expr) (string, bool) { + bl, ok := e.(*ast.BasicLit) + if !ok || bl.Kind != token.STRING { + return "", false + } + v, err := strconv.Unquote(bl.Value) + if err != nil { + return "", false + } + return v, true +} + +func calleeName(fun ast.Expr) string { + switch f := fun.(type) { + case *ast.SelectorExpr: + if id, ok := f.X.(*ast.Ident); ok { + return id.Name + "." + f.Sel.Name + } + case *ast.Ident: + return f.Name + } + return "" +} + +func isS4Exempt(path string) bool { + clean := filepath.ToSlash(path) + switch { + case strings.HasSuffix(clean, "/network/topology/names.go"): + return true + case strings.HasSuffix(clean, "/network/topology/names_test.go"): + return true + case strings.Contains(clean, "/network/invariants/"): + return true + } + return false +} + +func isVendoredOrGenerated(dir string) bool { + switch dir { + case "vendor", "testdata", "node_modules", ".git": + return true + } + return false +} + +func s4ScanRoots(t *testing.T) []string { + t.Helper() + root := repoRoot(t) + return []string{ + filepath.Join(root, "spinifex", "network"), + filepath.Join(root, "spinifex", "vpcd"), + filepath.Join(root, "spinifex", "daemon"), + filepath.Join(root, "spinifex", "handlers"), + filepath.Join(root, "spinifex", "vm"), + } +} + +func relTo(p, root string) string { + r, err := filepath.Rel(root, p) + if err != nil { + return p + } + return filepath.ToSlash(r) +} diff --git a/spinifex/network/invariants/names_unit_test.go b/spinifex/network/invariants/names_unit_test.go new file mode 100644 index 000000000..aae9b5bd9 --- /dev/null +++ b/spinifex/network/invariants/names_unit_test.go @@ -0,0 +1,26 @@ +package invariants + +import "testing" + +func TestS4_HasOVNPrefix(t *testing.T) { + cases := []struct { + in string + want bool + }{ + {"vpc-abc", true}, + {"subnet-1", true}, + {"gw-port-x", true}, + {"ts-az1-vpc1", true}, + {"vpc-", false}, // bare prefix, no ID + {"vpc", false}, // not prefixed + {"hello-vpc-x", false}, + {"port-to-br", true}, // ovs-vsctl subcommand; precision comes from AST context, not matcher + {"vpc-cidr-assoc-", false}, + {"vpc-cidr-assoc-deadbeef", false}, + } + for _, tc := range cases { + if got := hasOVNPrefix(tc.in); got != tc.want { + t.Errorf("hasOVNPrefix(%q) = %v; want %v", tc.in, got, tc.want) + } + } +} diff --git a/spinifex/network/invariants/nat_mode_test.go b/spinifex/network/invariants/nat_mode_test.go new file mode 100644 index 000000000..191c956be --- /dev/null +++ b/spinifex/network/invariants/nat_mode_test.go @@ -0,0 +1,150 @@ +package invariants + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "sort" + "strings" + "testing" +) + +// TestS3_NATModeInitTimeConstant enforces ADR-0006 clause S3. +// +// "NAT distribution mode is an init-time constant. Centralised-vs- +// distributed NAT is determined once at startup from L0's UplinkMode() +// and never re-evaluated at runtime. No layer above L0 branches on NAT +// mode dynamically." +func TestS3_NATModeInitTimeConstant(t *testing.T) { + const clause = `ADR-0006 S3: "NAT distribution mode is an init-time ` + + `constant. Centralised-vs-distributed NAT is determined once at ` + + `startup from L0's UplinkMode() and never re-evaluated at runtime. ` + + `No layer above L0 branches on NAT mode dynamically."` + + type hit struct { + file string + line int + sym string + } + var hits []hit + + roots := []string{ + filepath.Join(repoRoot(t), "spinifex", "network"), + filepath.Join(repoRoot(t), "spinifex", "vpcd"), + filepath.Join(repoRoot(t), "spinifex", "daemon"), + } + for _, root := range roots { + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if isVendoredOrGenerated(d.Name()) { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + if strings.HasSuffix(path, "_test.go") { + return nil + } + if isS3Exempt(path) { + return nil + } + fset := token.NewFileSet() + file, perr := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) + if perr != nil { + t.Fatalf("parse %s: %v", path, perr) + } + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + name := calleeName(call.Fun) + selName := selectorTail(call.Fun) + switch { + case selName == "UplinkMode": + pos := fset.Position(call.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, selName + "()"}) + case strings.HasSuffix(name, "NATModeFromUplinkMode"): + pos := fset.Position(call.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, "NATModeFromUplinkMode"}) + } + return true + }) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } + + if len(hits) == 0 { + return + } + + sort.Slice(hits, func(i, j int) bool { + if hits[i].file != hits[j].file { + return hits[i].file < hits[j].file + } + return hits[i].line < hits[j].line + }) + + rootForRel := repoRoot(t) + var b strings.Builder + b.WriteString(clause) + b.WriteString("\n") + limit := 5 + for i, h := range hits { + if i >= limit { + b.WriteString(" …\n") + break + } + b.WriteString(" ") + b.WriteString(relTo(h.file, rootForRel)) + b.WriteString(":") + b.WriteString(itoa(h.line)) + b.WriteString(": ") + b.WriteString(h.sym) + b.WriteString(" called outside the init-time resolver location\n") + } + b.WriteString(" Fix: resolve NAT mode once at startup (vpcd entrypoint) ") + b.WriteString("and pass the typed enum to constructors. Store as an ") + b.WriteString("immutable field; do not re-resolve.\n") + if len(hits) > limit { + b.WriteString(" ") + b.WriteString(itoa(len(hits) - limit)) + b.WriteString(" further violations suppressed.\n") + } + t.Fatalf("%s", b.String()) +} + +func isS3Exempt(path string) bool { + clean := filepath.ToSlash(path) + switch { + case strings.HasSuffix(clean, "/network/host"): + return true + case strings.Contains(clean, "/network/host/"): + return true + case strings.HasSuffix(clean, "/network/policy/policy.go"): + return true + case strings.HasSuffix(clean, "/vpcd/vpcd.go"): + return true + case strings.Contains(clean, "/network/invariants/"): + return true + } + return false +} + +// selectorTail returns the trailing selector name (`.Foo`) of a call, or "". +func selectorTail(fun ast.Expr) string { + if sel, ok := fun.(*ast.SelectorExpr); ok { + return sel.Sel.Name + } + return "" +} diff --git a/spinifex/network/invariants/nats_subjects_test.go b/spinifex/network/invariants/nats_subjects_test.go new file mode 100644 index 000000000..030cb5187 --- /dev/null +++ b/spinifex/network/invariants/nats_subjects_test.go @@ -0,0 +1,138 @@ +package invariants + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// TestS7_NATSSubjectsCarryAZPrefix enforces ADR-0006 clause S7. +// +// "Every vpcd-originated NATS publication uses a subject prefixed +// vpc.{azID}. A subscription never processes a message whose subject +// AZ prefix does not match the local node's AZ identifier." +// +// Aspirational: legacy `vpc.` subjects still in use pending cluster-wide +// rename. Test runs audit and t.Skip's with gap report; flip to t.Fatalf once +// the rename ships. +func TestS7_NATSSubjectsCarryAZPrefix(t *testing.T) { + const clause = `ADR-0006 S7: "Every vpcd-originated NATS publication ` + + `uses a subject prefixed vpc.{azID}. A subscription never processes ` + + `a message whose subject AZ prefix does not match the local node's ` + + `AZ identifier."` + + compliant := regexp.MustCompile(`^vpc\.\{`) + subjectLike := regexp.MustCompile(`^vpc\.[a-zA-Z]`) + + roots := []string{ + filepath.Join(repoRoot(t), "spinifex", "network"), + filepath.Join(repoRoot(t), "spinifex", "vpcd"), + filepath.Join(repoRoot(t), "spinifex", "daemon"), + filepath.Join(repoRoot(t), "spinifex", "handlers"), + filepath.Join(repoRoot(t), "spinifex", "testutil"), + filepath.Join(repoRoot(t), "spinifex", "utils"), + } + + type hit struct { + file string + line int + subject string + } + var hits []hit + + for _, root := range roots { + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if isVendoredOrGenerated(d.Name()) { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + if strings.HasSuffix(path, "_test.go") { + return nil + } + if strings.Contains(filepath.ToSlash(path), "/network/invariants/") { + return nil + } + fset := token.NewFileSet() + file, perr := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) + if perr != nil { + t.Fatalf("parse %s: %v", path, perr) + } + ast.Inspect(file, func(n ast.Node) bool { + bl, ok := n.(*ast.BasicLit) + if !ok || bl.Kind != token.STRING { + return true + } + lit, ok := stringLit(bl) + if !ok { + return true + } + if !subjectLike.MatchString(lit) { + return true + } + if compliant.MatchString(lit) { + return true + } + pos := fset.Position(bl.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, lit}) + return true + }) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } + + if len(hits) == 0 { + return + } + + sort.Slice(hits, func(i, j int) bool { + if hits[i].file != hits[j].file { + return hits[i].file < hits[j].file + } + return hits[i].line < hits[j].line + }) + + rootForRel := repoRoot(t) + var b strings.Builder + b.WriteString("S7 gap (aspirational): subject rename pending. ") + b.WriteString("Once the cluster-wide cutover lands, switch t.Skip to ") + b.WriteString("t.Fatalf in nats_subjects_test.go.\n") + b.WriteString(clause) + b.WriteString("\n") + limit := 5 + for i, h := range hits { + if i >= limit { + b.WriteString(" …\n") + break + } + b.WriteString(" ") + b.WriteString(relTo(h.file, rootForRel)) + b.WriteString(":") + b.WriteString(itoa(h.line)) + b.WriteString(": subject ") + b.WriteString(h.subject) + b.WriteString(" lacks {azID} segment\n") + } + if len(hits) > limit { + b.WriteString(" ") + b.WriteString(itoa(len(hits) - limit)) + b.WriteString(" further subjects suppressed.\n") + } + t.Skip(b.String()) +} diff --git a/spinifex/network/invariants/ovn_named_create_test.go b/spinifex/network/invariants/ovn_named_create_test.go new file mode 100644 index 000000000..31a29dea8 --- /dev/null +++ b/spinifex/network/invariants/ovn_named_create_test.go @@ -0,0 +1,130 @@ +package invariants + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "sort" + "strings" + "testing" +) + +// TestS4_OVNNamedCreatesUseEnsure enforces ADR-0006 S4.7. +// +// "OVN logical entities owned by spinifex MUST be unique on +// (table, Name). The libovsdb client enforces this via wait-then-insert +// transactions (EnsureLogicalRouter / EnsureLogicalSwitch / +// EnsurePortGroup). Callers outside network/ovn MUST use the Ensure* +// primitive." +func TestS4_OVNNamedCreatesUseEnsure(t *testing.T) { + const clause = `ADR-0006 S4.7: "OVN logical entities owned by spinifex ` + + `MUST be unique on (table, Name). The libovsdb client enforces this ` + + `via wait-then-insert transactions (EnsureLogicalRouter / ` + + `EnsureLogicalSwitch / EnsurePortGroup). Callers outside network/ovn ` + + `MUST use the Ensure* primitive."` + + forbidden := map[string]string{ + "CreateLogicalRouter": "EnsureLogicalRouter", + "CreateLogicalSwitch": "EnsureLogicalSwitch", + "CreatePortGroup": "EnsurePortGroup", + } + + root := filepath.Join(repoRoot(t), "spinifex") + exemptDirs := map[string]struct{}{ + filepath.Join(root, "network", "ovn"): {}, + filepath.Join(root, "network", "invariants"): {}, + } + + type hit struct { + file string + line int + sym string + replace string + } + var hits []hit + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if isVendoredOrGenerated(d.Name()) { + return filepath.SkipDir + } + for exempt := range exemptDirs { + if path == exempt || strings.HasPrefix(path, exempt+string(filepath.Separator)) { + return filepath.SkipDir + } + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + fset := token.NewFileSet() + file, perr := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) + if perr != nil { + t.Fatalf("parse %s: %v", path, perr) + } + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + tail := selectorTail(call.Fun) + replace, bad := forbidden[tail] + if !bad { + return true + } + pos := fset.Position(call.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, tail, replace}) + return true + }) + return nil + }) + if err != nil { + t.Fatalf("walk: %v", err) + } + + if len(hits) == 0 { + return + } + + sort.Slice(hits, func(i, j int) bool { + if hits[i].file != hits[j].file { + return hits[i].file < hits[j].file + } + return hits[i].line < hits[j].line + }) + + rootForRel := repoRoot(t) + var b strings.Builder + b.WriteString(clause) + b.WriteString("\n") + limit := 8 + for i, h := range hits { + if i >= limit { + b.WriteString(" …\n") + break + } + b.WriteString(" ") + b.WriteString(relTo(h.file, rootForRel)) + b.WriteString(":") + b.WriteString(itoa(h.line)) + b.WriteString(": calls ") + b.WriteString(h.sym) + b.WriteString(" — use ") + b.WriteString(h.replace) + b.WriteString(" instead\n") + } + b.WriteString(" Fix: swap the call site to the Ensure* primitive ") + b.WriteString("(returns canonical row; survives cross-node creation race).\n") + if len(hits) > limit { + b.WriteString(" ") + b.WriteString(itoa(len(hits) - limit)) + b.WriteString(" further violations suppressed.\n") + } + t.Fatalf("%s", b.String()) +} diff --git a/spinifex/network/invariants/policy_no_create_test.go b/spinifex/network/invariants/policy_no_create_test.go new file mode 100644 index 000000000..092c0f7ad --- /dev/null +++ b/spinifex/network/invariants/policy_no_create_test.go @@ -0,0 +1,122 @@ +package invariants + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "sort" + "strings" + "testing" +) + +// TestS5_PolicyNeverCreatesOrDeletes enforces ADR-0006 clause S5. +// +// "L3 never creates or deletes logical objects. L3 only attaches policy +// (ACLs, NAT rules, static routes) to objects that L2 already owns. L3 +// never calls create or delete on logical switches, routers, or ports." +func TestS5_PolicyNeverCreatesOrDeletes(t *testing.T) { + const clause = `ADR-0006 S5: "L3 never creates or deletes logical ` + + `objects. L3 only attaches policy (ACLs, NAT rules, static routes) ` + + `to objects that L2 already owns. L3 never calls create or delete ` + + `on logical switches, routers, or ports."` + + forbidden := map[string]struct{}{ + "CreateLogicalSwitch": {}, + "CreateLogicalRouter": {}, + "CreateLogicalSwitchPort": {}, + "CreateLogicalRouterPort": {}, + "DeleteLogicalSwitch": {}, + "DeleteLogicalRouter": {}, + "DeleteLogicalSwitchPort": {}, + "DeleteLogicalRouterPort": {}, + } + + root := filepath.Join(repoRoot(t), "spinifex", "network", "policy") + type hit struct { + file string + line int + sym string + } + var hits []hit + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if isVendoredOrGenerated(d.Name()) { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + if strings.HasSuffix(path, "_test.go") { + return nil + } + fset := token.NewFileSet() + file, perr := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) + if perr != nil { + t.Fatalf("parse %s: %v", path, perr) + } + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + tail := selectorTail(call.Fun) + if _, bad := forbidden[tail]; !bad { + return true + } + pos := fset.Position(call.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, tail}) + return true + }) + return nil + }) + if err != nil { + t.Fatalf("walk: %v", err) + } + + if len(hits) == 0 { + return + } + + sort.Slice(hits, func(i, j int) bool { + if hits[i].file != hits[j].file { + return hits[i].file < hits[j].file + } + return hits[i].line < hits[j].line + }) + + rootForRel := repoRoot(t) + var b strings.Builder + b.WriteString(clause) + b.WriteString("\n") + limit := 5 + for i, h := range hits { + if i >= limit { + b.WriteString(" …\n") + break + } + b.WriteString(" ") + b.WriteString(relTo(h.file, rootForRel)) + b.WriteString(":") + b.WriteString(itoa(h.line)) + b.WriteString(": L3 calls ") + b.WriteString(h.sym) + b.WriteString("\n") + } + b.WriteString(" Fix: move the create/delete to L2 (topology) and have ") + b.WriteString("L3 attach policy only. L3 must never mutate logical ") + b.WriteString("switch/router/port lifecycle.\n") + if len(hits) > limit { + b.WriteString(" ") + b.WriteString(itoa(len(hits) - limit)) + b.WriteString(" further violations suppressed.\n") + } + t.Fatalf("%s", b.String()) +} diff --git a/spinifex/network/invariants/reconciler_singleton_test.go b/spinifex/network/invariants/reconciler_singleton_test.go new file mode 100644 index 000000000..6db1e01c3 --- /dev/null +++ b/spinifex/network/invariants/reconciler_singleton_test.go @@ -0,0 +1,110 @@ +package invariants + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// TestS9_SingleReconcilerNoRetrofitPasses enforces ADR-0006 clause S9. +// +// "On each leader-gated startup, exactly one intent-actual reconciliation +// pass runs against the NATS KV snapshot and OVN NB DB actual state. No +// additional serial retrofit passes exist." +func TestS9_SingleReconcilerNoRetrofitPasses(t *testing.T) { + const clause = `ADR-0006 S9: "On each leader-gated startup, exactly one ` + + `intent-actual reconciliation pass runs against the NATS KV snapshot ` + + `and OVN NB DB actual state. No additional serial retrofit passes exist."` + + loopRE := regexp.MustCompile(`^Reconcile[A-Za-z]*Loop$`) + passRE := regexp.MustCompile(`^Reconcile[A-Za-z]*Pass\d+$`) + + root := filepath.Join(repoRoot(t), "spinifex", "network") + type hit struct { + file string + line int + sym string + why string + } + var hits []hit + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if isVendoredOrGenerated(d.Name()) { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + if strings.HasSuffix(path, "_test.go") { + return nil + } + fset := token.NewFileSet() + file, perr := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) + if perr != nil { + t.Fatalf("parse %s: %v", path, perr) + } + ast.Inspect(file, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if !ok || fn.Name == nil { + return true + } + name := fn.Name.Name + switch { + case loopRE.MatchString(name): + pos := fset.Position(fn.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, name, "Reconcile…Loop (retrofit pass)"}) + case passRE.MatchString(name): + pos := fset.Position(fn.Pos()) + hits = append(hits, hit{pos.Filename, pos.Line, name, "Reconcile…Pass (serial retrofit)"}) + } + return true + }) + return nil + }) + if err != nil { + t.Fatalf("walk: %v", err) + } + + if len(hits) == 0 { + return + } + + sort.Slice(hits, func(i, j int) bool { + if hits[i].file != hits[j].file { + return hits[i].file < hits[j].file + } + return hits[i].line < hits[j].line + }) + + rootForRel := repoRoot(t) + var b strings.Builder + b.WriteString(clause) + b.WriteString("\n") + for _, h := range hits { + b.WriteString(" ") + b.WriteString(relTo(h.file, rootForRel)) + b.WriteString(":") + b.WriteString(itoa(h.line)) + b.WriteString(": ") + b.WriteString(h.sym) + b.WriteString(" — ") + b.WriteString(h.why) + b.WriteString("\n") + } + b.WriteString(" Fix: collapse this function into the single intent-actual ") + b.WriteString("reconciler in network/reconcile/. Periodic re-runs schedule ") + b.WriteString("the same entrypoint; they do not warrant a new function.\n") + t.Fatalf("%s", b.String()) +} diff --git a/spinifex/network/ovn/client.go b/spinifex/network/ovn/client.go new file mode 100644 index 000000000..3f1dc25c0 --- /dev/null +++ b/spinifex/network/ovn/client.go @@ -0,0 +1,158 @@ +// Package ovn is L1 of the spinifex network stack: the OVN Northbound DB +// client interface and implementations. Higher layers (topology, policy, +// external, federation) must call OVN only through this interface. +package ovn + +import ( + "context" + "errors" + + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" +) + +// ErrNATNotFound is returned when a NAT rule is absent (use errors.Is for idempotency). +var ErrNATNotFound = errors.New("NAT not found") + +// ErrPortGroupNotFound is returned when a port group is absent (use errors.Is for idempotency). +var ErrPortGroupNotFound = errors.New("port group not found") + +// ACLSpec describes an OVN ACL rule for attachment to a port group. +type ACLSpec struct { + Direction string // "to-lport" or "from-lport" + Priority int + Match string + Action string // "allow-related", "drop", "allow", "reject" + Name string + Log bool + Severity string // "alert", "warning", "notice", "info", "debug" +} + +// Client is the OVN Northbound DB abstraction. The vpcd in-tree +// implementation is LiveClient (live.go); tests use network/ovn/mock. +type Client interface { + // Connection lifecycle + Connect(ctx context.Context) error + Close() + Connected() bool + + // Logical Switch (subnet) + CreateLogicalSwitch(ctx context.Context, ls *nbdb.LogicalSwitch) error + // EnsureLogicalSwitch atomically creates a logical switch if no row with + // the same Name already exists, or returns the existing row. Wraps the + // insert in an OVSDB wait-op that asserts "zero rows match Name == ls.Name" + // at commit time; concurrent writers across nodes are serialised by + // ovsdb-server rather than by the local libovsdb cache (which can be + // stale relative to another node's just-committed insert). See + // EnsureLogicalRouter for the rationale. + EnsureLogicalSwitch(ctx context.Context, ls *nbdb.LogicalSwitch) (*nbdb.LogicalSwitch, error) + DeleteLogicalSwitch(ctx context.Context, name string) error + GetLogicalSwitch(ctx context.Context, name string) (*nbdb.LogicalSwitch, error) + ListLogicalSwitches(ctx context.Context) ([]nbdb.LogicalSwitch, error) + + // Logical Switch Port (VM/ENI) + CreateLogicalSwitchPort(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort) error + // CreateLogicalSwitchPortInGroups creates an LSP, adds it to its switch, + // and joins it to the named port groups — all in a single OVSDB + // transaction. Required for SG enforcement: a non-atomic create-then-join + // leaves a window where the LSP exists outside any port group (OVN + // default = unrestricted). portGroupNames may be empty (e.g. router/ + // localnet ports). The per-port-group `_ip4`/`_ip6` Address_Set rows in + // SB are auto-derived by ovn-northd from each port group's port + // addresses; SG-to-SG match expressions resolve against those. + CreateLogicalSwitchPortInGroups(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort, portGroupNames []string) error + DeleteLogicalSwitchPort(ctx context.Context, switchName string, portName string) error + GetLogicalSwitchPort(ctx context.Context, name string) (*nbdb.LogicalSwitchPort, error) + UpdateLogicalSwitchPort(ctx context.Context, lsp *nbdb.LogicalSwitchPort) error + + // Logical Router (VPC router) + CreateLogicalRouter(ctx context.Context, lr *nbdb.LogicalRouter) error + // EnsureLogicalRouter atomically creates a logical router if no row with + // the same Name already exists, or returns the existing row. The insert + // is bundled with an OVSDB wait-op that asserts "zero rows match + // Name == lr.Name" at commit time, so two writers (different goroutines + // or different nodes) that both observe absence cannot both succeed — + // ovsdb-server rejects the second transaction. The returned pointer is + // the canonical row: newly created with the supplied fields, or the + // pre-existing row when a competing writer raced us. + // + // Required because OVN NB has no unique constraint on Name. Without the + // wait-op, the cross-handler / cross-node race between vpc.create and + // vpc.create-subnet's defensive EnsureVPC call produces duplicate routers + // per VPC. + EnsureLogicalRouter(ctx context.Context, lr *nbdb.LogicalRouter) (*nbdb.LogicalRouter, error) + DeleteLogicalRouter(ctx context.Context, name string) error + GetLogicalRouter(ctx context.Context, name string) (*nbdb.LogicalRouter, error) + ListLogicalRouters(ctx context.Context) ([]nbdb.LogicalRouter, error) + // UpdateLogicalRouterExternalIDs rewrites the ExternalIDs map of an + // existing logical router. Used by topology.EnsureVPC to backfill the + // `spinifex:cidr` / `spinifex:vni` keys when the row was first created by + // a defensive EnsureVPC call that did not carry the parent VPC's full + // metadata. The whole map is replaced; callers must pass the merged set. + UpdateLogicalRouterExternalIDs(ctx context.Context, name string, externalIDs map[string]string) error + + // Logical Router Port + CreateLogicalRouterPort(ctx context.Context, routerName string, lrp *nbdb.LogicalRouterPort) error + DeleteLogicalRouterPort(ctx context.Context, routerName string, portName string) error + GetLogicalRouterPort(ctx context.Context, name string) (*nbdb.LogicalRouterPort, error) + UpdateLogicalRouterPort(ctx context.Context, lrp *nbdb.LogicalRouterPort) error + ListLogicalRouterPorts(ctx context.Context) ([]nbdb.LogicalRouterPort, error) + + // DHCP Options + CreateDHCPOptions(ctx context.Context, opts *nbdb.DHCPOptions) (string, error) + DeleteDHCPOptions(ctx context.Context, uuid string) error + FindDHCPOptionsByCIDR(ctx context.Context, cidr string) (*nbdb.DHCPOptions, error) + FindDHCPOptionsByExternalID(ctx context.Context, key, value string) (*nbdb.DHCPOptions, error) + ListDHCPOptions(ctx context.Context) ([]nbdb.DHCPOptions, error) + + // NAT rules + AddNAT(ctx context.Context, routerName string, nat *nbdb.NAT) error + DeleteNAT(ctx context.Context, routerName string, natType, logicalIP string) error + DeleteNATByExternalIP(ctx context.Context, routerName string, natType, externalIP string) error + DeleteAllNATsByExternalIP(ctx context.Context, natType, externalIP string) (int, error) + FindNATByExternalIP(ctx context.Context, natType, externalIP string) (*nbdb.NAT, error) + + // Static routes + AddStaticRoute(ctx context.Context, routerName string, route *nbdb.LogicalRouterStaticRoute) error + DeleteStaticRoute(ctx context.Context, routerName string, ipPrefix string) error + FindStaticRoute(ctx context.Context, routerName, ipPrefix string) (*nbdb.LogicalRouterStaticRoute, error) + + // Port Groups (security group enforcement) + CreatePortGroup(ctx context.Context, name string, ports []string) error + // EnsurePortGroup is the wait-op-protected create-or-get analogue of + // CreatePortGroup. Returns the canonical row. See EnsureLogicalRouter for + // the wait-op pattern and rationale. + EnsurePortGroup(ctx context.Context, name string, ports []string) (*nbdb.PortGroup, error) + DeletePortGroup(ctx context.Context, name string) error + // UpdatePortGroupMemberships applies all port-group joins and leaves for a + // single LSP in one atomic OVSDB transaction. Required by reconcilePortSGs + // so a 5-SG → different-5-SG modify never exposes an intermediate state + // with fewer port groups (which would let the OVN default = unrestricted + // apply for the gap). The per-port-group `_ip4`/`_ip6` Address_Set rows + // in SB are auto-derived by ovn-northd from each port group's port + // addresses; no explicit address-set update is required here. + UpdatePortGroupMemberships(ctx context.Context, lspName string, addPGs, removePGs []string) error + // ListPortGroupsForPort returns the names of every port group whose Ports + // set contains the given LSP. Used by reconcilePortSGs to discover current + // membership before computing the add/remove diff against desired. + ListPortGroupsForPort(ctx context.Context, lspName string) ([]string, error) + // GetPortGroup returns the port group with the given name, or an error if + // it doesn't exist. Used by the reconciler to detect SGs whose port group + // has gone missing in OVN NB. + GetPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) + // ListPortGroups returns every port group in OVN NB. Used by the + // reconciler's orphan-PG scan to detect spinifex-managed port groups + // (`sg_*`) that no longer have a matching SG record in KV. + ListPortGroups(ctx context.Context) ([]nbdb.PortGroup, error) + + // ACLs (attached to port groups). AddACLs creates all rows and links + // them to the port group in one OVSDB transaction — important when a + // single SG can carry up to 60 ingress + 60 egress rules. + AddACLs(ctx context.Context, portGroupName string, specs []ACLSpec) error + ClearACLs(ctx context.Context, portGroupName string) error + + // Gateway Chassis (HA scheduling for gateway router ports) + SetGatewayChassis(ctx context.Context, lrpName string, chassisName string, priority int) error + GetGatewayChassisByName(ctx context.Context, name string) (*nbdb.GatewayChassis, error) + ListGatewayChassis(ctx context.Context) ([]nbdb.GatewayChassis, error) + DeleteGatewayChassis(ctx context.Context, lrpName string, gcUUID string) error +} diff --git a/spinifex/services/vpcd/ovn.go b/spinifex/network/ovn/live.go similarity index 59% rename from spinifex/services/vpcd/ovn.go rename to spinifex/network/ovn/live.go index 78d14c3c5..e23f0e9b5 100644 --- a/spinifex/services/vpcd/ovn.go +++ b/spinifex/network/ovn/live.go @@ -1,34 +1,60 @@ -package vpcd +package ovn import ( "context" - "errors" "fmt" "log/slog" "slices" + "time" - "github.com/mulgadc/spinifex/spinifex/services/vpcd/nbdb" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" "github.com/ovn-kubernetes/libovsdb/client" "github.com/ovn-kubernetes/libovsdb/model" "github.com/ovn-kubernetes/libovsdb/ovsdb" ) -// ErrNATNotFound is returned by DeleteNAT / DeleteNATByExternalIP when the -// rule is already absent. Callers that want idempotent semantics (e.g. the -// vpc.delete-nat NATS handler, which races with vm-cleanup paths that emit -// the same delete) match it with errors.Is and treat as success. -var ErrNATNotFound = errors.New("NAT not found") +// ensureRefetchTimeout bounds the post-conflict refetch wait. 200ms tolerates +// clustered NB raft follower lag under steady-state. +const ensureRefetchTimeout = 200 * time.Millisecond -// transactOps executes a set of OVSDB operations as a single transaction, -// checking both the RPC error and individual operation results. -func (c *LiveOVNClient) transactOps(ctx context.Context, ops []ovsdb.Operation) error { +// ensureRefetchInterval polls at libovsdb's monitor flush cadence. +const ensureRefetchInterval = 20 * time.Millisecond + +// ensureWaitTimeoutMS=0 makes the wait-op fail fast so we fall through to the +// refetch path on conflict. +const ensureWaitTimeoutMS = 0 + +// ensureNamedRowOps returns ops that atomically "insert createObj unless a row +// with this Name exists". OVN NB lacks a unique-Name constraint; the paired +// wait+create lets ovsdb-server serialise concurrent writers, and the +// `until=!=` inverse formulation is required because RFC 7047 §5.2.6 mandates +// a non-empty `rows` member (libovsdb drops `[]` via `omitempty`). +func (c *LiveClient) ensureNamedRowOps(table, name string, createObj model.Model) ([]ovsdb.Operation, error) { + createOps, err := c.client.Create(createObj) + if err != nil { + return nil, fmt.Errorf("create %s ops: %w", table, err) + } + timeout := ensureWaitTimeoutMS + waitOp := ovsdb.Operation{ + Op: ovsdb.OperationWait, + Table: table, + Timeout: &timeout, + Where: []ovsdb.Condition{{Column: "name", Function: ovsdb.ConditionEqual, Value: name}}, + Columns: []string{"name"}, + Until: string(ovsdb.WaitConditionNotEqual), + Rows: []ovsdb.Row{{"name": name}}, + } + return append([]ovsdb.Operation{waitOp}, createOps...), nil +} + +// transactOps runs ops as one transaction and checks per-op results. +func (c *LiveClient) transactOps(ctx context.Context, ops []ovsdb.Operation) error { results, err := c.client.Transact(ctx, ops...) if err != nil { return err } _, err = ovsdb.CheckOperationResults(results, ops) if err != nil { - // Log detailed per-operation errors for debugging for i, r := range results { if r.Error != "" { opTable := "" @@ -42,126 +68,10 @@ func (c *LiveOVNClient) transactOps(ctx context.Context, ops []ovsdb.Operation) return err } -// ACLSpec describes an OVN ACL rule for attachment to a port group. -// Name, Severity, and Meter are optional — set Severity when Log is true. -type ACLSpec struct { - Direction string // "to-lport" or "from-lport" - Priority int - Match string - Action string // "allow-related", "drop", "allow", "reject" - Name string - Log bool - Severity string // "alert", "warning", "notice", "info", "debug" -} - -// OVNClient defines the interface for interacting with the OVN Northbound Database. -// This abstraction allows for mock implementations in tests. -type OVNClient interface { - // Connection lifecycle - Connect(ctx context.Context) error - Close() - Connected() bool - - // Logical Switch (subnet) - CreateLogicalSwitch(ctx context.Context, ls *nbdb.LogicalSwitch) error - DeleteLogicalSwitch(ctx context.Context, name string) error - GetLogicalSwitch(ctx context.Context, name string) (*nbdb.LogicalSwitch, error) - ListLogicalSwitches(ctx context.Context) ([]nbdb.LogicalSwitch, error) - - // Logical Switch Port (VM/ENI) - CreateLogicalSwitchPort(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort) error - // CreateLogicalSwitchPortInGroups creates an LSP, adds it to its switch, - // and joins it to the named port groups — all in a single OVSDB - // transaction. Required for SG enforcement: a non-atomic create-then-join - // leaves a window where the LSP exists outside any port group (OVN - // default = unrestricted). portGroupNames may be empty (e.g. router/ - // localnet ports). The per-port-group `_ip4`/`_ip6` Address_Set rows in - // SB are auto-derived by ovn-northd from each port group's port - // addresses; SG-to-SG match expressions resolve against those. - CreateLogicalSwitchPortInGroups(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort, portGroupNames []string) error - DeleteLogicalSwitchPort(ctx context.Context, switchName string, portName string) error - GetLogicalSwitchPort(ctx context.Context, name string) (*nbdb.LogicalSwitchPort, error) - UpdateLogicalSwitchPort(ctx context.Context, lsp *nbdb.LogicalSwitchPort) error - - // Logical Router (VPC router) - CreateLogicalRouter(ctx context.Context, lr *nbdb.LogicalRouter) error - DeleteLogicalRouter(ctx context.Context, name string) error - GetLogicalRouter(ctx context.Context, name string) (*nbdb.LogicalRouter, error) - ListLogicalRouters(ctx context.Context) ([]nbdb.LogicalRouter, error) - - // Logical Router Port - CreateLogicalRouterPort(ctx context.Context, routerName string, lrp *nbdb.LogicalRouterPort) error - DeleteLogicalRouterPort(ctx context.Context, routerName string, portName string) error - GetLogicalRouterPort(ctx context.Context, name string) (*nbdb.LogicalRouterPort, error) - UpdateLogicalRouterPort(ctx context.Context, lrp *nbdb.LogicalRouterPort) error - ListLogicalRouterPorts(ctx context.Context) ([]nbdb.LogicalRouterPort, error) - - // DHCP Options - CreateDHCPOptions(ctx context.Context, opts *nbdb.DHCPOptions) (string, error) - DeleteDHCPOptions(ctx context.Context, uuid string) error - FindDHCPOptionsByCIDR(ctx context.Context, cidr string) (*nbdb.DHCPOptions, error) - FindDHCPOptionsByExternalID(ctx context.Context, key, value string) (*nbdb.DHCPOptions, error) - ListDHCPOptions(ctx context.Context) ([]nbdb.DHCPOptions, error) - - // NAT rules - AddNAT(ctx context.Context, routerName string, nat *nbdb.NAT) error - DeleteNAT(ctx context.Context, routerName string, natType, logicalIP string) error - DeleteNATByExternalIP(ctx context.Context, routerName string, natType, externalIP string) error - DeleteAllNATsByExternalIP(ctx context.Context, natType, externalIP string) (int, error) - FindNATByExternalIP(ctx context.Context, natType, externalIP string) (*nbdb.NAT, error) - - // Static routes - AddStaticRoute(ctx context.Context, routerName string, route *nbdb.LogicalRouterStaticRoute) error - DeleteStaticRoute(ctx context.Context, routerName string, ipPrefix string) error - FindStaticRoute(ctx context.Context, routerName, ipPrefix string) (*nbdb.LogicalRouterStaticRoute, error) - - // Port Groups (security group enforcement) - CreatePortGroup(ctx context.Context, name string, ports []string) error - DeletePortGroup(ctx context.Context, name string) error - // UpdatePortGroupMemberships applies all port-group joins and leaves for a - // single LSP in one atomic OVSDB transaction. Required by reconcilePortSGs - // so a 5-SG → different-5-SG modify never exposes an intermediate state - // with fewer port groups (which would let the OVN default = unrestricted - // apply for the gap). The per-port-group `_ip4`/`_ip6` Address_Set rows - // in SB are auto-derived by ovn-northd from each port group's port - // addresses; no explicit address-set update is required here. - UpdatePortGroupMemberships(ctx context.Context, lspName string, addPGs, removePGs []string) error - // ListPortGroupsForPort returns the names of every port group whose Ports - // set contains the given LSP. Used by reconcilePortSGs to discover current - // membership before computing the add/remove diff against desired. - ListPortGroupsForPort(ctx context.Context, lspName string) ([]string, error) - // GetPortGroup returns the port group with the given name, or an error if - // it doesn't exist. Used by the reconciler to detect SGs whose port group - // has gone missing in OVN NB. - GetPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) - // ListPortGroups returns every port group in OVN NB. Used by the - // reconciler's orphan-PG scan to detect spinifex-managed port groups - // (`sg_*`) that no longer have a matching SG record in KV. - ListPortGroups(ctx context.Context) ([]nbdb.PortGroup, error) - - // ACLs (attached to port groups). AddACLs creates all rows and links - // them to the port group in one OVSDB transaction — important when a - // single SG can carry up to 60 ingress + 60 egress rules. - AddACLs(ctx context.Context, portGroupName string, specs []ACLSpec) error - ClearACLs(ctx context.Context, portGroupName string) error - - // Gateway Chassis (HA scheduling for gateway router ports) - SetGatewayChassis(ctx context.Context, lrpName string, chassisName string, priority int) error - GetGatewayChassisByName(ctx context.Context, name string) (*nbdb.GatewayChassis, error) - ListGatewayChassis(ctx context.Context) ([]nbdb.GatewayChassis, error) - DeleteGatewayChassis(ctx context.Context, lrpName string, gcUUID string) error -} - -// namedUUID generates a valid OVSDB named-uuid from a prefix and name. -// OVSDB named-uuids must match [_a-zA-Z][_a-zA-Z0-9]* — anything outside -// that class (spaces, '@', '=', '&', '-', '.', '/', etc.) is replaced with -// an underscore. Callers always supply a prefix that begins with a letter, -// so the leading-character constraint is satisfied implicitly. -// -// Without exhaustive sanitisation, ACL named-uuids that embed a Match -// expression like "outport == @pg-sg-XYZ && ip4" produce strings with -// spaces and '@' that OVSDB rejects with "Type mismatch for member -// 'uuid-name'", which silently breaks every default-SG ACL transaction. +// namedUUID builds a valid OVSDB named-uuid by replacing every non +// [_a-zA-Z0-9] byte with `_`. Required for ACL UUIDs that embed Match +// expressions like "outport == @pg-sg-XYZ && ip4"; without this, OVSDB +// rejects them with "Type mismatch for member 'uuid-name'". func namedUUID(prefix, name string) string { s := prefix + name result := make([]byte, len(s)) @@ -180,21 +90,21 @@ func namedUUID(prefix, name string) string { return string(result) } -// LiveOVNClient implements OVNClient using libovsdb against a real OVN NB DB. -type LiveOVNClient struct { +// LiveClient implements Client using libovsdb against a real OVN NB DB. +type LiveClient struct { endpoint string client client.Client } -var _ OVNClient = (*LiveOVNClient)(nil) +var _ Client = (*LiveClient)(nil) -// NewLiveOVNClient creates a new LiveOVNClient targeting the given OVN NB DB endpoint. -// The endpoint should be in the format "tcp:host:port" or "unix:/path/to/socket". -func NewLiveOVNClient(endpoint string) *LiveOVNClient { - return &LiveOVNClient{endpoint: endpoint} +// NewLiveClient creates a LiveClient targeting an OVN NB DB endpoint +// ("tcp:host:port" or "unix:/path/to/socket"). +func NewLiveClient(endpoint string) *LiveClient { + return &LiveClient{endpoint: endpoint} } -func (c *LiveOVNClient) Connect(ctx context.Context) error { +func (c *LiveClient) Connect(ctx context.Context) error { dbModel, err := nbdb.FullDatabaseModel() if err != nil { return fmt.Errorf("failed to create database model: %w", err) @@ -220,18 +130,18 @@ func (c *LiveOVNClient) Connect(ctx context.Context) error { return nil } -func (c *LiveOVNClient) Close() { +func (c *LiveClient) Close() { if c.client != nil { c.client.Close() slog.Info("Disconnected from OVN NB DB") } } -func (c *LiveOVNClient) Connected() bool { +func (c *LiveClient) Connected() bool { return c.client != nil } -func (c *LiveOVNClient) CreateLogicalSwitch(ctx context.Context, ls *nbdb.LogicalSwitch) error { +func (c *LiveClient) CreateLogicalSwitch(ctx context.Context, ls *nbdb.LogicalSwitch) error { ops, err := c.client.Create(ls) if err != nil { return fmt.Errorf("create logical switch ops: %w", err) @@ -243,7 +153,49 @@ func (c *LiveOVNClient) CreateLogicalSwitch(ctx context.Context, ls *nbdb.Logica return nil } -func (c *LiveOVNClient) DeleteLogicalSwitch(ctx context.Context, name string) error { +func (c *LiveClient) EnsureLogicalSwitch(ctx context.Context, ls *nbdb.LogicalSwitch) (*nbdb.LogicalSwitch, error) { + if existing, err := c.GetLogicalSwitch(ctx, ls.Name); err == nil { + return existing, nil + } + if ls.UUID == "" { + ls.UUID = namedUUID("ls_", ls.Name) + } + ops, err := c.ensureNamedRowOps("Logical_Switch", ls.Name, ls) + if err != nil { + return nil, err + } + if err := c.transactOps(ctx, ops); err != nil { + if existing, refErr := c.waitForCachedSwitch(ctx, ls.Name); refErr == nil { + slog.Info("ovn: EnsureLogicalSwitch lost race, reusing existing", + "name", ls.Name, "existing_uuid", existing.UUID) + return existing, nil + } + return nil, fmt.Errorf("ensure logical switch %q: %w", ls.Name, err) + } + return ls, nil +} + +func (c *LiveClient) waitForCachedSwitch(ctx context.Context, name string) (*nbdb.LogicalSwitch, error) { + deadline := time.Now().Add(ensureRefetchTimeout) + var lastErr error + for { + v, err := c.GetLogicalSwitch(ctx, name) + if err == nil { + return v, nil + } + lastErr = err + if time.Now().After(deadline) { + return nil, lastErr + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(ensureRefetchInterval): + } + } +} + +func (c *LiveClient) DeleteLogicalSwitch(ctx context.Context, name string) error { ls, err := c.GetLogicalSwitch(ctx, name) if err != nil { return fmt.Errorf("delete logical switch lookup: %w", err) @@ -259,7 +211,7 @@ func (c *LiveOVNClient) DeleteLogicalSwitch(ctx context.Context, name string) er return nil } -func (c *LiveOVNClient) GetLogicalSwitch(ctx context.Context, name string) (*nbdb.LogicalSwitch, error) { +func (c *LiveClient) GetLogicalSwitch(ctx context.Context, name string) (*nbdb.LogicalSwitch, error) { var switches []nbdb.LogicalSwitch err := c.client.WhereCache(func(ls *nbdb.LogicalSwitch) bool { return ls.Name == name @@ -273,7 +225,7 @@ func (c *LiveOVNClient) GetLogicalSwitch(ctx context.Context, name string) (*nbd return &switches[0], nil } -func (c *LiveOVNClient) ListLogicalSwitches(ctx context.Context) ([]nbdb.LogicalSwitch, error) { +func (c *LiveClient) ListLogicalSwitches(ctx context.Context) ([]nbdb.LogicalSwitch, error) { var switches []nbdb.LogicalSwitch err := c.client.List(ctx, &switches) if err != nil { @@ -282,25 +234,22 @@ func (c *LiveOVNClient) ListLogicalSwitches(ctx context.Context) ([]nbdb.Logical return switches, nil } -func (c *LiveOVNClient) CreateLogicalSwitchPort(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort) error { - // Set a named UUID so the port can be referenced in the same transaction +func (c *LiveClient) CreateLogicalSwitchPort(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort) error { + // Named UUID lets the same transaction reference the port. if lsp.UUID == "" { lsp.UUID = namedUUID("lsp_", lsp.Name) } - // Create the port createOps, err := c.client.Create(lsp) if err != nil { return fmt.Errorf("create logical switch port ops: %w", err) } - // Look up the switch to get its UUID for the Where clause ls, err := c.GetLogicalSwitch(ctx, switchName) if err != nil { return fmt.Errorf("get logical switch for port add: %w", err) } - // Add the port to the switch's ports set (uses named UUID from Create) mutateOps, err := c.client.Where(ls).Mutate(ls, model.Mutation{ Field: &ls.Ports, Mutator: "insert", @@ -318,14 +267,11 @@ func (c *LiveOVNClient) CreateLogicalSwitchPort(ctx context.Context, switchName return nil } -// CreateLogicalSwitchPortInGroups bundles LSP create + switch ports mutate + -// per-port-group ports mutates into one transaction. Within the transaction -// the LSP's named UUID resolves consistently across all ops, so the port -// group mutates can reference the not-yet-committed LSP without a cache -// lookup. The per-port-group `_ip4`/`_ip6` Address_Set rows in SB are -// auto-derived by ovn-northd from each port group's port addresses, so vpcd -// must not write them explicitly — see provisionSG for details. -func (c *LiveOVNClient) CreateLogicalSwitchPortInGroups(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort, portGroupNames []string) error { +// CreateLogicalSwitchPortInGroups bundles LSP create, switch-ports mutate, and +// per-port-group ports mutates into one transaction so the LSP's named UUID +// resolves across all ops. The per-PG `_ip4`/`_ip6` Address_Set rows are +// auto-derived by ovn-northd; vpcd must not write them (see provisionSG). +func (c *LiveClient) CreateLogicalSwitchPortInGroups(ctx context.Context, switchName string, lsp *nbdb.LogicalSwitchPort, portGroupNames []string) error { if lsp.UUID == "" { lsp.UUID = namedUUID("lsp_", lsp.Name) } @@ -373,20 +319,17 @@ func (c *LiveOVNClient) CreateLogicalSwitchPortInGroups(ctx context.Context, swi return nil } -func (c *LiveOVNClient) DeleteLogicalSwitchPort(ctx context.Context, switchName string, portName string) error { - // Look up the port to get its UUID +func (c *LiveClient) DeleteLogicalSwitchPort(ctx context.Context, switchName string, portName string) error { lsp, err := c.GetLogicalSwitchPort(ctx, portName) if err != nil { return fmt.Errorf("get logical switch port for delete: %w", err) } - // Look up the switch to get its UUID ls, err := c.GetLogicalSwitch(ctx, switchName) if err != nil { return fmt.Errorf("get logical switch for port delete: %w", err) } - // Remove the port from the switch's ports set mutateOps, err := c.client.Where(ls).Mutate(ls, model.Mutation{ Field: &ls.Ports, Mutator: "delete", @@ -396,7 +339,6 @@ func (c *LiveOVNClient) DeleteLogicalSwitchPort(ctx context.Context, switchName return fmt.Errorf("mutate logical switch ports ops: %w", err) } - // Delete the port deleteOps, err := c.client.Where(lsp).Delete() if err != nil { return fmt.Errorf("delete logical switch port ops: %w", err) @@ -410,7 +352,7 @@ func (c *LiveOVNClient) DeleteLogicalSwitchPort(ctx context.Context, switchName return nil } -func (c *LiveOVNClient) GetLogicalSwitchPort(ctx context.Context, name string) (*nbdb.LogicalSwitchPort, error) { +func (c *LiveClient) GetLogicalSwitchPort(ctx context.Context, name string) (*nbdb.LogicalSwitchPort, error) { var ports []nbdb.LogicalSwitchPort err := c.client.WhereCache(func(lsp *nbdb.LogicalSwitchPort) bool { return lsp.Name == name @@ -424,8 +366,7 @@ func (c *LiveOVNClient) GetLogicalSwitchPort(ctx context.Context, name string) ( return &ports[0], nil } -func (c *LiveOVNClient) UpdateLogicalSwitchPort(ctx context.Context, lsp *nbdb.LogicalSwitchPort) error { - // Ensure we have the UUID for the Where clause +func (c *LiveClient) UpdateLogicalSwitchPort(ctx context.Context, lsp *nbdb.LogicalSwitchPort) error { if lsp.UUID == "" { existing, err := c.GetLogicalSwitchPort(ctx, lsp.Name) if err != nil { @@ -444,7 +385,7 @@ func (c *LiveOVNClient) UpdateLogicalSwitchPort(ctx context.Context, lsp *nbdb.L return nil } -func (c *LiveOVNClient) CreateLogicalRouter(ctx context.Context, lr *nbdb.LogicalRouter) error { +func (c *LiveClient) CreateLogicalRouter(ctx context.Context, lr *nbdb.LogicalRouter) error { ops, err := c.client.Create(lr) if err != nil { return fmt.Errorf("create logical router ops: %w", err) @@ -456,7 +397,65 @@ func (c *LiveOVNClient) CreateLogicalRouter(ctx context.Context, lr *nbdb.Logica return nil } -func (c *LiveOVNClient) DeleteLogicalRouter(ctx context.Context, name string) error { +func (c *LiveClient) EnsureLogicalRouter(ctx context.Context, lr *nbdb.LogicalRouter) (*nbdb.LogicalRouter, error) { + if existing, err := c.GetLogicalRouter(ctx, lr.Name); err == nil { + return existing, nil + } + if lr.UUID == "" { + lr.UUID = namedUUID("lr_", lr.Name) + } + ops, err := c.ensureNamedRowOps("Logical_Router", lr.Name, lr) + if err != nil { + return nil, err + } + if err := c.transactOps(ctx, ops); err != nil { + if existing, refErr := c.waitForCachedRouter(ctx, lr.Name); refErr == nil { + slog.Info("ovn: EnsureLogicalRouter lost race, reusing existing", + "name", lr.Name, "existing_uuid", existing.UUID) + return existing, nil + } + return nil, fmt.Errorf("ensure logical router %q: %w", lr.Name, err) + } + return lr, nil +} + +func (c *LiveClient) UpdateLogicalRouterExternalIDs(ctx context.Context, name string, externalIDs map[string]string) error { + lr, err := c.GetLogicalRouter(ctx, name) + if err != nil { + return fmt.Errorf("get logical router for external_ids update: %w", err) + } + lr.ExternalIDs = externalIDs + ops, err := c.client.Where(lr).Update(lr, &lr.ExternalIDs) + if err != nil { + return fmt.Errorf("update logical router external_ids ops: %w", err) + } + if err := c.transactOps(ctx, ops); err != nil { + return fmt.Errorf("update logical router external_ids transact: %w", err) + } + return nil +} + +func (c *LiveClient) waitForCachedRouter(ctx context.Context, name string) (*nbdb.LogicalRouter, error) { + deadline := time.Now().Add(ensureRefetchTimeout) + var lastErr error + for { + v, err := c.GetLogicalRouter(ctx, name) + if err == nil { + return v, nil + } + lastErr = err + if time.Now().After(deadline) { + return nil, lastErr + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(ensureRefetchInterval): + } + } +} + +func (c *LiveClient) DeleteLogicalRouter(ctx context.Context, name string) error { lr, err := c.GetLogicalRouter(ctx, name) if err != nil { return fmt.Errorf("delete logical router lookup: %w", err) @@ -472,7 +471,7 @@ func (c *LiveOVNClient) DeleteLogicalRouter(ctx context.Context, name string) er return nil } -func (c *LiveOVNClient) GetLogicalRouter(ctx context.Context, name string) (*nbdb.LogicalRouter, error) { +func (c *LiveClient) GetLogicalRouter(ctx context.Context, name string) (*nbdb.LogicalRouter, error) { var routers []nbdb.LogicalRouter err := c.client.WhereCache(func(lr *nbdb.LogicalRouter) bool { return lr.Name == name @@ -486,7 +485,7 @@ func (c *LiveOVNClient) GetLogicalRouter(ctx context.Context, name string) (*nbd return &routers[0], nil } -func (c *LiveOVNClient) ListLogicalRouters(ctx context.Context) ([]nbdb.LogicalRouter, error) { +func (c *LiveClient) ListLogicalRouters(ctx context.Context) ([]nbdb.LogicalRouter, error) { var routers []nbdb.LogicalRouter err := c.client.List(ctx, &routers) if err != nil { @@ -495,8 +494,8 @@ func (c *LiveOVNClient) ListLogicalRouters(ctx context.Context) ([]nbdb.LogicalR return routers, nil } -func (c *LiveOVNClient) CreateLogicalRouterPort(ctx context.Context, routerName string, lrp *nbdb.LogicalRouterPort) error { - // Set a named UUID so the port can be referenced in the same transaction +func (c *LiveClient) CreateLogicalRouterPort(ctx context.Context, routerName string, lrp *nbdb.LogicalRouterPort) error { + // Named UUID lets the same transaction reference the port. if lrp.UUID == "" { lrp.UUID = namedUUID("lrp_", lrp.Name) } @@ -506,7 +505,6 @@ func (c *LiveOVNClient) CreateLogicalRouterPort(ctx context.Context, routerName return fmt.Errorf("create logical router port ops: %w", err) } - // Look up the router to get its UUID for the Where clause lr, err := c.GetLogicalRouter(ctx, routerName) if err != nil { return fmt.Errorf("get logical router for port add: %w", err) @@ -529,14 +527,12 @@ func (c *LiveOVNClient) CreateLogicalRouterPort(ctx context.Context, routerName return nil } -func (c *LiveOVNClient) DeleteLogicalRouterPort(ctx context.Context, routerName string, portName string) error { - // Look up the port to get its UUID +func (c *LiveClient) DeleteLogicalRouterPort(ctx context.Context, routerName string, portName string) error { lrp, err := c.GetLogicalRouterPort(ctx, portName) if err != nil { return fmt.Errorf("get logical router port for delete: %w", err) } - // Look up the router to get its UUID lr, err := c.GetLogicalRouter(ctx, routerName) if err != nil { return fmt.Errorf("get logical router for port delete: %w", err) @@ -564,7 +560,7 @@ func (c *LiveOVNClient) DeleteLogicalRouterPort(ctx context.Context, routerName return nil } -func (c *LiveOVNClient) GetLogicalRouterPort(ctx context.Context, name string) (*nbdb.LogicalRouterPort, error) { +func (c *LiveClient) GetLogicalRouterPort(ctx context.Context, name string) (*nbdb.LogicalRouterPort, error) { var ports []nbdb.LogicalRouterPort err := c.client.WhereCache(func(lrp *nbdb.LogicalRouterPort) bool { return lrp.Name == name @@ -579,10 +575,7 @@ func (c *LiveOVNClient) GetLogicalRouterPort(ctx context.Context, name string) ( } // UpdateLogicalRouterPort rewrites mutable columns on an existing LRP. -// Used by ensureGatewayPortNetworks to retrofit the link-local Networks -// CIDR onto gateway ports created by older code that used pool IPs -// (mulga-siv-26 D8). Mirrors UpdateLogicalSwitchPort. -func (c *LiveOVNClient) UpdateLogicalRouterPort(ctx context.Context, lrp *nbdb.LogicalRouterPort) error { +func (c *LiveClient) UpdateLogicalRouterPort(ctx context.Context, lrp *nbdb.LogicalRouterPort) error { if lrp.UUID == "" { existing, err := c.GetLogicalRouterPort(ctx, lrp.Name) if err != nil { @@ -600,12 +593,10 @@ func (c *LiveOVNClient) UpdateLogicalRouterPort(ctx context.Context, lrp *nbdb.L return nil } -// SetGatewayChassis binds a chassis to an LRP for HA gateway scheduling. Read- -// then-decide so it tolerates re-runs of reconcile/IGW-attach: absent → create -// + LRP-mutate; present + same priority → no-op; present + different priority -// → mutate priority on the existing row. Required for the reconcile-time -// rebind step that recovers from chassis_name drift (mulga-999). -func (c *LiveOVNClient) SetGatewayChassis(ctx context.Context, lrpName string, chassisName string, priority int) error { +// SetGatewayChassis binds a chassis to an LRP for HA gateway scheduling. +// Read-then-decide so reconcile/IGW-attach can re-run safely: absent → +// create + LRP-mutate; same priority → no-op; differing priority → mutate. +func (c *LiveClient) SetGatewayChassis(ctx context.Context, lrpName string, chassisName string, priority int) error { gcName := lrpName + "-" + chassisName existing, err := c.GetGatewayChassisByName(ctx, gcName) if err != nil { @@ -653,7 +644,7 @@ func (c *LiveOVNClient) SetGatewayChassis(ctx context.Context, lrpName string, c return nil } -func (c *LiveOVNClient) updateGatewayChassisPriority(ctx context.Context, gc *nbdb.GatewayChassis, priority int) error { +func (c *LiveClient) updateGatewayChassisPriority(ctx context.Context, gc *nbdb.GatewayChassis, priority int) error { gc.Priority = priority ops, err := c.client.Where(gc).Update(gc, &gc.Priority) if err != nil { @@ -666,10 +657,8 @@ func (c *LiveOVNClient) updateGatewayChassisPriority(ctx context.Context, gc *nb } // GetGatewayChassisByName looks up a Gateway_Chassis row by its `name` column -// (the deterministic "lrpName-chassisName" form). Returns (nil, nil) when no -// row matches — matches the cache-lookup convention used by GetLogicalSwitch -// et al. -func (c *LiveOVNClient) GetGatewayChassisByName(ctx context.Context, name string) (*nbdb.GatewayChassis, error) { +// (the "lrpName-chassisName" form). Returns (nil, nil) when no row matches. +func (c *LiveClient) GetGatewayChassisByName(ctx context.Context, name string) (*nbdb.GatewayChassis, error) { var rows []nbdb.GatewayChassis err := c.client.WhereCache(func(gc *nbdb.GatewayChassis) bool { return gc.Name == name @@ -683,11 +672,9 @@ func (c *LiveOVNClient) GetGatewayChassisByName(ctx context.Context, name string return &rows[0], nil } -// ListGatewayChassis returns every Gateway_Chassis row. The reconcile loop -// uses this to find rows referencing chassis names that no longer exist in -// the SBDB (e.g. because the OVS system-id changed across a reboot — -// mulga-999). -func (c *LiveOVNClient) ListGatewayChassis(ctx context.Context) ([]nbdb.GatewayChassis, error) { +// ListGatewayChassis returns every Gateway_Chassis row; reconcile uses this +// to find rows whose chassis_name no longer exists in SBDB. +func (c *LiveClient) ListGatewayChassis(ctx context.Context) ([]nbdb.GatewayChassis, error) { var rows []nbdb.GatewayChassis if err := c.client.List(ctx, &rows); err != nil { return nil, fmt.Errorf("list gateway_chassis: %w", err) @@ -695,10 +682,9 @@ func (c *LiveOVNClient) ListGatewayChassis(ctx context.Context) ([]nbdb.GatewayC return rows, nil } -// DeleteGatewayChassis removes a Gateway_Chassis row and detaches it from the -// owning LRP in one transaction. lrpName is required so the mutation targets -// the correct router port; gcUUID identifies the row to remove. -func (c *LiveOVNClient) DeleteGatewayChassis(ctx context.Context, lrpName string, gcUUID string) error { +// DeleteGatewayChassis removes a Gateway_Chassis row and detaches it from +// the owning LRP in one transaction. +func (c *LiveClient) DeleteGatewayChassis(ctx context.Context, lrpName string, gcUUID string) error { lrp, err := c.GetLogicalRouterPort(ctx, lrpName) if err != nil { return fmt.Errorf("get logical router port for gateway chassis delete: %w", err) @@ -726,10 +712,8 @@ func (c *LiveOVNClient) DeleteGatewayChassis(ctx context.Context, lrpName string return nil } -// ListLogicalRouterPorts returns every LRP across all routers. Used by the -// reconcile-time gateway-chassis rebind to find every LRP tagged -// external_ids:spinifex:role=gateway (mulga-999). -func (c *LiveOVNClient) ListLogicalRouterPorts(ctx context.Context) ([]nbdb.LogicalRouterPort, error) { +// ListLogicalRouterPorts returns every LRP across all routers. +func (c *LiveClient) ListLogicalRouterPorts(ctx context.Context) ([]nbdb.LogicalRouterPort, error) { var rows []nbdb.LogicalRouterPort if err := c.client.List(ctx, &rows); err != nil { return nil, fmt.Errorf("list logical_router_port: %w", err) @@ -737,7 +721,7 @@ func (c *LiveOVNClient) ListLogicalRouterPorts(ctx context.Context) ([]nbdb.Logi return rows, nil } -func (c *LiveOVNClient) CreateDHCPOptions(ctx context.Context, opts *nbdb.DHCPOptions) (string, error) { +func (c *LiveClient) CreateDHCPOptions(ctx context.Context, opts *nbdb.DHCPOptions) (string, error) { ops, err := c.client.Create(opts) if err != nil { return "", fmt.Errorf("create DHCP options ops: %w", err) @@ -755,7 +739,7 @@ func (c *LiveOVNClient) CreateDHCPOptions(ctx context.Context, opts *nbdb.DHCPOp return "", nil } -func (c *LiveOVNClient) DeleteDHCPOptions(ctx context.Context, uuid string) error { +func (c *LiveClient) DeleteDHCPOptions(ctx context.Context, uuid string) error { opts := &nbdb.DHCPOptions{UUID: uuid} ops, err := c.client.Where(opts).Delete() if err != nil { @@ -768,7 +752,7 @@ func (c *LiveOVNClient) DeleteDHCPOptions(ctx context.Context, uuid string) erro return nil } -func (c *LiveOVNClient) FindDHCPOptionsByCIDR(ctx context.Context, cidr string) (*nbdb.DHCPOptions, error) { +func (c *LiveClient) FindDHCPOptionsByCIDR(ctx context.Context, cidr string) (*nbdb.DHCPOptions, error) { var options []nbdb.DHCPOptions err := c.client.WhereCache(func(o *nbdb.DHCPOptions) bool { return o.CIDR == cidr @@ -782,7 +766,7 @@ func (c *LiveOVNClient) FindDHCPOptionsByCIDR(ctx context.Context, cidr string) return &options[0], nil } -func (c *LiveOVNClient) FindDHCPOptionsByExternalID(ctx context.Context, key, value string) (*nbdb.DHCPOptions, error) { +func (c *LiveClient) FindDHCPOptionsByExternalID(ctx context.Context, key, value string) (*nbdb.DHCPOptions, error) { var options []nbdb.DHCPOptions err := c.client.WhereCache(func(o *nbdb.DHCPOptions) bool { return o.ExternalIDs[key] == value @@ -796,7 +780,7 @@ func (c *LiveOVNClient) FindDHCPOptionsByExternalID(ctx context.Context, key, va return &options[0], nil } -func (c *LiveOVNClient) ListDHCPOptions(ctx context.Context) ([]nbdb.DHCPOptions, error) { +func (c *LiveClient) ListDHCPOptions(ctx context.Context) ([]nbdb.DHCPOptions, error) { var options []nbdb.DHCPOptions err := c.client.List(ctx, &options) if err != nil { @@ -805,8 +789,7 @@ func (c *LiveOVNClient) ListDHCPOptions(ctx context.Context) ([]nbdb.DHCPOptions return options, nil } -func (c *LiveOVNClient) AddNAT(ctx context.Context, routerName string, nat *nbdb.NAT) error { - // Set a named UUID so the NAT can be referenced in the same transaction +func (c *LiveClient) AddNAT(ctx context.Context, routerName string, nat *nbdb.NAT) error { if nat.UUID == "" { nat.UUID = namedUUID("nat_", nat.Type+"_"+nat.LogicalIP) } @@ -838,25 +821,35 @@ func (c *LiveOVNClient) AddNAT(ctx context.Context, routerName string, nat *nbdb return nil } -func (c *LiveOVNClient) DeleteNAT(ctx context.Context, routerName string, natType, logicalIP string) error { - // Find the NAT entry by type and logical IP +// DeleteNAT removes a NAT rule matching (natType, logicalIP) from routerName. +// Lookup is scoped to routerName.NAT — rules owned by other routers with the +// same logical IP are left alone. Required because AWS subnet CIDRs repeat +// across VPCs, so (natType, logicalIP) is not globally unique. +func (c *LiveClient) DeleteNAT(ctx context.Context, routerName string, natType, logicalIP string) error { + lr, err := c.GetLogicalRouter(ctx, routerName) + if err != nil { + return fmt.Errorf("get logical router for NAT delete: %w", err) + } + routerNATs := make(map[string]struct{}, len(lr.NAT)) + for _, uuid := range lr.NAT { + routerNATs[uuid] = struct{}{} + } + var nats []nbdb.NAT - err := c.client.WhereCache(func(n *nbdb.NAT) bool { + err = c.client.WhereCache(func(n *nbdb.NAT) bool { + if _, ok := routerNATs[n.UUID]; !ok { + return false + } return n.Type == natType && n.LogicalIP == logicalIP }).List(ctx, &nats) if err != nil { return fmt.Errorf("find NAT: %w", err) } if len(nats) == 0 { - return fmt.Errorf("NAT %s %s: %w", natType, logicalIP, ErrNATNotFound) + return fmt.Errorf("NAT %s %s on %s: %w", natType, logicalIP, routerName, ErrNATNotFound) } nat := &nats[0] - lr, err := c.GetLogicalRouter(ctx, routerName) - if err != nil { - return fmt.Errorf("get logical router for NAT delete: %w", err) - } - mutateOps, err := c.client.Where(lr).Mutate(lr, model.Mutation{ Field: &lr.NAT, Mutator: "delete", @@ -879,23 +872,33 @@ func (c *LiveOVNClient) DeleteNAT(ctx context.Context, routerName string, natTyp return nil } -// DeleteNATByExternalIP removes a NAT rule matching the given external IP. -// Returns an error if no matching rule is found (callers can ignore this). -func (c *LiveOVNClient) DeleteNATByExternalIP(ctx context.Context, routerName string, natType, externalIP string) error { +// DeleteNATByExternalIP removes a NAT rule matching the given external IP from +// routerName only. Lookup is scoped to routerName.NAT — rules on other routers +// with the same external IP are left alone (use DeleteAllNATsByExternalIP for +// cross-router cleanup). Returns ErrNATNotFound if no matching rule is found +// on this router (callers can ignore this). +func (c *LiveClient) DeleteNATByExternalIP(ctx context.Context, routerName string, natType, externalIP string) error { + lr, err := c.GetLogicalRouter(ctx, routerName) + if err != nil { + return fmt.Errorf("get logical router for NAT delete: %w", err) + } + routerNATs := make(map[string]struct{}, len(lr.NAT)) + for _, uuid := range lr.NAT { + routerNATs[uuid] = struct{}{} + } + var nats []nbdb.NAT - err := c.client.WhereCache(func(n *nbdb.NAT) bool { + err = c.client.WhereCache(func(n *nbdb.NAT) bool { + if _, ok := routerNATs[n.UUID]; !ok { + return false + } return n.Type == natType && n.ExternalIP == externalIP }).List(ctx, &nats) if err != nil { return fmt.Errorf("find NAT by external IP: %w", err) } if len(nats) == 0 { - return fmt.Errorf("NAT %s external_ip=%s: %w", natType, externalIP, ErrNATNotFound) - } - - lr, err := c.GetLogicalRouter(ctx, routerName) - if err != nil { - return fmt.Errorf("get logical router for NAT delete: %w", err) + return fmt.Errorf("NAT %s external_ip=%s on %s: %w", natType, externalIP, routerName, ErrNATNotFound) } var allOps []ovsdb.Operation @@ -923,11 +926,11 @@ func (c *LiveOVNClient) DeleteNATByExternalIP(ctx context.Context, routerName st return nil } -// DeleteAllNATsByExternalIP removes all NAT rules matching the given external -// IP from every router that references them. This handles cross-VPC stale NAT -// rules that remain when vpc.delete-nat (fire-and-forget) hasn't been processed -// before an IP is reused by a different VPC. Returns the number of rules deleted. -func (c *LiveOVNClient) DeleteAllNATsByExternalIP(ctx context.Context, natType, externalIP string) (int, error) { +// DeleteAllNATsByExternalIP removes NAT rules matching externalIP from every +// router that references them, handling cross-VPC stale rules left when +// vpc.delete-nat hasn't been processed before the IP is reused. Returns the +// number deleted. +func (c *LiveClient) DeleteAllNATsByExternalIP(ctx context.Context, natType, externalIP string) (int, error) { var nats []nbdb.NAT if err := c.client.WhereCache(func(n *nbdb.NAT) bool { return n.Type == natType && n.ExternalIP == externalIP @@ -948,7 +951,6 @@ func (c *LiveOVNClient) DeleteAllNATsByExternalIP(ctx context.Context, natType, staleUUIDs[n.UUID] = struct{}{} } - // Build a single transaction: remove NAT refs from all routers, then delete NAT rows. var allOps []ovsdb.Operation for i := range routers { lr := &routers[i] @@ -978,10 +980,9 @@ func (c *LiveOVNClient) DeleteAllNATsByExternalIP(ctx context.Context, natType, return len(nats), nil } -// FindNATByExternalIP returns the first NAT rule matching the given type and -// external IP, or nil if none exists. Used by the startup reconcile to check -// whether an EIP already has a dnat_and_snat rule before (re-)creating one. -func (c *LiveOVNClient) FindNATByExternalIP(ctx context.Context, natType, externalIP string) (*nbdb.NAT, error) { +// FindNATByExternalIP returns the first NAT rule matching (natType, +// externalIP) or nil if none exists. +func (c *LiveClient) FindNATByExternalIP(ctx context.Context, natType, externalIP string) (*nbdb.NAT, error) { var nats []nbdb.NAT if err := c.client.WhereCache(func(n *nbdb.NAT) bool { return n.Type == natType && n.ExternalIP == externalIP @@ -994,8 +995,7 @@ func (c *LiveOVNClient) FindNATByExternalIP(ctx context.Context, natType, extern return &nats[0], nil } -func (c *LiveOVNClient) AddStaticRoute(ctx context.Context, routerName string, route *nbdb.LogicalRouterStaticRoute) error { - // Set a named UUID so the route can be referenced in the same transaction +func (c *LiveClient) AddStaticRoute(ctx context.Context, routerName string, route *nbdb.LogicalRouterStaticRoute) error { if route.UUID == "" { route.UUID = namedUUID("route_", route.IPPrefix) } @@ -1027,12 +1027,10 @@ func (c *LiveOVNClient) AddStaticRoute(ctx context.Context, routerName string, r return nil } -// FindStaticRoute returns the first static route on routerName whose IPPrefix -// matches. (nil, nil) when no row matches — caller compares Nexthop and -// OutputPort to detect operator-overridden entries. AddStaticRoute is -// non-idempotent (every retry leaves a fresh duplicate row), so callers that -// run on every reconcile pass must dedupe via this helper first. -func (c *LiveOVNClient) FindStaticRoute(ctx context.Context, routerName, ipPrefix string) (*nbdb.LogicalRouterStaticRoute, error) { +// FindStaticRoute returns the first owned static route on routerName whose +// IPPrefix matches, or (nil, nil). Reconcile callers must dedupe via this +// helper because AddStaticRoute leaves a duplicate row on every retry. +func (c *LiveClient) FindStaticRoute(ctx context.Context, routerName, ipPrefix string) (*nbdb.LogicalRouterStaticRoute, error) { lr, err := c.GetLogicalRouter(ctx, routerName) if err != nil { return nil, fmt.Errorf("get logical router for static route lookup: %w", err) @@ -1057,8 +1055,7 @@ func (c *LiveOVNClient) FindStaticRoute(ctx context.Context, routerName, ipPrefi return &routes[0], nil } -func (c *LiveOVNClient) DeleteStaticRoute(ctx context.Context, routerName string, ipPrefix string) error { - // Find the route by IP prefix +func (c *LiveClient) DeleteStaticRoute(ctx context.Context, routerName string, ipPrefix string) error { var routes []nbdb.LogicalRouterStaticRoute err := c.client.WhereCache(func(r *nbdb.LogicalRouterStaticRoute) bool { return r.IPPrefix == ipPrefix @@ -1100,7 +1097,7 @@ func (c *LiveOVNClient) DeleteStaticRoute(ctx context.Context, routerName string // Port Groups -func (c *LiveOVNClient) CreatePortGroup(ctx context.Context, name string, ports []string) error { +func (c *LiveClient) CreatePortGroup(ctx context.Context, name string, ports []string) error { pg := &nbdb.PortGroup{ UUID: namedUUID("pg_", name), Name: name, @@ -1117,7 +1114,52 @@ func (c *LiveOVNClient) CreatePortGroup(ctx context.Context, name string, ports return nil } -func (c *LiveOVNClient) DeletePortGroup(ctx context.Context, name string) error { +func (c *LiveClient) EnsurePortGroup(ctx context.Context, name string, ports []string) (*nbdb.PortGroup, error) { + if existing, err := c.getPortGroup(ctx, name); err == nil { + return existing, nil + } + pg := &nbdb.PortGroup{ + UUID: namedUUID("pg_", name), + Name: name, + Ports: ports, + ExternalIDs: map[string]string{}, + } + ops, err := c.ensureNamedRowOps("Port_Group", name, pg) + if err != nil { + return nil, err + } + if err := c.transactOps(ctx, ops); err != nil { + if existing, refErr := c.waitForCachedPortGroup(ctx, name); refErr == nil { + slog.Info("ovn: EnsurePortGroup lost race, reusing existing", + "name", name, "existing_uuid", existing.UUID) + return existing, nil + } + return nil, fmt.Errorf("ensure port group %q: %w", name, err) + } + return pg, nil +} + +func (c *LiveClient) waitForCachedPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) { + deadline := time.Now().Add(ensureRefetchTimeout) + var lastErr error + for { + v, err := c.getPortGroup(ctx, name) + if err == nil { + return v, nil + } + lastErr = err + if time.Now().After(deadline) { + return nil, lastErr + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(ensureRefetchInterval): + } + } +} + +func (c *LiveClient) DeletePortGroup(ctx context.Context, name string) error { pg, err := c.getPortGroup(ctx, name) if err != nil { return fmt.Errorf("delete port group lookup: %w", err) @@ -1132,7 +1174,7 @@ func (c *LiveOVNClient) DeletePortGroup(ctx context.Context, name string) error return nil } -func (c *LiveOVNClient) UpdatePortGroupMemberships(ctx context.Context, lspName string, addPGs, removePGs []string) error { +func (c *LiveClient) UpdatePortGroupMemberships(ctx context.Context, lspName string, addPGs, removePGs []string) error { if len(addPGs) == 0 && len(removePGs) == 0 { return nil } @@ -1176,7 +1218,7 @@ func (c *LiveOVNClient) UpdatePortGroupMemberships(ctx context.Context, lspName return nil } -func (c *LiveOVNClient) getPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) { +func (c *LiveClient) getPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) { var pgs []nbdb.PortGroup err := c.client.WhereCache(func(pg *nbdb.PortGroup) bool { return pg.Name == name @@ -1185,18 +1227,17 @@ func (c *LiveOVNClient) getPortGroup(ctx context.Context, name string) (*nbdb.Po return nil, fmt.Errorf("get port group: %w", err) } if len(pgs) == 0 { - return nil, fmt.Errorf("port group %q not found", name) + return nil, fmt.Errorf("%w: %q", ErrPortGroupNotFound, name) } return &pgs[0], nil } -// GetPortGroup is the exported wrapper around getPortGroup. Returns an error -// when the named port group is not present in the cache. -func (c *LiveOVNClient) GetPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) { +// GetPortGroup is the exported wrapper around getPortGroup. +func (c *LiveClient) GetPortGroup(ctx context.Context, name string) (*nbdb.PortGroup, error) { return c.getPortGroup(ctx, name) } -func (c *LiveOVNClient) ListPortGroups(ctx context.Context) ([]nbdb.PortGroup, error) { +func (c *LiveClient) ListPortGroups(ctx context.Context) ([]nbdb.PortGroup, error) { var pgs []nbdb.PortGroup if err := c.client.List(ctx, &pgs); err != nil { return nil, fmt.Errorf("list port groups: %w", err) @@ -1204,11 +1245,9 @@ func (c *LiveOVNClient) ListPortGroups(ctx context.Context) ([]nbdb.PortGroup, e return pgs, nil } -// ListPortGroupsForPort scans the cache for port groups whose Ports set -// contains the given LSP's UUID. Returns the names. Returns an empty slice -// (not an error) when the LSP has no memberships, so reconcilePortSGs handles -// the "not currently in any group" case naturally. -func (c *LiveOVNClient) ListPortGroupsForPort(ctx context.Context, lspName string) ([]string, error) { +// ListPortGroupsForPort returns the names of port groups whose Ports set +// contains the LSP's UUID. Empty slice (not error) when no memberships. +func (c *LiveClient) ListPortGroupsForPort(ctx context.Context, lspName string) ([]string, error) { lsp, err := c.GetLogicalSwitchPort(ctx, lspName) if err != nil { return nil, fmt.Errorf("list port groups for port lookup: %w", err) @@ -1228,7 +1267,7 @@ func (c *LiveOVNClient) ListPortGroupsForPort(ctx context.Context, lspName strin // ACLs -func (c *LiveOVNClient) AddACLs(ctx context.Context, portGroupName string, specs []ACLSpec) error { +func (c *LiveClient) AddACLs(ctx context.Context, portGroupName string, specs []ACLSpec) error { if len(specs) == 0 { return nil } @@ -1240,10 +1279,9 @@ func (c *LiveOVNClient) AddACLs(ctx context.Context, portGroupName string, specs var ops []ovsdb.Operation uuids := make([]string, 0, len(specs)) for i, spec := range specs { - // Index disambiguates ACLs that collapse to the same (direction,match) - // after sanitisation — e.g. a default-deny egress and a 0.0.0.0/0 - // allow egress both produce "inport == @pg && ip4". Without the - // index, OVSDB rejects the second insert with "duplicate uuid-name". + // Index `i` disambiguates ACLs that collapse to the same + // (direction,match) — without it OVSDB rejects the second insert + // with "duplicate uuid-name". acl := &nbdb.ACL{ UUID: namedUUID("acl_", fmt.Sprintf("%s_%d_%s", portGroupName, i, spec.Direction)), Direction: spec.Direction, @@ -1287,7 +1325,7 @@ func (c *LiveOVNClient) AddACLs(ctx context.Context, portGroupName string, specs // ClearACLs removes every ACL row referenced by the port group and detaches // them from the port group's ACLs set in one transaction. -func (c *LiveOVNClient) ClearACLs(ctx context.Context, portGroupName string) error { +func (c *LiveClient) ClearACLs(ctx context.Context, portGroupName string) error { pg, err := c.getPortGroup(ctx, portGroupName) if err != nil { return fmt.Errorf("clear ACLs port group lookup: %w", err) diff --git a/spinifex/network/ovn/live_test.go b/spinifex/network/ovn/live_test.go new file mode 100644 index 000000000..a94e3f28b --- /dev/null +++ b/spinifex/network/ovn/live_test.go @@ -0,0 +1,38 @@ +package ovn + +import "testing" + +func TestNamedUUID(t *testing.T) { + tests := []struct { + name string + prefix string + input string + expected string + }{ + {"hyphen replacement", "sw_", "vpc-abc123", "sw_vpc_abc123"}, + {"slash and dot replacement", "rp_", "10.0.0.1/24", "rp_10_0_0_1_24"}, + {"no special characters", "lr_", "vpcabc", "lr_vpcabc"}, + {"empty name", "sw_", "", "sw_"}, + {"empty prefix", "", "vpc-1", "vpc_1"}, + {"both empty", "", "", ""}, + {"multiple consecutive hyphens", "x_", "a--b", "x_a__b"}, + {"mixed special chars", "p_", "a.b-c/d", "p_a_b_c_d"}, + // ACL match expressions embed @, =, &, space; OVSDB rejects unsanitised + // named-uuids with "Type mismatch for member 'uuid-name'". + {"acl match at sign", "acl_", "outport == @pg-sg-XYZ && ip4", "acl_outport_____pg_sg_XYZ____ip4"}, + {"space replacement", "acl_", "a b c", "acl_a_b_c"}, + {"equals replacement", "acl_", "a==b", "acl_a__b"}, + {"ampersand replacement", "acl_", "a&&b", "acl_a__b"}, + {"colon replacement", "acl_", "ip6.src == fe80::1", "acl_ip6_src____fe80__1"}, + {"parens and quotes", "acl_", `ip4.src == "10.0.0.1"`, "acl_ip4_src_____10_0_0_1_"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := namedUUID(tt.prefix, tt.input) + if got != tt.expected { + t.Errorf("namedUUID(%q, %q) = %q, want %q", tt.prefix, tt.input, got, tt.expected) + } + }) + } +} diff --git a/spinifex/network/ovn/mock/mock.go b/spinifex/network/ovn/mock/mock.go new file mode 100644 index 000000000..08c03b7d8 --- /dev/null +++ b/spinifex/network/ovn/mock/mock.go @@ -0,0 +1,921 @@ +// Package mock provides an in-memory implementation of ovn.Client for tests. +package mock + +import ( + "context" + "fmt" + "maps" + "slices" + "sync" + + "github.com/mulgadc/spinifex/spinifex/network/ovn" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/utils" +) + +// Client implements ovn.Client with in-memory storage for testing. +type Client struct { + Mu sync.Mutex + connected bool + + Switches map[string]*nbdb.LogicalSwitch + Ports map[string]*nbdb.LogicalSwitchPort + Routers map[string]*nbdb.LogicalRouter + RouterPorts map[string]*nbdb.LogicalRouterPort + DHCPOpts map[string]*nbdb.DHCPOptions + NATs map[string]*nbdb.NAT // keyed by UUID + StaticRoutes map[string]*nbdb.LogicalRouterStaticRoute // keyed by UUID + PortGroups map[string]*nbdb.PortGroup // keyed by name + ACLs map[string]*nbdb.ACL // keyed by UUID + GatewayChassis map[string]*nbdb.GatewayChassis // keyed by UUID + + // Call counters so tests can assert idempotent / drift-only write paths. + UpdateLogicalSwitchPortCalls int + UpdateLogicalRouterPortCalls int + + SetGatewayChassisCalls int + DeleteGatewayChassisCalls int + UpdateGatewayChassisPriorityCalls int + + // AddACLErrAfter, when > 0, makes AddACLs return an error on the Nth + // (1-indexed) call. Lets tests inject a transient failure mid-flow. + AddACLErrAfter int + AddACLCalls int +} + +var _ ovn.Client = (*Client)(nil) + +// New creates a new mock Client for testing. +func New() *Client { + return &Client{ + Switches: make(map[string]*nbdb.LogicalSwitch), + Ports: make(map[string]*nbdb.LogicalSwitchPort), + Routers: make(map[string]*nbdb.LogicalRouter), + RouterPorts: make(map[string]*nbdb.LogicalRouterPort), + DHCPOpts: make(map[string]*nbdb.DHCPOptions), + NATs: make(map[string]*nbdb.NAT), + StaticRoutes: make(map[string]*nbdb.LogicalRouterStaticRoute), + PortGroups: make(map[string]*nbdb.PortGroup), + ACLs: make(map[string]*nbdb.ACL), + GatewayChassis: make(map[string]*nbdb.GatewayChassis), + } +} + +func (m *Client) Connect(_ context.Context) error { + m.Mu.Lock() + defer m.Mu.Unlock() + m.connected = true + return nil +} + +func (m *Client) Close() { + m.Mu.Lock() + defer m.Mu.Unlock() + m.connected = false +} + +func (m *Client) Connected() bool { + m.Mu.Lock() + defer m.Mu.Unlock() + return m.connected +} + +// Logical Switch + +func (m *Client) CreateLogicalSwitch(_ context.Context, ls *nbdb.LogicalSwitch) error { + m.Mu.Lock() + defer m.Mu.Unlock() + if _, exists := m.Switches[ls.Name]; exists { + return fmt.Errorf("logical switch %q already exists", ls.Name) + } + if ls.UUID == "" { + ls.UUID = utils.GenerateResourceID("ovn") + } + stored := *ls + m.Switches[ls.Name] = &stored + return nil +} + +// EnsureLogicalSwitch mirrors the live client's wait-op semantics under the +// mock's single mutex: concurrent callers see the row on the second arrival. +func (m *Client) EnsureLogicalSwitch(_ context.Context, ls *nbdb.LogicalSwitch) (*nbdb.LogicalSwitch, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + if existing, ok := m.Switches[ls.Name]; ok { + result := *existing + return &result, nil + } + if ls.UUID == "" { + ls.UUID = utils.GenerateResourceID("ovn") + } + stored := *ls + m.Switches[ls.Name] = &stored + result := stored + return &result, nil +} + +func (m *Client) DeleteLogicalSwitch(_ context.Context, name string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + if _, exists := m.Switches[name]; !exists { + return fmt.Errorf("logical switch %q not found", name) + } + delete(m.Switches, name) + return nil +} + +func (m *Client) GetLogicalSwitch(_ context.Context, name string) (*nbdb.LogicalSwitch, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + ls, exists := m.Switches[name] + if !exists { + return nil, fmt.Errorf("logical switch %q not found", name) + } + result := *ls + return &result, nil +} + +func (m *Client) ListLogicalSwitches(_ context.Context) ([]nbdb.LogicalSwitch, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + result := make([]nbdb.LogicalSwitch, 0, len(m.Switches)) + for _, ls := range m.Switches { + result = append(result, *ls) + } + return result, nil +} + +// Logical Switch Port + +func (m *Client) CreateLogicalSwitchPort(_ context.Context, switchName string, lsp *nbdb.LogicalSwitchPort) error { + m.Mu.Lock() + defer m.Mu.Unlock() + ls, exists := m.Switches[switchName] + if !exists { + return fmt.Errorf("logical switch %q not found", switchName) + } + if _, exists := m.Ports[lsp.Name]; exists { + return fmt.Errorf("logical switch port %q already exists", lsp.Name) + } + if lsp.UUID == "" { + lsp.UUID = utils.GenerateResourceID("ovn") + } + stored := *lsp + m.Ports[lsp.Name] = &stored + ls.Ports = append(ls.Ports, lsp.UUID) + return nil +} + +// CreateLogicalSwitchPortInGroups mirrors the live client's atomic create + +// port-group join: validates all port groups up front so a missing PG leaves +// no LSP behind. SB Address_Set rows are derived by ovn-northd and not +// modelled here. +func (m *Client) CreateLogicalSwitchPortInGroups(_ context.Context, switchName string, lsp *nbdb.LogicalSwitchPort, portGroupNames []string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + ls, exists := m.Switches[switchName] + if !exists { + return fmt.Errorf("logical switch %q not found", switchName) + } + if _, exists := m.Ports[lsp.Name]; exists { + return fmt.Errorf("logical switch port %q already exists", lsp.Name) + } + for _, pgName := range portGroupNames { + if _, ok := m.PortGroups[pgName]; !ok { + return fmt.Errorf("port group %q not found", pgName) + } + } + if lsp.UUID == "" { + lsp.UUID = utils.GenerateResourceID("ovn") + } + stored := *lsp + m.Ports[lsp.Name] = &stored + ls.Ports = append(ls.Ports, lsp.UUID) + for _, pgName := range portGroupNames { + pg := m.PortGroups[pgName] + if !slices.Contains(pg.Ports, lsp.UUID) { + pg.Ports = append(pg.Ports, lsp.UUID) + } + } + return nil +} + +func (m *Client) DeleteLogicalSwitchPort(_ context.Context, switchName string, portName string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + port, exists := m.Ports[portName] + if !exists { + return fmt.Errorf("logical switch port %q not found", portName) + } + ls, exists := m.Switches[switchName] + if !exists { + return fmt.Errorf("logical switch %q not found", switchName) + } + // Mirror libovsdb's reference-integrity rejection: an LSP still in any + // port group's Ports set cannot be deleted. + for pgName, pg := range m.PortGroups { + if slices.Contains(pg.Ports, port.UUID) { + return fmt.Errorf("logical switch port %q still in port group %q", portName, pgName) + } + } + for i, uuid := range ls.Ports { + if uuid == port.UUID { + ls.Ports = append(ls.Ports[:i], ls.Ports[i+1:]...) + break + } + } + delete(m.Ports, portName) + return nil +} + +func (m *Client) GetLogicalSwitchPort(_ context.Context, name string) (*nbdb.LogicalSwitchPort, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + lsp, exists := m.Ports[name] + if !exists { + return nil, fmt.Errorf("logical switch port %q not found", name) + } + result := *lsp + return &result, nil +} + +func (m *Client) UpdateLogicalSwitchPort(_ context.Context, lsp *nbdb.LogicalSwitchPort) error { + m.Mu.Lock() + defer m.Mu.Unlock() + if _, exists := m.Ports[lsp.Name]; !exists { + return fmt.Errorf("logical switch port %q not found", lsp.Name) + } + stored := *lsp + m.Ports[lsp.Name] = &stored + m.UpdateLogicalSwitchPortCalls++ + return nil +} + +// Logical Router + +func (m *Client) CreateLogicalRouter(_ context.Context, lr *nbdb.LogicalRouter) error { + m.Mu.Lock() + defer m.Mu.Unlock() + if _, exists := m.Routers[lr.Name]; exists { + return fmt.Errorf("logical router %q already exists", lr.Name) + } + if lr.UUID == "" { + lr.UUID = utils.GenerateResourceID("ovn") + } + stored := *lr + m.Routers[lr.Name] = &stored + return nil +} + +// EnsureLogicalRouter mirrors the live client's wait-op semantics under the +// mock's single mutex. +func (m *Client) EnsureLogicalRouter(_ context.Context, lr *nbdb.LogicalRouter) (*nbdb.LogicalRouter, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + if existing, ok := m.Routers[lr.Name]; ok { + result := *existing + return &result, nil + } + if lr.UUID == "" { + lr.UUID = utils.GenerateResourceID("ovn") + } + stored := *lr + m.Routers[lr.Name] = &stored + result := stored + return &result, nil +} + +func (m *Client) DeleteLogicalRouter(_ context.Context, name string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + if _, exists := m.Routers[name]; !exists { + return fmt.Errorf("logical router %q not found", name) + } + delete(m.Routers, name) + return nil +} + +func (m *Client) GetLogicalRouter(_ context.Context, name string) (*nbdb.LogicalRouter, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + lr, exists := m.Routers[name] + if !exists { + return nil, fmt.Errorf("logical router %q not found", name) + } + result := *lr + return &result, nil +} + +func (m *Client) ListLogicalRouters(_ context.Context) ([]nbdb.LogicalRouter, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + result := make([]nbdb.LogicalRouter, 0, len(m.Routers)) + for _, lr := range m.Routers { + result = append(result, *lr) + } + return result, nil +} + +func (m *Client) UpdateLogicalRouterExternalIDs(_ context.Context, name string, externalIDs map[string]string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + lr, ok := m.Routers[name] + if !ok { + return fmt.Errorf("logical router %q not found", name) + } + lr.ExternalIDs = make(map[string]string, len(externalIDs)) + maps.Copy(lr.ExternalIDs, externalIDs) + return nil +} + +// Logical Router Port + +func (m *Client) CreateLogicalRouterPort(_ context.Context, routerName string, lrp *nbdb.LogicalRouterPort) error { + m.Mu.Lock() + defer m.Mu.Unlock() + lr, exists := m.Routers[routerName] + if !exists { + return fmt.Errorf("logical router %q not found", routerName) + } + if _, exists := m.RouterPorts[lrp.Name]; exists { + return fmt.Errorf("logical router port %q already exists", lrp.Name) + } + if lrp.UUID == "" { + lrp.UUID = utils.GenerateResourceID("ovn") + } + stored := *lrp + m.RouterPorts[lrp.Name] = &stored + lr.Ports = append(lr.Ports, lrp.UUID) + return nil +} + +func (m *Client) DeleteLogicalRouterPort(_ context.Context, routerName string, portName string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + port, exists := m.RouterPorts[portName] + if !exists { + return fmt.Errorf("logical router port %q not found", portName) + } + lr, exists := m.Routers[routerName] + if !exists { + return fmt.Errorf("logical router %q not found", routerName) + } + for i, uuid := range lr.Ports { + if uuid == port.UUID { + lr.Ports = append(lr.Ports[:i], lr.Ports[i+1:]...) + break + } + } + delete(m.RouterPorts, portName) + return nil +} + +func (m *Client) GetLogicalRouterPort(_ context.Context, name string) (*nbdb.LogicalRouterPort, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + lrp, exists := m.RouterPorts[name] + if !exists { + return nil, fmt.Errorf("logical router port %q not found", name) + } + result := *lrp + return &result, nil +} + +func (m *Client) UpdateLogicalRouterPort(_ context.Context, lrp *nbdb.LogicalRouterPort) error { + m.Mu.Lock() + defer m.Mu.Unlock() + if _, exists := m.RouterPorts[lrp.Name]; !exists { + return fmt.Errorf("logical router port %q not found", lrp.Name) + } + stored := *lrp + m.RouterPorts[lrp.Name] = &stored + m.UpdateLogicalRouterPortCalls++ + return nil +} + +// DHCP Options + +func (m *Client) CreateDHCPOptions(_ context.Context, opts *nbdb.DHCPOptions) (string, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + if opts.UUID == "" { + opts.UUID = utils.GenerateResourceID("dhcp") + } + stored := *opts + m.DHCPOpts[opts.UUID] = &stored + return opts.UUID, nil +} + +func (m *Client) DeleteDHCPOptions(_ context.Context, uuid string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + if _, exists := m.DHCPOpts[uuid]; !exists { + return fmt.Errorf("DHCP options %q not found", uuid) + } + delete(m.DHCPOpts, uuid) + return nil +} + +func (m *Client) FindDHCPOptionsByCIDR(_ context.Context, cidr string) (*nbdb.DHCPOptions, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + for _, opts := range m.DHCPOpts { + if opts.CIDR == cidr { + result := *opts + return &result, nil + } + } + return nil, fmt.Errorf("DHCP options for CIDR %q not found", cidr) +} + +func (m *Client) FindDHCPOptionsByExternalID(_ context.Context, key, value string) (*nbdb.DHCPOptions, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + for _, opts := range m.DHCPOpts { + if opts.ExternalIDs[key] == value { + result := *opts + return &result, nil + } + } + return nil, fmt.Errorf("DHCP options with external_id %s=%s not found", key, value) +} + +func (m *Client) ListDHCPOptions(_ context.Context) ([]nbdb.DHCPOptions, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + result := make([]nbdb.DHCPOptions, 0, len(m.DHCPOpts)) + for _, opts := range m.DHCPOpts { + result = append(result, *opts) + } + return result, nil +} + +// NAT + +func (m *Client) AddNAT(_ context.Context, routerName string, nat *nbdb.NAT) error { + m.Mu.Lock() + defer m.Mu.Unlock() + lr, exists := m.Routers[routerName] + if !exists { + return fmt.Errorf("logical router %q not found", routerName) + } + if nat.UUID == "" { + nat.UUID = utils.GenerateResourceID("nat") + } + stored := *nat + m.NATs[nat.UUID] = &stored + lr.NAT = append(lr.NAT, nat.UUID) + return nil +} + +func (m *Client) DeleteNAT(_ context.Context, routerName string, natType, logicalIP string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + lr, exists := m.Routers[routerName] + if !exists { + return fmt.Errorf("logical router %q not found", routerName) + } + var foundUUID string + var foundIdx int + for i, uuid := range lr.NAT { + n, ok := m.NATs[uuid] + if !ok { + continue + } + if n.Type == natType && n.LogicalIP == logicalIP { + foundUUID = uuid + foundIdx = i + break + } + } + if foundUUID == "" { + return fmt.Errorf("NAT %s %s on %s: %w", natType, logicalIP, routerName, ovn.ErrNATNotFound) + } + lr.NAT = append(lr.NAT[:foundIdx], lr.NAT[foundIdx+1:]...) + delete(m.NATs, foundUUID) + return nil +} + +func (m *Client) DeleteNATByExternalIP(_ context.Context, routerName string, natType, externalIP string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + lr, exists := m.Routers[routerName] + if !exists { + return fmt.Errorf("logical router %q not found", routerName) + } + var foundUUIDs []string + for _, uuid := range lr.NAT { + n, ok := m.NATs[uuid] + if !ok { + continue + } + if n.Type == natType && n.ExternalIP == externalIP { + foundUUIDs = append(foundUUIDs, uuid) + } + } + if len(foundUUIDs) == 0 { + return fmt.Errorf("NAT %s external_ip=%s on %s: %w", natType, externalIP, routerName, ovn.ErrNATNotFound) + } + stale := make(map[string]struct{}, len(foundUUIDs)) + for _, u := range foundUUIDs { + stale[u] = struct{}{} + } + filtered := lr.NAT[:0] + for _, uuid := range lr.NAT { + if _, drop := stale[uuid]; drop { + continue + } + filtered = append(filtered, uuid) + } + lr.NAT = filtered + for _, u := range foundUUIDs { + delete(m.NATs, u) + } + return nil +} + +func (m *Client) DeleteAllNATsByExternalIP(_ context.Context, natType, externalIP string) (int, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + + var foundUUIDs []string + for uuid, n := range m.NATs { + if n.Type == natType && n.ExternalIP == externalIP { + foundUUIDs = append(foundUUIDs, uuid) + } + } + if len(foundUUIDs) == 0 { + return 0, nil + } + + uuidSet := make(map[string]struct{}, len(foundUUIDs)) + for _, u := range foundUUIDs { + uuidSet[u] = struct{}{} + } + for _, lr := range m.Routers { + filtered := lr.NAT[:0] + for _, uuid := range lr.NAT { + if _, stale := uuidSet[uuid]; !stale { + filtered = append(filtered, uuid) + } + } + lr.NAT = filtered + } + + for _, u := range foundUUIDs { + delete(m.NATs, u) + } + return len(foundUUIDs), nil +} + +func (m *Client) FindNATByExternalIP(_ context.Context, natType, externalIP string) (*nbdb.NAT, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + for _, n := range m.NATs { + if n.Type == natType && n.ExternalIP == externalIP { + return n, nil + } + } + return nil, nil +} + +// Static Routes + +func (m *Client) AddStaticRoute(_ context.Context, routerName string, route *nbdb.LogicalRouterStaticRoute) error { + m.Mu.Lock() + defer m.Mu.Unlock() + lr, exists := m.Routers[routerName] + if !exists { + return fmt.Errorf("logical router %q not found", routerName) + } + if route.UUID == "" { + route.UUID = utils.GenerateResourceID("route") + } + stored := *route + m.StaticRoutes[route.UUID] = &stored + lr.StaticRoutes = append(lr.StaticRoutes, route.UUID) + return nil +} + +func (m *Client) FindStaticRoute(_ context.Context, routerName, ipPrefix string) (*nbdb.LogicalRouterStaticRoute, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + lr, exists := m.Routers[routerName] + if !exists { + return nil, fmt.Errorf("logical router %q not found", routerName) + } + for _, uuid := range lr.StaticRoutes { + r, ok := m.StaticRoutes[uuid] + if !ok { + continue + } + if r.IPPrefix == ipPrefix { + result := *r + return &result, nil + } + } + return nil, nil +} + +func (m *Client) DeleteStaticRoute(_ context.Context, routerName string, ipPrefix string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + lr, exists := m.Routers[routerName] + if !exists { + return fmt.Errorf("logical router %q not found", routerName) + } + var foundUUID string + for uuid, r := range m.StaticRoutes { + if r.IPPrefix == ipPrefix { + foundUUID = uuid + break + } + } + if foundUUID == "" { + return fmt.Errorf("static route %s not found", ipPrefix) + } + for i, uuid := range lr.StaticRoutes { + if uuid == foundUUID { + lr.StaticRoutes = append(lr.StaticRoutes[:i], lr.StaticRoutes[i+1:]...) + break + } + } + delete(m.StaticRoutes, foundUUID) + return nil +} + +// Port Groups + +func (m *Client) CreatePortGroup(_ context.Context, name string, ports []string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + if _, exists := m.PortGroups[name]; exists { + return fmt.Errorf("port group %q already exists", name) + } + pg := &nbdb.PortGroup{ + UUID: utils.GenerateResourceID("pg"), + Name: name, + Ports: ports, + } + m.PortGroups[name] = pg + return nil +} + +// EnsurePortGroup mirrors the live client's wait-op semantics. +func (m *Client) EnsurePortGroup(_ context.Context, name string, ports []string) (*nbdb.PortGroup, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + if existing, ok := m.PortGroups[name]; ok { + result := *existing + return &result, nil + } + pg := &nbdb.PortGroup{ + UUID: utils.GenerateResourceID("pg"), + Name: name, + Ports: ports, + } + m.PortGroups[name] = pg + result := *pg + return &result, nil +} + +func (m *Client) DeletePortGroup(_ context.Context, name string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + pg, exists := m.PortGroups[name] + if !exists { + return fmt.Errorf("port group %q not found", name) + } + // Mirror libovsdb: PG with referenced ACL rows cannot be deleted; live + // `Where(pg).Delete()` would leak the ACLs. Force ClearACLs first. + if len(pg.ACLs) > 0 { + return fmt.Errorf("port group %q has %d ACLs still attached; clear ACLs before delete", name, len(pg.ACLs)) + } + delete(m.PortGroups, name) + return nil +} + +// UpdatePortGroupMemberships applies all add/remove PG joins for one LSP. +func (m *Client) UpdatePortGroupMemberships(_ context.Context, lspName string, addPGs, removePGs []string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + lsp, exists := m.Ports[lspName] + if !exists { + return fmt.Errorf("logical switch port %q not found", lspName) + } + for _, pgName := range addPGs { + pg, exists := m.PortGroups[pgName] + if !exists { + return fmt.Errorf("port group %q not found", pgName) + } + if !slices.Contains(pg.Ports, lsp.UUID) { + pg.Ports = append(pg.Ports, lsp.UUID) + } + } + for _, pgName := range removePGs { + pg, exists := m.PortGroups[pgName] + if !exists { + return fmt.Errorf("port group %q not found", pgName) + } + for i, u := range pg.Ports { + if u == lsp.UUID { + pg.Ports = append(pg.Ports[:i], pg.Ports[i+1:]...) + break + } + } + } + return nil +} + +// ListPortGroupsForPort returns names of PGs containing the LSP's UUID. +// Empty slice (not error) when no memberships. +func (m *Client) ListPortGroupsForPort(_ context.Context, lspName string) ([]string, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + lsp, exists := m.Ports[lspName] + if !exists { + return nil, fmt.Errorf("logical switch port %q not found", lspName) + } + names := make([]string, 0) + for name, pg := range m.PortGroups { + if slices.Contains(pg.Ports, lsp.UUID) { + names = append(names, name) + } + } + return names, nil +} + +// GetPortGroup returns the port group, or ovn.ErrPortGroupNotFound. +func (m *Client) GetPortGroup(_ context.Context, name string) (*nbdb.PortGroup, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + pg, exists := m.PortGroups[name] + if !exists { + return nil, fmt.Errorf("%w: %q", ovn.ErrPortGroupNotFound, name) + } + return pg, nil +} + +// ListPortGroups returns a snapshot of every port group. +func (m *Client) ListPortGroups(_ context.Context) ([]nbdb.PortGroup, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + out := make([]nbdb.PortGroup, 0, len(m.PortGroups)) + for _, pg := range m.PortGroups { + out = append(out, *pg) + } + return out, nil +} + +// ACLs + +func (m *Client) AddACLs(_ context.Context, portGroupName string, specs []ovn.ACLSpec) error { + m.Mu.Lock() + defer m.Mu.Unlock() + m.AddACLCalls++ + if m.AddACLErrAfter > 0 && m.AddACLCalls == m.AddACLErrAfter { + return fmt.Errorf("injected AddACLs failure on call %d", m.AddACLCalls) + } + pg, exists := m.PortGroups[portGroupName] + if !exists { + return fmt.Errorf("port group %q not found", portGroupName) + } + for _, spec := range specs { + acl := &nbdb.ACL{ + UUID: utils.GenerateResourceID("acl"), + Direction: spec.Direction, + Priority: spec.Priority, + Match: spec.Match, + Action: spec.Action, + Log: spec.Log, + } + if spec.Name != "" { + name := spec.Name + acl.Name = &name + } + if spec.Severity != "" { + severity := spec.Severity + acl.Severity = &severity + } + m.ACLs[acl.UUID] = acl + pg.ACLs = append(pg.ACLs, acl.UUID) + } + return nil +} + +func (m *Client) ClearACLs(_ context.Context, portGroupName string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + pg, exists := m.PortGroups[portGroupName] + if !exists { + return fmt.Errorf("port group %q not found", portGroupName) + } + for _, aclUUID := range pg.ACLs { + delete(m.ACLs, aclUUID) + } + pg.ACLs = nil + return nil +} + +// ListLogicalRouterPorts returns every LRP across the mock state. +func (m *Client) ListLogicalRouterPorts(_ context.Context) ([]nbdb.LogicalRouterPort, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + result := make([]nbdb.LogicalRouterPort, 0, len(m.RouterPorts)) + for _, lrp := range m.RouterPorts { + result = append(result, *lrp) + } + return result, nil +} + +// SetGatewayChassis is the idempotent read-then-decide path mirroring +// LiveClient; counters distinguish create vs. update vs. no-op. +func (m *Client) SetGatewayChassis(_ context.Context, lrpName string, chassisName string, priority int) error { + m.Mu.Lock() + defer m.Mu.Unlock() + lrp, exists := m.RouterPorts[lrpName] + if !exists { + return fmt.Errorf("logical router port %q not found", lrpName) + } + gcName := lrpName + "-" + chassisName + for _, gc := range m.GatewayChassis { + if gc.Name != gcName { + continue + } + if gc.Priority == priority { + return nil + } + gc.Priority = priority + m.UpdateGatewayChassisPriorityCalls++ + return nil + } + gc := &nbdb.GatewayChassis{ + UUID: utils.GenerateResourceID("gc"), + Name: gcName, + ChassisName: chassisName, + Priority: priority, + ExternalIDs: map[string]string{}, + Options: map[string]string{}, + } + m.GatewayChassis[gc.UUID] = gc + lrp.GatewayChassis = append(lrp.GatewayChassis, gc.UUID) + m.SetGatewayChassisCalls++ + return nil +} + +func (m *Client) GetGatewayChassisByName(_ context.Context, name string) (*nbdb.GatewayChassis, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + for _, gc := range m.GatewayChassis { + if gc.Name == name { + result := *gc + return &result, nil + } + } + return nil, nil +} + +func (m *Client) ListGatewayChassis(_ context.Context) ([]nbdb.GatewayChassis, error) { + m.Mu.Lock() + defer m.Mu.Unlock() + result := make([]nbdb.GatewayChassis, 0, len(m.GatewayChassis)) + for _, gc := range m.GatewayChassis { + result = append(result, *gc) + } + return result, nil +} + +func (m *Client) DeleteGatewayChassis(_ context.Context, lrpName string, gcUUID string) error { + m.Mu.Lock() + defer m.Mu.Unlock() + lrp, exists := m.RouterPorts[lrpName] + if !exists { + return fmt.Errorf("logical router port %q not found", lrpName) + } + if _, ok := m.GatewayChassis[gcUUID]; !ok { + return fmt.Errorf("gateway_chassis %q not found", gcUUID) + } + for i, u := range lrp.GatewayChassis { + if u == gcUUID { + lrp.GatewayChassis = append(lrp.GatewayChassis[:i], lrp.GatewayChassis[i+1:]...) + break + } + } + delete(m.GatewayChassis, gcUUID) + m.DeleteGatewayChassisCalls++ + return nil +} + +// SeedGatewayChassis pre-populates a Gateway_Chassis row, bypassing +// SetGatewayChassis. Used to construct "stale row" scenarios in tests. +func (m *Client) SeedGatewayChassis(lrpName string, gc *nbdb.GatewayChassis) { + m.Mu.Lock() + defer m.Mu.Unlock() + if gc.UUID == "" { + gc.UUID = utils.GenerateResourceID("gc") + } + m.GatewayChassis[gc.UUID] = gc + if lrp, ok := m.RouterPorts[lrpName]; ok { + lrp.GatewayChassis = append(lrp.GatewayChassis, gc.UUID) + } +} diff --git a/spinifex/network/ovn/mock/mock_test.go b/spinifex/network/ovn/mock/mock_test.go new file mode 100644 index 000000000..2a260fbe9 --- /dev/null +++ b/spinifex/network/ovn/mock/mock_test.go @@ -0,0 +1,287 @@ +package mock + +import ( + "context" + "sync" + "testing" + + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" +) + +func TestClient_DeleteAllNATsByExternalIP(t *testing.T) { + m := New() + _ = m.Connect(context.Background()) + ctx := context.Background() + + _ = m.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ + Name: "vpc-r1", + ExternalIDs: map[string]string{"spinifex:vpc_id": "r1"}, + }) + _ = m.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ + Name: "vpc-r2", + ExternalIDs: map[string]string{"spinifex:vpc_id": "r2"}, + }) + + _ = m.AddNAT(ctx, "vpc-r1", &nbdb.NAT{ + Type: "dnat_and_snat", ExternalIP: "192.168.1.50", LogicalIP: "10.0.0.1", + }) + _ = m.AddNAT(ctx, "vpc-r2", &nbdb.NAT{ + Type: "dnat_and_snat", ExternalIP: "192.168.1.50", LogicalIP: "10.200.0.1", + }) + // Unrelated NAT on r1 that should be untouched. + _ = m.AddNAT(ctx, "vpc-r1", &nbdb.NAT{ + Type: "dnat_and_snat", ExternalIP: "192.168.1.99", LogicalIP: "10.0.0.2", + }) + + removed, err := m.DeleteAllNATsByExternalIP(ctx, "dnat_and_snat", "192.168.1.50") + if err != nil { + t.Fatalf("DeleteAllNATsByExternalIP: %v", err) + } + if removed != 2 { + t.Errorf("expected 2 removed, got %d", removed) + } + + r1, _ := m.GetLogicalRouter(ctx, "vpc-r1") + if len(r1.NAT) != 1 { + t.Errorf("r1 should retain 1 unrelated NAT, got %d", len(r1.NAT)) + } + + r2, _ := m.GetLogicalRouter(ctx, "vpc-r2") + if len(r2.NAT) != 0 { + t.Errorf("r2 should have 0 NAT rules, got %d", len(r2.NAT)) + } + + removed, err = m.DeleteAllNATsByExternalIP(ctx, "dnat_and_snat", "192.168.1.50") + if err != nil { + t.Fatalf("second DeleteAllNATsByExternalIP: %v", err) + } + if removed != 0 { + t.Errorf("expected 0 removed on second call, got %d", removed) + } +} + +func TestSetGatewayChassis_Idempotent(t *testing.T) { + m := New() + _ = m.Connect(context.Background()) + ctx := context.Background() + + if err := m.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-A"}); err != nil { + t.Fatalf("CreateLogicalRouter: %v", err) + } + if err := m.CreateLogicalRouterPort(ctx, "vpc-A", &nbdb.LogicalRouterPort{Name: "gw-A"}); err != nil { + t.Fatalf("CreateLogicalRouterPort: %v", err) + } + + if err := m.SetGatewayChassis(ctx, "gw-A", "chassis-1", 20); err != nil { + t.Fatalf("SetGatewayChassis (1): %v", err) + } + if err := m.SetGatewayChassis(ctx, "gw-A", "chassis-1", 20); err != nil { + t.Fatalf("SetGatewayChassis (2): %v", err) + } + + rows, err := m.ListGatewayChassis(ctx) + if err != nil { + t.Fatalf("ListGatewayChassis: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected 1 row, got %d (%v)", len(rows), rows) + } + if m.SetGatewayChassisCalls != 1 { + t.Errorf("expected 1 create call, got %d", m.SetGatewayChassisCalls) + } + if m.UpdateGatewayChassisPriorityCalls != 0 { + t.Errorf("expected 0 priority updates, got %d", m.UpdateGatewayChassisPriorityCalls) + } +} + +func TestSetGatewayChassis_UpdatesPriority(t *testing.T) { + m := New() + _ = m.Connect(context.Background()) + ctx := context.Background() + + if err := m.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-A"}); err != nil { + t.Fatalf("CreateLogicalRouter: %v", err) + } + if err := m.CreateLogicalRouterPort(ctx, "vpc-A", &nbdb.LogicalRouterPort{Name: "gw-A"}); err != nil { + t.Fatalf("CreateLogicalRouterPort: %v", err) + } + + if err := m.SetGatewayChassis(ctx, "gw-A", "chassis-1", 20); err != nil { + t.Fatalf("SetGatewayChassis at 20: %v", err) + } + if err := m.SetGatewayChassis(ctx, "gw-A", "chassis-1", 15); err != nil { + t.Fatalf("SetGatewayChassis at 15: %v", err) + } + + rows, err := m.ListGatewayChassis(ctx) + if err != nil { + t.Fatalf("ListGatewayChassis: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected 1 row (mutated, not duplicated), got %d", len(rows)) + } + if rows[0].Priority != 15 { + t.Errorf("expected priority 15 after update, got %d", rows[0].Priority) + } + if m.UpdateGatewayChassisPriorityCalls != 1 { + t.Errorf("expected 1 priority update, got %d", m.UpdateGatewayChassisPriorityCalls) + } +} + +// N concurrent EnsureLogicalRouter calls for the same Name must converge to +// a single row. +func TestEnsureLogicalRouter_ConcurrentSingleSurvivor(t *testing.T) { + const callers = 50 + m := New() + _ = m.Connect(context.Background()) + ctx := context.Background() + + uuids := make([]string, callers) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range callers { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + lr, err := m.EnsureLogicalRouter(ctx, &nbdb.LogicalRouter{ + Name: "vpc-vpc-X", + ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-X"}, + }) + if err != nil { + t.Errorf("EnsureLogicalRouter[%d]: %v", i, err) + return + } + uuids[i] = lr.UUID + }(i) + } + close(start) + wg.Wait() + + first := uuids[0] + if first == "" { + t.Fatalf("caller 0 got empty UUID") + } + for i, u := range uuids { + if u != first { + t.Errorf("caller %d UUID mismatch: %q vs %q", i, u, first) + } + } + + rows, err := m.ListLogicalRouters(ctx) + if err != nil { + t.Fatalf("ListLogicalRouters: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected 1 router after concurrent EnsureLogicalRouter, got %d", len(rows)) + } +} + +// Sequential second EnsureLogicalRouter for an existing Name returns the +// pre-existing row. +func TestEnsureLogicalRouter_ReturnsExistingOnSecondCall(t *testing.T) { + m := New() + _ = m.Connect(context.Background()) + ctx := context.Background() + + first, err := m.EnsureLogicalRouter(ctx, &nbdb.LogicalRouter{ + Name: "vpc-vpc-Y", + ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-Y", "spinifex:cidr": ""}, + }) + if err != nil { + t.Fatalf("EnsureLogicalRouter first call: %v", err) + } + second, err := m.EnsureLogicalRouter(ctx, &nbdb.LogicalRouter{ + Name: "vpc-vpc-Y", + ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-Y", "spinifex:cidr": "10.0.0.0/16"}, + }) + if err != nil { + t.Fatalf("EnsureLogicalRouter second call: %v", err) + } + if first.UUID != second.UUID { + t.Errorf("second call returned different UUID: %q vs %q", second.UUID, first.UUID) + } +} + +func TestEnsureLogicalSwitch_ConcurrentSingleSurvivor(t *testing.T) { + const callers = 30 + m := New() + _ = m.Connect(context.Background()) + ctx := context.Background() + + uuids := make([]string, callers) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range callers { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + ls, err := m.EnsureLogicalSwitch(ctx, &nbdb.LogicalSwitch{ + Name: "subnet-subnet-Z", + ExternalIDs: map[string]string{"spinifex:subnet_id": "subnet-Z"}, + }) + if err != nil { + t.Errorf("EnsureLogicalSwitch[%d]: %v", i, err) + return + } + uuids[i] = ls.UUID + }(i) + } + close(start) + wg.Wait() + + first := uuids[0] + for i, u := range uuids { + if u != first { + t.Errorf("caller %d UUID mismatch: %q vs %q", i, u, first) + } + } + rows, err := m.ListLogicalSwitches(ctx) + if err != nil { + t.Fatalf("ListLogicalSwitches: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected 1 switch, got %d", len(rows)) + } +} + +func TestEnsurePortGroup_ConcurrentSingleSurvivor(t *testing.T) { + const callers = 30 + m := New() + _ = m.Connect(context.Background()) + ctx := context.Background() + + uuids := make([]string, callers) + var wg sync.WaitGroup + start := make(chan struct{}) + for i := range callers { + wg.Add(1) + go func(i int) { + defer wg.Done() + <-start + pg, err := m.EnsurePortGroup(ctx, "sg-pg-A", nil) + if err != nil { + t.Errorf("EnsurePortGroup[%d]: %v", i, err) + return + } + uuids[i] = pg.UUID + }(i) + } + close(start) + wg.Wait() + + first := uuids[0] + for i, u := range uuids { + if u != first { + t.Errorf("caller %d UUID mismatch: %q vs %q", i, u, first) + } + } + rows, err := m.ListPortGroups(ctx) + if err != nil { + t.Fatalf("ListPortGroups: %v", err) + } + if len(rows) != 1 { + t.Fatalf("expected 1 port group, got %d", len(rows)) + } +} diff --git a/spinifex/services/vpcd/nbdb/model.go b/spinifex/network/ovn/nbdb/model.go similarity index 85% rename from spinifex/services/vpcd/nbdb/model.go rename to spinifex/network/ovn/nbdb/model.go index 46e4395ac..e1f5c32a1 100644 --- a/spinifex/services/vpcd/nbdb/model.go +++ b/spinifex/network/ovn/nbdb/model.go @@ -1,13 +1,5 @@ -// Package nbdb contains Go structs representing the OVN Northbound Database schema. -// These models are used with libovsdb to interact with the OVN NB DB. -// -// The structs cover the core tables needed for Spinifex VPC networking: -// LogicalSwitch, LogicalSwitchPort, LogicalRouter, LogicalRouterPort, and DHCPOptions. -// -// To regenerate from the full OVN NB schema (requires OVN installed): -// -// go install github.com/ovn-kubernetes/libovsdb/cmd/modelgen@latest -// modelgen -p nbdb -o spinifex/services/vpcd/nbdb /usr/share/ovn/ovn-nb.ovsschema +// Package nbdb contains libovsdb model structs for the OVN Northbound tables +// used by Spinifex VPC networking. package nbdb import "github.com/ovn-kubernetes/libovsdb/model" @@ -91,8 +83,7 @@ type LogicalRouterStaticRoute struct { ExternalIDs map[string]string `ovsdb:"external_ids"` } -// PortGroup represents an OVN Port_Group for grouping logical switch ports -// (used for security group enforcement). +// PortGroup represents an OVN Port_Group (used for SG enforcement). type PortGroup struct { UUID string `ovsdb:"_uuid"` Name string `ovsdb:"name"` @@ -115,9 +106,8 @@ type ACL struct { ExternalIDs map[string]string `ovsdb:"external_ids"` } -// AddressSet represents an OVN Address_Set — a named set of IP addresses -// referenced by ACL match expressions (e.g. ip4.src == $sg_xxx_ip4) for -// SG-to-SG rule enforcement. +// AddressSet represents an OVN Address_Set referenced by ACL match exprs +// (e.g. ip4.src == $sg_xxx_ip4) for SG-to-SG rules. type AddressSet struct { UUID string `ovsdb:"_uuid"` Name string `ovsdb:"name"` @@ -135,8 +125,7 @@ type GatewayChassis struct { Options map[string]string `ovsdb:"options"` } -// FullDatabaseModel returns a ClientDBModel for the OVN Northbound database -// containing all tables needed for Spinifex VPC networking. +// FullDatabaseModel returns a ClientDBModel for the OVN NB tables used. func FullDatabaseModel() (model.ClientDBModel, error) { return model.NewClientDBModel("OVN_Northbound", map[string]model.Model{ "Logical_Switch": &LogicalSwitch{}, diff --git a/spinifex/services/vpcd/nbdb/model_test.go b/spinifex/network/ovn/nbdb/model_test.go similarity index 100% rename from spinifex/services/vpcd/nbdb/model_test.go rename to spinifex/network/ovn/nbdb/model_test.go diff --git a/spinifex/network/policy/acl.go b/spinifex/network/policy/acl.go new file mode 100644 index 000000000..16d43e2c2 --- /dev/null +++ b/spinifex/network/policy/acl.go @@ -0,0 +1,187 @@ +package policy + +import ( + "fmt" + "strings" + + "github.com/mulgadc/spinifex/spinifex/network/ovn" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +// ACL priority bands. Higher wins. Tenants only ever populate 1000. +const ( + // ACLPriorityAllowIntraSG: unconditional intra-PG allow (reserved). + ACLPriorityAllowIntraSG = 1100 + + // ACLPriorityAllowDHCP must sit above tenant rules so a narrow-egress SG + // doesn't drop DHCPDISCOVER against the 900 default-deny. AWS users + // can't write a 255.255.255.255 rule. + ACLPriorityAllowDHCP = 1050 + + // ACLPriorityTenantAllow: tenant ingress/egress allows. Not logged. + ACLPriorityTenantAllow = 1000 + + // ACLPriorityDefaultDenyIngress: logged drop. CMMC SC.L1-3.13.1. + ACLPriorityDefaultDenyIngress = 900 + + // ACLPriorityDefaultDenyEgress: egress default-deny. + ACLPriorityDefaultDenyEgress = 800 +) + +// denyACLSeverity: syslog severity for default-deny hits. "info" captures +// without paging on port scans; operators can promote at the collector. +const denyACLSeverity = "info" + +// Rule is the policy-layer view of an AWS-style SG rule. CIDR / SourceSG +// MUST be validated upstream; values are interpolated verbatim. +type Rule struct { + IPProtocol string // "tcp", "udp", "icmp", or "-1" (all) + FromPort int64 + ToPort int64 + CIDR string + SourceSG string +} + +// InfrastructureACLs returns the platform ACLs every PG carries: the +// 900/800 logged default-denies (CMMC SC.L1-3.13.1) and the 1050 DHCPv4 +// client/server allows (OVN's DHCP responder runs after the PG ACL stage; +// without these a narrow-egress SG would drop DHCPDISCOVER). +func InfrastructureACLs(portGroupName string) []ovn.ACLSpec { + return []ovn.ACLSpec{ + denyIngressACL(portGroupName), + denyEgressACL(portGroupName), + dhcpEgressACL(portGroupName), + dhcpIngressACL(portGroupName), + } +} + +// RuleACLSpecs builds priority-1000 allow ACLs. Action is "allow-related" +// so reply traffic on established flows passes without a reverse rule. +func RuleACLSpecs(portGroupName string, ingress, egress []Rule) []ovn.ACLSpec { + specs := make([]ovn.ACLSpec, 0, len(ingress)+len(egress)) + for _, rule := range ingress { + specs = append(specs, ovn.ACLSpec{ + Direction: "to-lport", + Priority: ACLPriorityTenantAllow, + Match: BuildIngressACLMatch(portGroupName, rule), + Action: "allow-related", + }) + } + for _, rule := range egress { + specs = append(specs, ovn.ACLSpec{ + Direction: "from-lport", + Priority: ACLPriorityTenantAllow, + Match: BuildEgressACLMatch(portGroupName, rule), + Action: "allow-related", + }) + } + return specs +} + +// BuildIngressACLMatch builds an OVN to-lport match. Tenant CIDR/SourceSG +// MUST be validated upstream; values are interpolated verbatim. +func BuildIngressACLMatch(portGroupName string, rule Rule) string { + parts := []string{fmt.Sprintf("outport == @%s", portGroupName), "ip4"} + parts = appendProtocolMatch(parts, rule) + if rule.CIDR != "" && rule.CIDR != "0.0.0.0/0" { + parts = append(parts, fmt.Sprintf("ip4.src == %s", rule.CIDR)) + } + if rule.SourceSG != "" { + parts = append(parts, fmt.Sprintf("ip4.src == $%s", addressSetName(topology.SecurityGroupPortGroup(rule.SourceSG)))) + } + return strings.Join(parts, " && ") +} + +// BuildEgressACLMatch builds an OVN from-lport match. +func BuildEgressACLMatch(portGroupName string, rule Rule) string { + parts := []string{fmt.Sprintf("inport == @%s", portGroupName), "ip4"} + parts = appendProtocolMatch(parts, rule) + if rule.CIDR != "" && rule.CIDR != "0.0.0.0/0" { + parts = append(parts, fmt.Sprintf("ip4.dst == %s", rule.CIDR)) + } + if rule.SourceSG != "" { + parts = append(parts, fmt.Sprintf("ip4.dst == $%s", addressSetName(topology.SecurityGroupPortGroup(rule.SourceSG)))) + } + return strings.Join(parts, " && ") +} + +// addressSetName returns the ovn-northd-derived SB Address_Set name for a +// PG's IPv4 addresses. Spinifex must NOT create NB Address_Set rows under +// these names — duplicate SB sync wedges northd in a commit-failed loop. +func addressSetName(portGroupName string) string { + return portGroupName + "_ip4" +} + +func appendProtocolMatch(parts []string, rule Rule) []string { + switch rule.IPProtocol { + case "tcp": + parts = appendPortMatch(parts, "tcp", rule.FromPort, rule.ToPort) + case "udp": + parts = appendPortMatch(parts, "udp", rule.FromPort, rule.ToPort) + case "icmp": + parts = append(parts, "icmp4") + case "-1", "": + default: + } + return parts +} + +func appendPortMatch(parts []string, proto string, fromPort, toPort int64) []string { + if fromPort == 0 && toPort == 0 { + return append(parts, proto) + } + if fromPort == toPort { + return append(parts, fmt.Sprintf("%s.dst == %d", proto, fromPort)) + } + parts = append(parts, fmt.Sprintf("%s.dst >= %d", proto, fromPort)) + parts = append(parts, fmt.Sprintf("%s.dst <= %d", proto, toPort)) + return parts +} + +func denyIngressACL(portGroupName string) ovn.ACLSpec { + return ovn.ACLSpec{ + Direction: "to-lport", + Priority: ACLPriorityDefaultDenyIngress, + Match: fmt.Sprintf("outport == @%s && ip4", portGroupName), + Action: "drop", + Name: portGroupName + "-deny-ingress", + Log: true, + Severity: denyACLSeverity, + } +} + +func denyEgressACL(portGroupName string) ovn.ACLSpec { + return ovn.ACLSpec{ + Direction: "from-lport", + Priority: ACLPriorityDefaultDenyEgress, + Match: fmt.Sprintf("inport == @%s && ip4", portGroupName), + Action: "drop", + Name: portGroupName + "-deny-egress", + Log: true, + Severity: denyACLSeverity, + } +} + +// dhcpEgressACL: DHCPDISCOVER/REQUEST out (udp 68→67). "allow" not +// "allow-related" — broadcast UDP interacts oddly with OVN CT zones. +func dhcpEgressACL(portGroupName string) ovn.ACLSpec { + return ovn.ACLSpec{ + Direction: "from-lport", + Priority: ACLPriorityAllowDHCP, + Match: fmt.Sprintf("inport == @%s && udp && udp.src == 68 && udp.dst == 67", portGroupName), + Action: "allow", + Name: portGroupName + "-allow-dhcp-egress", + } +} + +// dhcpIngressACL: DHCPOFFER/ACK reply (udp 67→68). Required because OVN's +// DHCP responder emits inside the LS pipeline after the ACL stage. +func dhcpIngressACL(portGroupName string) ovn.ACLSpec { + return ovn.ACLSpec{ + Direction: "to-lport", + Priority: ACLPriorityAllowDHCP, + Match: fmt.Sprintf("outport == @%s && udp && udp.src == 67 && udp.dst == 68", portGroupName), + Action: "allow", + Name: portGroupName + "-allow-dhcp-ingress", + } +} diff --git a/spinifex/network/policy/acl_test.go b/spinifex/network/policy/acl_test.go new file mode 100644 index 000000000..bb7f773ab --- /dev/null +++ b/spinifex/network/policy/acl_test.go @@ -0,0 +1,136 @@ +package policy + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestACL_TCPPortFromCIDR(t *testing.T) { + match := BuildIngressACLMatch("sg_test", Rule{ + IPProtocol: "tcp", + FromPort: 22, + ToPort: 22, + CIDR: "10.0.0.0/8", + }) + assert.Contains(t, match, "tcp.dst == 22") + assert.Contains(t, match, "ip4.src == 10.0.0.0/8") + assert.Contains(t, match, "outport == @sg_test") + assert.Contains(t, match, "ip4") +} + +func TestACL_AllTrafficFromSG(t *testing.T) { + match := BuildIngressACLMatch("sg_test", Rule{ + IPProtocol: "-1", + SourceSG: "sg-abc123", + }) + assert.Contains(t, match, "ip4.src == $sg_abc123_ip4") + assert.Contains(t, match, "outport == @sg_test") + assert.Contains(t, match, "ip4") +} + +func TestACL_PortRange(t *testing.T) { + match := BuildIngressACLMatch("sg_test", Rule{ + IPProtocol: "udp", + FromPort: 1024, + ToPort: 65535, + }) + assert.Contains(t, match, "udp.dst >= 1024") + assert.Contains(t, match, "udp.dst <= 65535") +} + +func TestACL_ICMP(t *testing.T) { + match := BuildIngressACLMatch("sg_test", Rule{ + IPProtocol: "icmp", + CIDR: "0.0.0.0/0", + }) + assert.Contains(t, match, "icmp4") + assert.NotContains(t, match, "tcp.dst") + assert.NotContains(t, match, "udp.dst") +} + +func TestACL_AllProtocols(t *testing.T) { + match := BuildIngressACLMatch("sg_test", Rule{ + IPProtocol: "-1", + CIDR: "10.0.0.0/16", + }) + assert.Contains(t, match, "ip4") + assert.Contains(t, match, "ip4.src == 10.0.0.0/16") + assert.NotContains(t, match, "tcp") + assert.NotContains(t, match, "udp") + assert.NotContains(t, match, "icmp") +} + +func TestACL_EgressAll(t *testing.T) { + match := BuildEgressACLMatch("sg_test", Rule{ + IPProtocol: "-1", + CIDR: "0.0.0.0/0", + }) + assert.Contains(t, match, "inport == @sg_test") + assert.NotContains(t, match, "outport") + assert.Contains(t, match, "ip4") +} + +func TestACL_TCPSinglePort(t *testing.T) { + match := BuildIngressACLMatch("sg_test", Rule{ + IPProtocol: "tcp", + FromPort: 443, + ToPort: 443, + CIDR: "10.0.0.0/8", + }) + assert.Contains(t, match, "tcp.dst == 443") + assert.NotContains(t, match, "tcp.dst >=") + assert.NotContains(t, match, "tcp.dst <=") +} + +func TestACL_NoSource(t *testing.T) { + match := BuildIngressACLMatch("sg_test", Rule{ + IPProtocol: "tcp", + FromPort: 80, + ToPort: 80, + CIDR: "0.0.0.0/0", + }) + assert.Contains(t, match, "tcp.dst == 80") + assert.NotContains(t, match, "ip4.src") +} + +func TestACL_EgressFromSGToSG(t *testing.T) { + match := BuildEgressACLMatch("sg_test", Rule{ + IPProtocol: "tcp", + FromPort: 3306, + ToPort: 3306, + SourceSG: "sg-db-tier", + }) + assert.Contains(t, match, "inport == @sg_test") + assert.Contains(t, match, "tcp.dst == 3306") + assert.Contains(t, match, "ip4.dst == $sg_db_tier_ip4") +} + +func TestInfrastructureACLs_Shape(t *testing.T) { + specs := InfrastructureACLs("sg_test") + if assert.Len(t, specs, 4) { + // Priorities: deny-ingress 900, deny-egress 800, dhcp-egress 1050, dhcp-ingress 1050 + assert.Equal(t, ACLPriorityDefaultDenyIngress, specs[0].Priority) + assert.Equal(t, "drop", specs[0].Action) + assert.True(t, specs[0].Log) + assert.Equal(t, ACLPriorityDefaultDenyEgress, specs[1].Priority) + assert.True(t, specs[1].Log) + assert.Equal(t, ACLPriorityAllowDHCP, specs[2].Priority) + assert.Equal(t, "allow", specs[2].Action) + assert.Equal(t, ACLPriorityAllowDHCP, specs[3].Priority) + } +} + +func TestRuleACLSpecs_PriorityAndAction(t *testing.T) { + specs := RuleACLSpecs("sg_test", + []Rule{{IPProtocol: "tcp", FromPort: 80, ToPort: 80, CIDR: "0.0.0.0/0"}}, + []Rule{{IPProtocol: "-1", CIDR: "0.0.0.0/0"}}, + ) + if assert.Len(t, specs, 2) { + assert.Equal(t, ACLPriorityTenantAllow, specs[0].Priority) + assert.Equal(t, "to-lport", specs[0].Direction) + assert.Equal(t, "allow-related", specs[0].Action) + assert.Equal(t, "from-lport", specs[1].Direction) + assert.Equal(t, "allow-related", specs[1].Action) + } +} diff --git a/spinifex/network/policy/nat.go b/spinifex/network/policy/nat.go new file mode 100644 index 000000000..a9d281ec7 --- /dev/null +++ b/spinifex/network/policy/nat.go @@ -0,0 +1,221 @@ +package policy + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/mulgadc/spinifex/spinifex/network/ovn" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +// EIPSpec is a 1:1 EIP NAT (dnat_and_snat). In NATModeDistributed PortName +// and MAC MUST be set for per-chassis flows; missing values fall back to +// centralised shape (hairpins via gateway chassis). +type EIPSpec struct { + VPCID string + ExternalIP string + LogicalIP string + PortName string + MAC string +} + +// NATGWSpec is a NAT Gateway SNAT rule keyed by (snat, SubnetCIDR). +type NATGWSpec struct { + VPCID string + NATGatewayID string + PublicIP string + SubnetCIDR string +} + +// NATManager owns NAT-rule lifecycle. Mode is fixed at construction. +type NATManager interface { + // AddEIP installs a dnat_and_snat rule, cleaning stale rules for the + // same ExternalIP on any router first (pool reuse across VPCs). + AddEIP(ctx context.Context, eip EIPSpec) error + + // DeleteEIP removes the rule by LogicalIP; idempotent. + DeleteEIP(ctx context.Context, vpcID, logicalIP string) error + + // AddNATGateway installs the (snat, SubnetCIDR) rule; rejects overlap. + AddNATGateway(ctx context.Context, gw NATGWSpec) error + + // DeleteNATGateway removes the (snat, SubnetCIDR) rule; idempotent. + DeleteNATGateway(ctx context.Context, vpcID, subnetCIDR string) error + + // AddSNAT installs the IGW default-outbound snat rewriting VPC CIDR + // to ExternalIP. AWS-parity callers typically skip this. + AddSNAT(ctx context.Context, vpcID, vpcCIDR, externalIP string) error + + // DeleteSNAT removes the IGW default-outbound snat for vpcCIDR. + DeleteSNAT(ctx context.Context, vpcID, vpcCIDR string) error +} + +// FlowsBarrier blocks until ovn-northd has compiled NB → SB and every +// chassis installed flows. Production wires a closure over +// `ovn-nbctl --wait=hv sync`; tests leave it nil. +type FlowsBarrier func() error + +type Option func(*natManager) + +// WithFlowsBarrier injects the post-write flow-install barrier so callers +// only see success once every chassis has the rewrite flow. +func WithFlowsBarrier(b FlowsBarrier) Option { + return func(m *natManager) { + if b != nil { + m.barrier = b + } + } +} + +type natManager struct { + ovn ovn.Client + mode NATMode + barrier FlowsBarrier +} + +var _ NATManager = (*natManager)(nil) + +// NewNATManager constructs a NATManager bound to one NAT mode (must not be +// NATModeUnknown). +func NewNATManager(client ovn.Client, mode NATMode, opts ...Option) (NATManager, error) { + if mode == NATModeUnknown { + return nil, fmt.Errorf("NAT mode is unknown; resolve from host.Wiring.UplinkMode()") + } + m := &natManager{ + ovn: client, + mode: mode, + barrier: func() error { return nil }, + } + for _, opt := range opts { + opt(m) + } + return m, nil +} + +func (m *natManager) AddEIP(ctx context.Context, eip EIPSpec) error { + router := topology.VPCRouter(eip.VPCID) + + natRule := &nbdb.NAT{ + Type: "dnat_and_snat", + ExternalIP: eip.ExternalIP, + LogicalIP: eip.LogicalIP, + ExternalIDs: map[string]string{ + "spinifex:vpc_id": eip.VPCID, + "spinifex:public_ip": eip.ExternalIP, + }, + } + distributed := m.mode == NATModeDistributed && eip.PortName != "" && eip.MAC != "" + if distributed { + mac := eip.MAC + port := eip.PortName + natRule.ExternalMAC = &mac + natRule.LogicalPort = &port + } + + // Skip when the existing row already matches; avoids the + // delete-then-add flow-install gap on duplicate publishes. + if existing, err := m.ovn.FindNATByExternalIP(ctx, "dnat_and_snat", eip.ExternalIP); err != nil { + slog.Warn("policy: AddEIP idempotency lookup failed", "external_ip", eip.ExternalIP, "err", err) + } else if existing != nil && existing.LogicalIP == eip.LogicalIP && + existing.ExternalIDs["spinifex:vpc_id"] == eip.VPCID && + (!distributed || + (existing.ExternalMAC != nil && *existing.ExternalMAC == eip.MAC && + existing.LogicalPort != nil && *existing.LogicalPort == eip.PortName)) { + slog.Info("policy: AddEIP idempotent skip — rule already current", + "router", router, "external_ip", eip.ExternalIP, "logical_ip", eip.LogicalIP) + return nil + } + + // Search every router for stale rules — vpc.delete-nat is + // fire-and-forget and may not have run before IP reuse. + if removed, err := m.ovn.DeleteAllNATsByExternalIP(ctx, "dnat_and_snat", eip.ExternalIP); err != nil { + slog.Warn("policy: stale NAT cleanup failed before AddEIP", "external_ip", eip.ExternalIP, "err", err) + } else if removed > 0 { + slog.Info("policy: cleaned stale dnat_and_snat rules before AddEIP", "external_ip", eip.ExternalIP, "removed", removed) + } + + if err := m.ovn.AddNAT(ctx, router, natRule); err != nil { + return fmt.Errorf("add dnat_and_snat %s -> %s on %s: %w", eip.LogicalIP, eip.ExternalIP, router, err) + } + if err := m.barrier(); err != nil { + slog.Warn("policy: AddEIP flows barrier failed", "external_ip", eip.ExternalIP, "logical_ip", eip.LogicalIP, "err", err) + } + return nil +} + +func (m *natManager) DeleteEIP(ctx context.Context, vpcID, logicalIP string) error { + router := topology.VPCRouter(vpcID) + if err := m.ovn.DeleteNAT(ctx, router, "dnat_and_snat", logicalIP); err != nil { + if errors.Is(err, ovn.ErrNATNotFound) { + return nil + } + return fmt.Errorf("delete dnat_and_snat %s on %s: %w", logicalIP, router, err) + } + return nil +} + +func (m *natManager) AddNATGateway(ctx context.Context, gw NATGWSpec) error { + router := topology.VPCRouter(gw.VPCID) + + snatRule := &nbdb.NAT{ + Type: "snat", + ExternalIP: gw.PublicIP, + LogicalIP: gw.SubnetCIDR, + ExternalIDs: map[string]string{ + "spinifex:vpc_id": gw.VPCID, + "spinifex:nat_gateway_id": gw.NATGatewayID, + }, + } + if err := m.ovn.AddNAT(ctx, router, snatRule); err != nil { + return fmt.Errorf("add NAT GW snat %s -> %s on %s: %w", gw.SubnetCIDR, gw.PublicIP, router, err) + } + // Block until SB + chassis have the SNAT flow; otherwise first packets + // from the private subnet drop. + if err := m.barrier(); err != nil { + slog.Warn("policy: AddNATGateway flows barrier failed", + "public_ip", gw.PublicIP, "subnet_cidr", gw.SubnetCIDR, "err", err) + } + return nil +} + +func (m *natManager) DeleteNATGateway(ctx context.Context, vpcID, subnetCIDR string) error { + router := topology.VPCRouter(vpcID) + if err := m.ovn.DeleteNAT(ctx, router, "snat", subnetCIDR); err != nil { + if errors.Is(err, ovn.ErrNATNotFound) { + return nil + } + return fmt.Errorf("delete NAT GW snat %s on %s: %w", subnetCIDR, router, err) + } + return nil +} + +func (m *natManager) AddSNAT(ctx context.Context, vpcID, vpcCIDR, externalIP string) error { + router := topology.VPCRouter(vpcID) + snatRule := &nbdb.NAT{ + Type: "snat", + ExternalIP: externalIP, + LogicalIP: vpcCIDR, + ExternalIDs: map[string]string{ + "spinifex:vpc_id": vpcID, + "spinifex:role": "igw-default-snat", + }, + } + if err := m.ovn.AddNAT(ctx, router, snatRule); err != nil { + return fmt.Errorf("add IGW snat %s -> %s on %s: %w", vpcCIDR, externalIP, router, err) + } + return nil +} + +func (m *natManager) DeleteSNAT(ctx context.Context, vpcID, vpcCIDR string) error { + router := topology.VPCRouter(vpcID) + if err := m.ovn.DeleteNAT(ctx, router, "snat", vpcCIDR); err != nil { + if errors.Is(err, ovn.ErrNATNotFound) { + return nil + } + return fmt.Errorf("delete IGW snat %s on %s: %w", vpcCIDR, router, err) + } + return nil +} diff --git a/spinifex/network/policy/nat_test.go b/spinifex/network/policy/nat_test.go new file mode 100644 index 000000000..18b03b481 --- /dev/null +++ b/spinifex/network/policy/nat_test.go @@ -0,0 +1,320 @@ +package policy + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mulgadc/spinifex/spinifex/network/host" + "github.com/mulgadc/spinifex/spinifex/network/ovn/mock" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +func seedRouter(t *testing.T, m *mock.Client, vpcID string) string { + t.Helper() + name := topology.VPCRouter(vpcID) + require.NoError(t, m.CreateLogicalRouter(context.Background(), &nbdb.LogicalRouter{Name: name})) + return name +} + +func findNAT(m *mock.Client, natType, logicalIP string) *nbdb.NAT { + for _, n := range m.NATs { + if n.Type == natType && n.LogicalIP == logicalIP { + return n + } + } + return nil +} + +func TestNATModeFromUplinkMode(t *testing.T) { + assert.Equal(t, NATModeDistributed, NATModeFromUplinkMode(host.UplinkModePhysical)) + assert.Equal(t, NATModeCentralized, NATModeFromUplinkMode(host.UplinkModeVeth)) + assert.Equal(t, NATModeUnknown, NATModeFromUplinkMode(host.UplinkModeUnknown)) +} + +func TestNewNATManager_RejectsUnknownMode(t *testing.T) { + _, err := NewNATManager(mock.New(), NATModeUnknown) + require.Error(t, err) +} + +func TestNATManager_AddEIP_FlowsBarrier_Fires(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + var calls int + nm, err := NewNATManager(m, NATModeDistributed, WithFlowsBarrier(func() error { + calls++ + return nil + })) + require.NoError(t, err) + + require.NoError(t, nm.AddEIP(ctx, EIPSpec{ + VPCID: "vpc-1", + ExternalIP: "1.2.3.4", + LogicalIP: "10.0.0.5", + PortName: "port-eni-abc", + MAC: "aa:bb:cc:dd:ee:ff", + })) + assert.Equal(t, 1, calls, "FlowsBarrier must fire once per AddEIP") +} + +func TestNATManager_AddEIP_IdempotentSkip(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + var barrierCalls int + nm, err := NewNATManager(m, NATModeDistributed, WithFlowsBarrier(func() error { + barrierCalls++ + return nil + })) + require.NoError(t, err) + + spec := EIPSpec{ + VPCID: "vpc-1", ExternalIP: "1.2.3.4", LogicalIP: "10.0.0.5", + PortName: "port-eni-abc", MAC: "aa:bb:cc:dd:ee:ff", + } + require.NoError(t, nm.AddEIP(ctx, spec)) + require.Equal(t, 1, barrierCalls, "first AddEIP must fire barrier") + firstUUID := findNAT(m, "dnat_and_snat", spec.LogicalIP).UUID + + // Second AddEIP with the same spec must skip delete-then-add. UUID + // staying constant confirms the existing row was not replaced. + require.NoError(t, nm.AddEIP(ctx, spec)) + assert.Equal(t, firstUUID, findNAT(m, "dnat_and_snat", spec.LogicalIP).UUID, + "idempotent re-add must reuse the existing NAT row") + assert.Equal(t, 1, barrierCalls, "FlowsBarrier must not fire on idempotent skip") +} + +func TestNATManager_AddEIP_NoBarrier_Default(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + nm, err := NewNATManager(m, NATModeDistributed) + require.NoError(t, err) + + require.NoError(t, nm.AddEIP(ctx, EIPSpec{ + VPCID: "vpc-1", ExternalIP: "1.2.3.4", LogicalIP: "10.0.0.5", + })) +} + +func TestNATManager_AddEIP_Distributed_SetsPortAndMAC(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + nm, err := NewNATManager(m, NATModeDistributed) + require.NoError(t, err) + + require.NoError(t, nm.AddEIP(ctx, EIPSpec{ + VPCID: "vpc-1", + ExternalIP: "1.2.3.4", + LogicalIP: "10.0.0.5", + PortName: "port-eni-abc", + MAC: "aa:bb:cc:dd:ee:ff", + })) + + got := findNAT(m, "dnat_and_snat", "10.0.0.5") + require.NotNil(t, got) + require.NotNil(t, got.LogicalPort) + require.NotNil(t, got.ExternalMAC) + assert.Equal(t, "port-eni-abc", *got.LogicalPort) + assert.Equal(t, "aa:bb:cc:dd:ee:ff", *got.ExternalMAC) +} + +func TestNATManager_AddEIP_Centralized_LeavesPortAndMACUnset(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + nm, err := NewNATManager(m, NATModeCentralized) + require.NoError(t, err) + + require.NoError(t, nm.AddEIP(ctx, EIPSpec{ + VPCID: "vpc-1", + ExternalIP: "1.2.3.4", + LogicalIP: "10.0.0.5", + PortName: "port-eni-abc", + MAC: "aa:bb:cc:dd:ee:ff", + })) + + got := findNAT(m, "dnat_and_snat", "10.0.0.5") + require.NotNil(t, got) + assert.Nil(t, got.LogicalPort) + assert.Nil(t, got.ExternalMAC) +} + +func TestNATManager_AddEIP_RemovesStaleRuleOnOtherRouter(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-old") + seedRouter(t, m, "vpc-new") + nm, err := NewNATManager(m, NATModeDistributed) + require.NoError(t, err) + + // Stale rule still attached to vpc-old's router for the same external IP. + require.NoError(t, nm.AddEIP(ctx, EIPSpec{ + VPCID: "vpc-old", + ExternalIP: "1.2.3.4", + LogicalIP: "10.0.0.5", + PortName: "port-a", MAC: "aa:aa:aa:aa:aa:aa", + })) + // Reuse the EIP in vpc-new. + require.NoError(t, nm.AddEIP(ctx, EIPSpec{ + VPCID: "vpc-new", + ExternalIP: "1.2.3.4", + LogicalIP: "10.1.0.7", + PortName: "port-b", MAC: "bb:bb:bb:bb:bb:bb", + })) + + // Only the new rule survives. + var matching []*nbdb.NAT + for _, n := range m.NATs { + if n.Type == "dnat_and_snat" && n.ExternalIP == "1.2.3.4" { + matching = append(matching, n) + } + } + require.Len(t, matching, 1) + assert.Equal(t, "10.1.0.7", matching[0].LogicalIP) +} + +func TestNATManager_DeleteEIP_IdempotentOnMissing(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + nm, _ := NewNATManager(m, NATModeDistributed) + + // No prior AddEIP — delete should still succeed. + require.NoError(t, nm.DeleteEIP(ctx, "vpc-1", "10.0.0.5")) +} + +func TestNATManager_AddNATGateway_FlowsBarrier_Fires(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + var calls int + nm, err := NewNATManager(m, NATModeCentralized, WithFlowsBarrier(func() error { + calls++ + return nil + })) + require.NoError(t, err) + + require.NoError(t, nm.AddNATGateway(ctx, NATGWSpec{ + VPCID: "vpc-1", + NATGatewayID: "nat-xyz", + PublicIP: "9.9.9.9", + SubnetCIDR: "10.0.1.0/24", + })) + assert.Equal(t, 1, calls, "FlowsBarrier must fire once per AddNATGateway") +} + +func TestNATManager_AddNATGateway_AndDelete(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + nm, _ := NewNATManager(m, NATModeCentralized) + + require.NoError(t, nm.AddNATGateway(ctx, NATGWSpec{ + VPCID: "vpc-1", + NATGatewayID: "nat-xyz", + PublicIP: "9.9.9.9", + SubnetCIDR: "10.0.1.0/24", + })) + got := findNAT(m, "snat", "10.0.1.0/24") + require.NotNil(t, got) + assert.Equal(t, "9.9.9.9", got.ExternalIP) + assert.Equal(t, "nat-xyz", got.ExternalIDs["spinifex:nat_gateway_id"]) + + require.NoError(t, nm.DeleteNATGateway(ctx, "vpc-1", "10.0.1.0/24")) + assert.Nil(t, findNAT(m, "snat", "10.0.1.0/24")) + + // Idempotent re-delete. + require.NoError(t, nm.DeleteNATGateway(ctx, "vpc-1", "10.0.1.0/24")) +} + +// TestNATManager_DeleteNATGateway_CrossRouterIsolation guards against a NAT +// row owned by router B being mutated/deleted when DeleteNATGateway is called +// on router A and both routers carry the same subnet CIDR. AWS allows subnet +// CIDRs to repeat per-VPC, so (snat, logicalIP) is not globally unique. Bug +// observed in CI Phase 8d: shared 172.31.x CIDRs across VPCs caused +// DeleteEIP/DeleteNAT to hit the wrong NAT row, leaving orphaned rules that +// corrupted ARP/conntrack for the surviving NATGW EIP. +func TestNATManager_DeleteNATGateway_CrossRouterIsolation(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-a") + seedRouter(t, m, "vpc-b") + nm, err := NewNATManager(m, NATModeCentralized) + require.NoError(t, err) + + // Same subnet CIDR routed via NATGW in two different VPCs. + require.NoError(t, nm.AddNATGateway(ctx, NATGWSpec{ + VPCID: "vpc-a", NATGatewayID: "nat-a", PublicIP: "9.9.9.9", SubnetCIDR: "172.31.0.0/20", + })) + require.NoError(t, nm.AddNATGateway(ctx, NATGWSpec{ + VPCID: "vpc-b", NATGatewayID: "nat-b", PublicIP: "8.8.8.8", SubnetCIDR: "172.31.0.0/20", + })) + + require.NoError(t, nm.DeleteNATGateway(ctx, "vpc-a", "172.31.0.0/20")) + + // vpc-b's snat must remain — only vpc-a's was deleted. + var survived *nbdb.NAT + for _, n := range m.NATs { + if n.Type == "snat" && n.LogicalIP == "172.31.0.0/20" { + survived = n + } + } + require.NotNil(t, survived, "vpc-b snat row must remain after vpc-a delete") + assert.Equal(t, "8.8.8.8", survived.ExternalIP, "surviving row must belong to vpc-b") + assert.Equal(t, "nat-b", survived.ExternalIDs["spinifex:nat_gateway_id"]) +} + +// TestNATManager_DeleteEIP_CrossRouterIsolation is the dnat_and_snat analogue +// of the cross-router NATGW isolation test. Two VPCs each with a private VM +// at 10.0.0.5 (legal — VPC CIDRs may overlap). DeleteEIP on vpc-a must not +// touch vpc-b's NAT row. +func TestNATManager_DeleteEIP_CrossRouterIsolation(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-a") + seedRouter(t, m, "vpc-b") + nm, err := NewNATManager(m, NATModeDistributed) + require.NoError(t, err) + + require.NoError(t, nm.AddEIP(ctx, EIPSpec{ + VPCID: "vpc-a", ExternalIP: "1.1.1.1", LogicalIP: "10.0.0.5", + PortName: "port-a", MAC: "aa:aa:aa:aa:aa:aa", + })) + require.NoError(t, nm.AddEIP(ctx, EIPSpec{ + VPCID: "vpc-b", ExternalIP: "2.2.2.2", LogicalIP: "10.0.0.5", + PortName: "port-b", MAC: "bb:bb:bb:bb:bb:bb", + })) + + require.NoError(t, nm.DeleteEIP(ctx, "vpc-a", "10.0.0.5")) + + var survived *nbdb.NAT + for _, n := range m.NATs { + if n.Type == "dnat_and_snat" && n.LogicalIP == "10.0.0.5" { + survived = n + } + } + require.NotNil(t, survived, "vpc-b dnat_and_snat row must remain after vpc-a delete") + assert.Equal(t, "2.2.2.2", survived.ExternalIP, "surviving row must belong to vpc-b") +} + +func TestNATManager_AddSNAT_AndDelete(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + nm, _ := NewNATManager(m, NATModeDistributed) + + require.NoError(t, nm.AddSNAT(ctx, "vpc-1", "10.0.0.0/16", "1.2.3.4")) + got := findNAT(m, "snat", "10.0.0.0/16") + require.NotNil(t, got) + assert.Equal(t, "1.2.3.4", got.ExternalIP) + assert.Equal(t, "igw-default-snat", got.ExternalIDs["spinifex:role"]) + + require.NoError(t, nm.DeleteSNAT(ctx, "vpc-1", "10.0.0.0/16")) + assert.Nil(t, findNAT(m, "snat", "10.0.0.0/16")) + require.NoError(t, nm.DeleteSNAT(ctx, "vpc-1", "10.0.0.0/16")) +} diff --git a/spinifex/network/policy/policy.go b/spinifex/network/policy/policy.go new file mode 100644 index 000000000..7a7434847 --- /dev/null +++ b/spinifex/network/policy/policy.go @@ -0,0 +1,50 @@ +// Package policy (L3) attaches data-plane policy — ACLs, NAT, routes — to +// L2 logical objects. It never creates or deletes L2 objects; missing +// targets surface as L1 "not found" errors. +package policy + +import ( + "github.com/mulgadc/spinifex/spinifex/network/host" +) + +// NATMode selects distributed vs. centralised NAT (ADR-0006 S3). Fixed at +// NATManager construction from the L0 uplink mode. +type NATMode int + +const ( + NATModeUnknown NATMode = iota + + // NATModeDistributed sets ExternalMAC + LogicalPort on every + // dnat_and_snat so OVN processes DNAT on the VM's own chassis. Required + // by UplinkModePhysical. + NATModeDistributed + + // NATModeCentralized leaves ExternalMAC/LogicalPort unset so the gateway + // chassis owns SNAT/DNAT. Required by UplinkModeVeth (bridge breaks + // distributed-NAT hairpin routing). + NATModeCentralized +) + +func (m NATMode) String() string { + switch m { + case NATModeDistributed: + return "distributed" + case NATModeCentralized: + return "centralized" + default: + return "unknown" + } +} + +// NATModeFromUplinkMode maps L0 uplink mode to NAT mode. Unknown maps to +// NATModeUnknown so misconfiguration fails loudly at construction. +func NATModeFromUplinkMode(m host.UplinkMode) NATMode { + switch m { + case host.UplinkModePhysical: + return NATModeDistributed + case host.UplinkModeVeth: + return NATModeCentralized + default: + return NATModeUnknown + } +} diff --git a/spinifex/network/policy/route.go b/spinifex/network/policy/route.go new file mode 100644 index 000000000..c7ad83dde --- /dev/null +++ b/spinifex/network/policy/route.go @@ -0,0 +1,125 @@ +package policy + +import ( + "context" + "fmt" + "net/netip" + + "github.com/mulgadc/spinifex/spinifex/network/ovn" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +const defaultRoutePrefix = "0.0.0.0/0" + +// RouteSpec is a static route on a VPC's LogicalRouter. OutputPort is +// required when Nexthop isn't directly-connected (e.g. IGW default route on +// 169.254.0.1/30) — ovn-northd silently drops the route from SB otherwise. +type RouteSpec struct { + Prefix netip.Prefix + Nexthop string + OutputPort string +} + +// RouteManager owns static routes on VPC LogicalRouters. Adds are +// idempotent (no-op on match, delete-then-add on drift) so the reconciler +// can replay safely. +type RouteManager interface { + AddDefaultRoute(ctx context.Context, vpcID, nexthop, outputPort string) error + DeleteDefaultRoute(ctx context.Context, vpcID string) error + AddStaticRoute(ctx context.Context, vpcID string, route RouteSpec) error + DeleteStaticRoute(ctx context.Context, vpcID string, prefix netip.Prefix) error +} + +type routeManager struct { + ovn ovn.Client +} + +var _ RouteManager = (*routeManager)(nil) + +// NewRouteManager constructs a RouteManager backed by client. +func NewRouteManager(client ovn.Client) RouteManager { + return &routeManager{ovn: client} +} + +func (m *routeManager) AddDefaultRoute(ctx context.Context, vpcID, nexthop, outputPort string) error { + prefix, err := netip.ParsePrefix(defaultRoutePrefix) + if err != nil { + return fmt.Errorf("parse default route prefix: %w", err) + } + return m.AddStaticRoute(ctx, vpcID, RouteSpec{ + Prefix: prefix, + Nexthop: nexthop, + OutputPort: outputPort, + }) +} + +func (m *routeManager) DeleteDefaultRoute(ctx context.Context, vpcID string) error { + return m.deleteByPrefixString(ctx, vpcID, defaultRoutePrefix) +} + +func (m *routeManager) AddStaticRoute(ctx context.Context, vpcID string, route RouteSpec) error { + router := topology.VPCRouter(vpcID) + prefixStr := route.Prefix.String() + + existing, err := m.ovn.FindStaticRoute(ctx, router, prefixStr) + if err != nil { + return fmt.Errorf("find static route %s on %s: %w", prefixStr, router, err) + } + if existing != nil { + if routeMatches(existing, route) { + return nil + } + if err := m.ovn.DeleteStaticRoute(ctx, router, prefixStr); err != nil { + return fmt.Errorf("delete drifted static route %s on %s: %w", prefixStr, router, err) + } + } + + row := &nbdb.LogicalRouterStaticRoute{ + IPPrefix: prefixStr, + Nexthop: route.Nexthop, + } + if route.OutputPort != "" { + op := route.OutputPort + row.OutputPort = &op + } + if err := m.ovn.AddStaticRoute(ctx, router, row); err != nil { + return fmt.Errorf("add static route %s -> %s on %s: %w", prefixStr, route.Nexthop, router, err) + } + return nil +} + +func (m *routeManager) DeleteStaticRoute(ctx context.Context, vpcID string, prefix netip.Prefix) error { + return m.deleteByPrefixString(ctx, vpcID, prefix.String()) +} + +// deleteByPrefixString returns nil when the route is already absent (every +// Delete* entrypoint needs this for fire-and-forget event flows). +func (m *routeManager) deleteByPrefixString(ctx context.Context, vpcID, prefix string) error { + router := topology.VPCRouter(vpcID) + existing, err := m.ovn.FindStaticRoute(ctx, router, prefix) + if err != nil { + return fmt.Errorf("find static route %s on %s: %w", prefix, router, err) + } + if existing == nil { + return nil + } + if err := m.ovn.DeleteStaticRoute(ctx, router, prefix); err != nil { + return fmt.Errorf("delete static route %s on %s: %w", prefix, router, err) + } + return nil +} + +func routeMatches(existing *nbdb.LogicalRouterStaticRoute, want RouteSpec) bool { + if existing.Nexthop != want.Nexthop { + return false + } + switch { + case existing.OutputPort == nil && want.OutputPort == "": + return true + case existing.OutputPort == nil || want.OutputPort == "": + return false + default: + return *existing.OutputPort == want.OutputPort + } +} diff --git a/spinifex/network/policy/route_test.go b/spinifex/network/policy/route_test.go new file mode 100644 index 000000000..e16a11457 --- /dev/null +++ b/spinifex/network/policy/route_test.go @@ -0,0 +1,104 @@ +package policy + +import ( + "context" + "net/netip" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mulgadc/spinifex/spinifex/network/ovn/mock" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +func findRoute(m *mock.Client, prefix string) *nbdb.LogicalRouterStaticRoute { + for _, r := range m.StaticRoutes { + if r.IPPrefix == prefix { + return r + } + } + return nil +} + +func countRoutes(m *mock.Client, prefix string) int { + n := 0 + for _, r := range m.StaticRoutes { + if r.IPPrefix == prefix { + n++ + } + } + return n +} + +func TestRouteManager_AddDefaultRoute_WritesOutputPort(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + rm := NewRouteManager(m) + + require.NoError(t, rm.AddDefaultRoute(ctx, "vpc-1", "192.168.1.1", topology.GatewayRouterPort("vpc-1"))) + + got := findRoute(m, "0.0.0.0/0") + require.NotNil(t, got) + assert.Equal(t, "192.168.1.1", got.Nexthop) + require.NotNil(t, got.OutputPort) + assert.Equal(t, "gw-vpc-1", *got.OutputPort) +} + +func TestRouteManager_AddStaticRoute_IsIdempotent(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + rm := NewRouteManager(m) + + spec := RouteSpec{ + Prefix: netip.MustParsePrefix("10.42.0.0/24"), + Nexthop: "10.0.0.254", + } + require.NoError(t, rm.AddStaticRoute(ctx, "vpc-1", spec)) + require.NoError(t, rm.AddStaticRoute(ctx, "vpc-1", spec)) + require.NoError(t, rm.AddStaticRoute(ctx, "vpc-1", spec)) + + assert.Equal(t, 1, countRoutes(m, "10.42.0.0/24")) +} + +func TestRouteManager_AddStaticRoute_DriftReplacesRow(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + rm := NewRouteManager(m) + + require.NoError(t, rm.AddDefaultRoute(ctx, "vpc-1", "192.168.1.1", "gw-vpc-1")) + // Drift the nexthop. + require.NoError(t, rm.AddDefaultRoute(ctx, "vpc-1", "192.168.1.254", "gw-vpc-1")) + + assert.Equal(t, 1, countRoutes(m, "0.0.0.0/0")) + got := findRoute(m, "0.0.0.0/0") + require.NotNil(t, got) + assert.Equal(t, "192.168.1.254", got.Nexthop) +} + +func TestRouteManager_DeleteDefaultRoute_IdempotentOnMissing(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + rm := NewRouteManager(m) + + require.NoError(t, rm.DeleteDefaultRoute(ctx, "vpc-1")) +} + +func TestRouteManager_DeleteStaticRoute_RemovesRow(t *testing.T) { + ctx := context.Background() + m := mock.New() + seedRouter(t, m, "vpc-1") + rm := NewRouteManager(m) + + prefix := netip.MustParsePrefix("10.42.0.0/24") + require.NoError(t, rm.AddStaticRoute(ctx, "vpc-1", RouteSpec{Prefix: prefix, Nexthop: "10.0.0.254"})) + require.NotNil(t, findRoute(m, "10.42.0.0/24")) + + require.NoError(t, rm.DeleteStaticRoute(ctx, "vpc-1", prefix)) + assert.Nil(t, findRoute(m, "10.42.0.0/24")) +} diff --git a/spinifex/network/policy/sg.go b/spinifex/network/policy/sg.go new file mode 100644 index 000000000..962593825 --- /dev/null +++ b/spinifex/network/policy/sg.go @@ -0,0 +1,76 @@ +package policy + +import ( + "context" + "fmt" + + "github.com/mulgadc/spinifex/spinifex/network/ovn" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +// SGSpec is the policy-layer view of a security group. The OVN port group +// named by topology.SecurityGroupPortGroup(GroupID) must already exist. +type SGSpec struct { + GroupID string + VPCID string + IngressRules []Rule + EgressRules []Rule +} + +// SecurityGroupManager attaches ACL policy to an SG's port group. EnsureSG +// and UpdateSG share replace-all semantics: each call clears then re-adds +// the full ACL set (infra + tenant) atomically per AddACLs transaction. +type SecurityGroupManager interface { + // EnsureSG sets the full ACL set; idempotent. + EnsureSG(ctx context.Context, sg SGSpec) error + + // UpdateSG replaces the SG's ACL set. Identical to EnsureSG; kept + // distinct to mirror §8.1 of the parent plan. + UpdateSG(ctx context.Context, sg SGSpec) error + + // DeleteSG clears every ACL on the SG's port group; the PG itself + // stays (topology.Manager owns the lifecycle). + DeleteSG(ctx context.Context, groupID string) error +} + +type sgManager struct { + ovn ovn.Client +} + +var _ SecurityGroupManager = (*sgManager)(nil) + +// NewSecurityGroupManager constructs a SecurityGroupManager backed by client. +func NewSecurityGroupManager(client ovn.Client) SecurityGroupManager { + return &sgManager{ovn: client} +} + +func (m *sgManager) EnsureSG(ctx context.Context, sg SGSpec) error { + return m.applyACLs(ctx, sg) +} + +func (m *sgManager) UpdateSG(ctx context.Context, sg SGSpec) error { + return m.applyACLs(ctx, sg) +} + +func (m *sgManager) DeleteSG(ctx context.Context, groupID string) error { + pg := topology.SecurityGroupPortGroup(groupID) + if err := m.ovn.ClearACLs(ctx, pg); err != nil { + return fmt.Errorf("clear ACLs on %s: %w", pg, err) + } + return nil +} + +// applyACLs clears the PG and re-adds infra + tenant ACLs. ClearACLs + +// AddACLs is not one OVSDB transaction; the observable gap is bounded by +// southbound flow-install latency. +func (m *sgManager) applyACLs(ctx context.Context, sg SGSpec) error { + pg := topology.SecurityGroupPortGroup(sg.GroupID) + if err := m.ovn.ClearACLs(ctx, pg); err != nil { + return fmt.Errorf("clear ACLs on %s: %w", pg, err) + } + specs := append(InfrastructureACLs(pg), RuleACLSpecs(pg, sg.IngressRules, sg.EgressRules)...) + if err := m.ovn.AddACLs(ctx, pg, specs); err != nil { + return fmt.Errorf("add ACLs on %s: %w", pg, err) + } + return nil +} diff --git a/spinifex/network/policy/sg_test.go b/spinifex/network/policy/sg_test.go new file mode 100644 index 000000000..0f3595f65 --- /dev/null +++ b/spinifex/network/policy/sg_test.go @@ -0,0 +1,111 @@ +package policy + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/mulgadc/spinifex/spinifex/network/ovn/mock" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +func newSGMockWithPG(t *testing.T, sgID string) (*mock.Client, string) { + t.Helper() + m := mock.New() + pg := topology.SecurityGroupPortGroup(sgID) + require.NoError(t, m.CreatePortGroup(context.Background(), pg, nil)) + return m, pg +} + +func TestSGManager_EnsureSG_AppliesInfraPlusTenantACLs(t *testing.T) { + ctx := context.Background() + m, pg := newSGMockWithPG(t, "sg-web") + sg := NewSecurityGroupManager(m) + + spec := SGSpec{ + GroupID: "sg-web", + VPCID: "vpc-1", + IngressRules: []Rule{ + {IPProtocol: "tcp", FromPort: 80, ToPort: 80, CIDR: "0.0.0.0/0"}, + {IPProtocol: "tcp", FromPort: 443, ToPort: 443, CIDR: "0.0.0.0/0"}, + }, + EgressRules: []Rule{ + {IPProtocol: "-1", CIDR: "0.0.0.0/0"}, + }, + } + require.NoError(t, sg.EnsureSG(ctx, spec)) + + storedPG, ok := m.PortGroups[pg] + require.True(t, ok) + // 4 infra + 2 ingress + 1 egress = 7 ACLs. + assert.Len(t, storedPG.ACLs, 7) +} + +func TestSGManager_UpdateSG_ReplacesACLSet(t *testing.T) { + ctx := context.Background() + m, pg := newSGMockWithPG(t, "sg-web") + sg := NewSecurityGroupManager(m) + + require.NoError(t, sg.EnsureSG(ctx, SGSpec{ + GroupID: "sg-web", + IngressRules: []Rule{{IPProtocol: "tcp", FromPort: 22, ToPort: 22, CIDR: "10.0.0.0/8"}}, + })) + firstCount := len(m.PortGroups[pg].ACLs) + assert.Equal(t, 5, firstCount) // 4 infra + 1 tenant + + // Replace with a different set — wider ingress, no egress. + require.NoError(t, sg.UpdateSG(ctx, SGSpec{ + GroupID: "sg-web", + IngressRules: []Rule{ + {IPProtocol: "tcp", FromPort: 80, ToPort: 80, CIDR: "0.0.0.0/0"}, + {IPProtocol: "tcp", FromPort: 443, ToPort: 443, CIDR: "0.0.0.0/0"}, + {IPProtocol: "icmp", CIDR: "0.0.0.0/0"}, + }, + })) + assert.Len(t, m.PortGroups[pg].ACLs, 7) // 4 infra + 3 tenant +} + +func TestSGManager_DeleteSG_ClearsACLsKeepsPortGroup(t *testing.T) { + ctx := context.Background() + m, pg := newSGMockWithPG(t, "sg-web") + sg := NewSecurityGroupManager(m) + + require.NoError(t, sg.EnsureSG(ctx, SGSpec{ + GroupID: "sg-web", + IngressRules: []Rule{{IPProtocol: "-1", CIDR: "0.0.0.0/0"}}, + })) + assert.NotEmpty(t, m.PortGroups[pg].ACLs) + + require.NoError(t, sg.DeleteSG(ctx, "sg-web")) + assert.Empty(t, m.PortGroups[pg].ACLs) + // Port group still present — L2 owns lifecycle. + _, exists := m.PortGroups[pg] + assert.True(t, exists) +} + +func TestSGManager_EnsureSG_MissingPortGroup_ReturnsError(t *testing.T) { + ctx := context.Background() + m := mock.New() + sg := NewSecurityGroupManager(m) + + err := sg.EnsureSG(ctx, SGSpec{GroupID: "sg-missing"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "port group") +} + +func TestSGManager_EnsureSG_Idempotent(t *testing.T) { + ctx := context.Background() + m, pg := newSGMockWithPG(t, "sg-web") + sg := NewSecurityGroupManager(m) + + spec := SGSpec{ + GroupID: "sg-web", + IngressRules: []Rule{{IPProtocol: "tcp", FromPort: 22, ToPort: 22, CIDR: "10.0.0.0/8"}}, + } + require.NoError(t, sg.EnsureSG(ctx, spec)) + first := len(m.PortGroups[pg].ACLs) + require.NoError(t, sg.EnsureSG(ctx, spec)) + assert.Equal(t, first, len(m.PortGroups[pg].ACLs)) +} diff --git a/spinifex/network/reconcile/actual.go b/spinifex/network/reconcile/actual.go new file mode 100644 index 000000000..06cb1da14 --- /dev/null +++ b/spinifex/network/reconcile/actual.go @@ -0,0 +1,86 @@ +package reconcile + +import ( + "context" + "fmt" + "strings" + + "github.com/mulgadc/spinifex/spinifex/network/ovn" +) + +// ActualState is the live OVN NB snapshot used to diff against IntentState. +// Orphan deletion is restricted to SG port groups (sg_*); other types are +// create-or-skip only. +type ActualState struct { + Routers map[string]struct{} + Switches map[string]struct{} + Ports map[string]struct{} + RouterPorts map[string]struct{} + PortGroups map[string]struct{} + ExternalSwch map[string]struct{} // vpcID → external switch present (spinifex:role=external) +} + +func newActualState() ActualState { + return ActualState{ + Routers: make(map[string]struct{}), + Switches: make(map[string]struct{}), + Ports: make(map[string]struct{}), + RouterPorts: make(map[string]struct{}), + PortGroups: make(map[string]struct{}), + ExternalSwch: make(map[string]struct{}), + } +} + +// scanActual walks OVN NB once. Any List failure is fatal: a partial scan +// would make missing entries look like create candidates. +func scanActual(ctx context.Context, client ovn.Client) (ActualState, error) { + actual := newActualState() + + routers, err := client.ListLogicalRouters(ctx) + if err != nil { + return ActualState{}, fmt.Errorf("list logical routers: %w", err) + } + for _, r := range routers { + actual.Routers[r.Name] = struct{}{} + } + + switches, err := client.ListLogicalSwitches(ctx) + if err != nil { + return ActualState{}, fmt.Errorf("list logical switches: %w", err) + } + for _, s := range switches { + actual.Switches[s.Name] = struct{}{} + if s.ExternalIDs["spinifex:role"] == "external" { + if vpcID := s.ExternalIDs["spinifex:vpc_id"]; vpcID != "" { + actual.ExternalSwch[vpcID] = struct{}{} + } + } + for _, port := range s.Ports { + actual.Ports[port] = struct{}{} + } + } + + routerPorts, err := client.ListLogicalRouterPorts(ctx) + if err != nil { + return ActualState{}, fmt.Errorf("list logical router ports: %w", err) + } + for _, rp := range routerPorts { + actual.RouterPorts[rp.Name] = struct{}{} + } + + portGroups, err := client.ListPortGroups(ctx) + if err != nil { + return ActualState{}, fmt.Errorf("list port groups: %w", err) + } + for _, pg := range portGroups { + actual.PortGroups[pg.Name] = struct{}{} + } + + return actual, nil +} + +// portGroupIsManaged reports whether the PG is in scope for orphan deletion +// (sg_* only). +func portGroupIsManaged(name string) bool { + return strings.HasPrefix(name, "sg_") +} diff --git a/spinifex/network/reconcile/apply.go b/spinifex/network/reconcile/apply.go new file mode 100644 index 000000000..983fd468a --- /dev/null +++ b/spinifex/network/reconcile/apply.go @@ -0,0 +1,300 @@ +package reconcile + +import ( + "context" + "fmt" + "log/slog" + + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/network/topology" + "github.com/mulgadc/spinifex/spinifex/utils" +) + +// applyVPCs ensures every intent VPC has a LogicalRouter. Stray OVN-only +// routers are left alone. +func (r *reconciler) applyVPCs(ctx context.Context, intent IntentState, actual ActualState) { + for vpcID, spec := range intent.VPCs { + routerName := topology.VPCRouter(vpcID) + if _, ok := actual.Routers[routerName]; ok { + continue + } + lr := &nbdb.LogicalRouter{ + Name: routerName, + ExternalIDs: map[string]string{ + "spinifex:vpc_id": vpcID, + "spinifex:cidr": spec.CIDR.String(), + }, + } + if _, err := r.ovn.EnsureLogicalRouter(ctx, lr); err != nil { + slog.Error("reconcile/apply: ensure VPC router failed", "vpc_id", vpcID, "err", err) + continue + } + actual.Routers[routerName] = struct{}{} + slog.Info("reconcile/apply: ensured VPC router", "vpc_id", vpcID, "router", routerName) + } +} + +// applySubnets ensures every intent Subnet has a LogicalSwitch, parent-LRP, +// router-side LSP, and DHCPOptions row. Each step is independently idempotent. +func (r *reconciler) applySubnets(ctx context.Context, intent IntentState, actual ActualState) { + for subnetID, spec := range intent.Subnets { + switchName := topology.SubnetSwitch(subnetID) + routerName := topology.VPCRouter(spec.VPCID) + routerPortName := topology.SubnetRouterPort(subnetID) + switchRouterPortName := topology.SubnetSwitchRouterPort(subnetID) + + gwIP, prefixBits, err := topology.SubnetGatewayCIDR(spec.CIDR) + if err != nil { + slog.Error("reconcile/apply: subnet gateway calc failed", "subnet_id", subnetID, "err", err) + continue + } + gwCIDRString := fmt.Sprintf("%s/%d", gwIP, prefixBits) + routerMAC := utils.HashMAC(subnetID) + + if _, ok := actual.Switches[switchName]; !ok { + ls := &nbdb.LogicalSwitch{ + Name: switchName, + ExternalIDs: map[string]string{ + "spinifex:subnet_id": subnetID, + "spinifex:vpc_id": spec.VPCID, + }, + } + if _, err := r.ovn.EnsureLogicalSwitch(ctx, ls); err != nil { + slog.Error("reconcile/apply: ensure subnet switch failed", "subnet_id", subnetID, "err", err) + continue + } + actual.Switches[switchName] = struct{}{} + } + + if _, ok := actual.RouterPorts[routerPortName]; !ok { + lrp := &nbdb.LogicalRouterPort{ + Name: routerPortName, + MAC: routerMAC, + Networks: []string{gwCIDRString}, + ExternalIDs: map[string]string{ + "spinifex:subnet_id": subnetID, + "spinifex:vpc_id": spec.VPCID, + }, + } + if err := r.ovn.CreateLogicalRouterPort(ctx, routerName, lrp); err != nil { + slog.Error("reconcile/apply: create subnet LRP failed", "subnet_id", subnetID, "err", err) + continue + } + actual.RouterPorts[routerPortName] = struct{}{} + } + + if _, err := r.ovn.GetLogicalSwitchPort(ctx, switchRouterPortName); err != nil { + lsp := &nbdb.LogicalSwitchPort{ + Name: switchRouterPortName, + Type: "router", + Addresses: []string{"router"}, + Options: map[string]string{"router-port": routerPortName}, + ExternalIDs: map[string]string{ + "spinifex:subnet_id": subnetID, + "spinifex:vpc_id": spec.VPCID, + }, + } + if err := r.ovn.CreateLogicalSwitchPort(ctx, switchName, lsp); err != nil { + slog.Error("reconcile/apply: create subnet router-LSP failed", "subnet_id", subnetID, "err", err) + continue + } + } + + if existing, err := r.ovn.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", subnetID); err != nil || existing == nil { + opts := &nbdb.DHCPOptions{ + CIDR: spec.CIDR.String(), + Options: map[string]string{ + "server_id": gwIP, + "server_mac": routerMAC, + "lease_time": "3600", + "router": gwIP, + "dns_server": "8.8.8.8", + "mtu": "1442", + }, + ExternalIDs: map[string]string{ + "spinifex:subnet_id": subnetID, + "spinifex:vpc_id": spec.VPCID, + }, + } + if _, dErr := r.ovn.CreateDHCPOptions(ctx, opts); dErr != nil { + slog.Warn("reconcile/apply: create DHCP options failed (non-fatal)", "subnet_id", subnetID, "err", dErr) + } + } + + slog.Info("reconcile/apply: ensured subnet topology", + "subnet_id", subnetID, "switch", switchName, "router_port", routerPortName) + } +} + +// applySGs ensures every intent SG has a port group, then (when +// pruneOrphans) deletes sg_* PGs without a matching intent SG. +// pruneOrphans=false is startup mode: a leader can load intent before peer +// subscribers have finished creating the PGs, so pruning at boot would +// delete in-flight resources. Drift always prunes. +func (r *reconciler) applySGs(ctx context.Context, intent IntentState, actual ActualState, pruneOrphans bool) { + for groupID, spec := range intent.SGs { + if err := r.topology.EnsureSGPortGroup(ctx, groupID); err != nil { + slog.Error("reconcile/apply: EnsureSGPortGroup failed", "sg", groupID, "err", err) + continue + } + actual.PortGroups[topology.SecurityGroupPortGroup(groupID)] = struct{}{} + if err := r.sg.EnsureSG(ctx, spec); err != nil { + slog.Error("reconcile/apply: EnsureSG failed", "sg", groupID, "err", err) + } + } + + if !pruneOrphans { + return + } + + wantPGs := make(map[string]struct{}, len(intent.SGs)) + for groupID := range intent.SGs { + wantPGs[topology.SecurityGroupPortGroup(groupID)] = struct{}{} + } + for pgName := range actual.PortGroups { + if !portGroupIsManaged(pgName) { + continue + } + if _, ok := wantPGs[pgName]; ok { + continue + } + if err := r.topology.DeleteSGPortGroupByName(ctx, pgName); err != nil { + slog.Warn("reconcile/apply: orphan DeleteSGPortGroupByName failed", "pg", pgName, "err", err) + continue + } + delete(actual.PortGroups, pgName) + slog.Info("reconcile/apply: removed orphan port group", "pg", pgName) + } +} + +// applyPorts ensures each intent ENI has an LSP with PG memberships matching +// its SGIDs. Existing ports get a diff-based UpdatePortGroupMemberships so an +// SG swap never exposes an "unrestricted" gap (OVN default). +func (r *reconciler) applyPorts(ctx context.Context, intent IntentState, actual ActualState) { + for portID, spec := range intent.Ports { + portName := topology.Port(portID) + switchName := topology.SubnetSwitch(spec.SubnetID) + desiredPGs := make([]string, 0, len(spec.SGIDs)) + for _, sgID := range spec.SGIDs { + pgName := topology.SecurityGroupPortGroup(sgID) + if _, ok := actual.PortGroups[pgName]; !ok { + slog.Warn("reconcile/apply: skipping port SG membership — port group missing in OVN", + "port", portName, "sg", sgID, "pg", pgName) + continue + } + desiredPGs = append(desiredPGs, pgName) + } + + if _, err := r.ovn.GetLogicalSwitchPort(ctx, portName); err != nil { + addrStr := spec.MAC.String() + " " + spec.PrivateIP.String() + lsp := &nbdb.LogicalSwitchPort{ + Name: portName, + Addresses: []string{addrStr}, + PortSecurity: []string{addrStr}, + ExternalIDs: map[string]string{ + "spinifex:eni_id": portID, + "spinifex:subnet_id": spec.SubnetID, + "spinifex:vpc_id": spec.VPCID, + }, + } + if dhcpOpts, derr := r.ovn.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", spec.SubnetID); derr == nil && dhcpOpts != nil { + lsp.DHCPv4Options = &dhcpOpts.UUID + } + if err := r.ovn.CreateLogicalSwitchPortInGroups(ctx, switchName, lsp, desiredPGs); err != nil { + slog.Error("reconcile/apply: create ENI port failed", "port", portName, "err", err) + } + continue + } + + currentPGs, err := r.ovn.ListPortGroupsForPort(ctx, portName) + if err != nil { + slog.Warn("reconcile/apply: list port groups for port failed", "port", portName, "err", err) + continue + } + addPGs, removePGs := diffSets(desiredPGs, currentPGs) + if len(addPGs) == 0 && len(removePGs) == 0 { + continue + } + if err := r.ovn.UpdatePortGroupMemberships(ctx, portName, addPGs, removePGs); err != nil { + slog.Warn("reconcile/apply: update port group memberships failed", "port", portName, "err", err) + } + } +} + +// applyIGWs ensures every intent IGW's OVN topology (external switch, +// localnet LSP, gateway LRP, default route, Gateway_Chassis bindings) and +// rebinds chassis on existing IGWs. AttachIGW is idempotent. +func (r *reconciler) applyIGWs(ctx context.Context, intent IntentState, actual ActualState) { + for vpcID, spec := range intent.IGWs { + if _, ok := actual.ExternalSwch[vpcID]; !ok { + if err := r.igw.AttachIGW(ctx, spec); err != nil { + slog.Error("reconcile/apply: AttachIGW failed", "vpc_id", vpcID, "err", err) + continue + } + actual.ExternalSwch[vpcID] = struct{}{} + } + r.rebindGatewayChassis(ctx, vpcID) + } +} + +// rebindGatewayChassis re-asserts (chassis,priority) tuples on an existing +// gateway LRP. SetGatewayChassis is idempotent. +func (r *reconciler) rebindGatewayChassis(ctx context.Context, vpcID string) { + if len(r.chassis) == 0 { + return + } + gwPortName := topology.GatewayRouterPort(vpcID) + if _, err := r.ovn.GetLogicalRouterPort(ctx, gwPortName); err != nil { + return + } + for i, chassis := range r.chassis { + priority := max(20-(i*5), 1) + if err := r.ovn.SetGatewayChassis(ctx, gwPortName, chassis, priority); err != nil { + slog.Warn("reconcile/apply: SetGatewayChassis failed", "vpc_id", vpcID, "chassis", chassis, "err", err) + } + } +} + +// applyEIPs runs every intent EIP through NATManager.AddEIP (idempotent; +// stale dnat_and_snat rules are cleaned first). +func (r *reconciler) applyEIPs(ctx context.Context, intent IntentState, _ ActualState) { + for _, spec := range intent.EIPs { + if err := r.nat.AddEIP(ctx, spec); err != nil { + slog.Error("reconcile/apply: AddEIP failed", "external_ip", spec.ExternalIP, "logical_ip", spec.LogicalIP, "err", err) + } + } +} + +// applyNATGWs runs every intent NAT gateway through NATManager.AddNATGateway. +// Duplicate (snat,SubnetCIDR) tuples are rejected by the underlying client. +func (r *reconciler) applyNATGWs(ctx context.Context, intent IntentState, _ ActualState) { + for _, spec := range intent.NATGWs { + if err := r.nat.AddNATGateway(ctx, spec); err != nil { + slog.Warn("reconcile/apply: AddNATGateway failed (likely already exists)", + "natgw_id", spec.NATGatewayID, "subnet_cidr", spec.SubnetCIDR, "err", err) + } + } +} + +// diffSets returns (desired - current, current - desired). +func diffSets(desired, current []string) (add, remove []string) { + desiredSet := make(map[string]struct{}, len(desired)) + for _, s := range desired { + desiredSet[s] = struct{}{} + } + currentSet := make(map[string]struct{}, len(current)) + for _, s := range current { + currentSet[s] = struct{}{} + } + for s := range desiredSet { + if _, ok := currentSet[s]; !ok { + add = append(add, s) + } + } + for s := range currentSet { + if _, ok := desiredSet[s]; !ok { + remove = append(remove, s) + } + } + return add, remove +} diff --git a/spinifex/network/reconcile/apply_test.go b/spinifex/network/reconcile/apply_test.go new file mode 100644 index 000000000..ee7776116 --- /dev/null +++ b/spinifex/network/reconcile/apply_test.go @@ -0,0 +1,378 @@ +package reconcile + +import ( + "context" + "net" + "net/netip" + "slices" + "testing" + + "github.com/mulgadc/spinifex/spinifex/network/external" + "github.com/mulgadc/spinifex/spinifex/network/ovn/mock" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +// newTestReconciler builds a reconciler around the in-memory OVN mock. +func newTestReconciler(t *testing.T) (*reconciler, *mock.Client) { + t.Helper() + m := mock.New() + sg := policy.NewSecurityGroupManager(m) + nat, err := policy.NewNATManager(m, policy.NATModeDistributed) + if err != nil { + t.Fatalf("NewNATManager: %v", err) + } + routes := policy.NewRouteManager(m) + igw, err := external.NewIGWManager(external.IGWManagerConfig{ + OVN: m, + Routes: routes, + NAT: nat, + Allocator: external.LinkLocalAllocator{}, + NATMode: policy.NATModeDistributed, + }) + if err != nil { + t.Fatalf("NewIGWManager: %v", err) + } + topo := topology.NewLiveManager(m) + rec, err := New(Config{ + OVN: m, SG: sg, NAT: nat, Routes: routes, IGW: igw, Topology: topo, + LocalAZ: "us-east-1a", NodeHostname: "test-host", + }) + if err != nil { + t.Fatalf("New: %v", err) + } + r, ok := rec.(*reconciler) + if !ok { + t.Fatalf("New returned %T, want *reconciler", rec) + } + return r, m +} + +func TestReconcile_TopoOrder_VPCThenSubnetThenPort(t *testing.T) { + rec, m := newTestReconciler(t) + ctx := context.Background() + + intent := freshIntent(t) + + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + if _, ok := m.Routers["vpc-"+intent.VPCs["vpc-a"].VPCID]; !ok { + t.Errorf("VPC router not created") + } + if _, ok := m.Switches["subnet-"+intent.Subnets["subnet-a"].SubnetID]; !ok { + t.Errorf("subnet switch not created") + } + if _, ok := m.Ports["port-"+intent.Ports["eni-a"].PortID]; !ok { + t.Errorf("ENI port not created") + } + if _, ok := m.PortGroups[topology.SecurityGroupPortGroup("sg-a")]; !ok { + t.Errorf("SG port group not created") + } +} + +func TestReconcile_PortJoinsPortGroupAtomically(t *testing.T) { + rec, m := newTestReconciler(t) + ctx := context.Background() + + intent := freshIntent(t) + + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + pg := m.PortGroups[topology.SecurityGroupPortGroup("sg-a")] + if pg == nil { + t.Fatal("port group not present") + } + port := m.Ports["port-"+intent.Ports["eni-a"].PortID] + if port == nil { + t.Fatal("ENI port not present") + } + if !slices.Contains(pg.Ports, port.UUID) { + t.Errorf("ENI port not joined to SG port group atomically — racy gap revived") + } +} + +func TestReconcile_Idempotent(t *testing.T) { + rec, m := newTestReconciler(t) + ctx := context.Background() + + intent := freshIntent(t) + + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile #1: %v", err) + } + routerCountBefore := len(m.Routers) + switchCountBefore := len(m.Switches) + portCountBefore := len(m.Ports) + pgCountBefore := len(m.PortGroups) + addACLsBefore := m.AddACLCalls + + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile #2: %v", err) + } + + if len(m.Routers) != routerCountBefore { + t.Errorf("second reconcile created duplicate routers: %d → %d", routerCountBefore, len(m.Routers)) + } + if len(m.Switches) != switchCountBefore { + t.Errorf("second reconcile created duplicate switches: %d → %d", switchCountBefore, len(m.Switches)) + } + if len(m.Ports) != portCountBefore { + t.Errorf("second reconcile created duplicate ports: %d → %d", portCountBefore, len(m.Ports)) + } + if len(m.PortGroups) != pgCountBefore { + t.Errorf("second reconcile created duplicate port groups: %d → %d", pgCountBefore, len(m.PortGroups)) + } + // EnsureSG has replace-all semantics: AddACL count must grow each pass. + if m.AddACLCalls < addACLsBefore { + t.Errorf("second reconcile fewer AddACL calls than first — replace-all semantics regressed") + } +} + +func TestReconcile_OrphanPortGroupRemoved(t *testing.T) { + rec, m := newTestReconciler(t) + ctx := context.Background() + + if err := m.CreatePortGroup(ctx, "sg_orphan", nil); err != nil { + t.Fatalf("seed orphan port group: %v", err) + } + + intent := IntentState{ + VPCs: map[string]topology.VPCSpec{}, + Subnets: map[string]topology.SubnetSpec{}, + Ports: map[string]topology.PortSpec{}, + SGs: map[string]policy.SGSpec{}, + IGWs: map[string]external.IGWSpec{}, + EIPs: map[string]policy.EIPSpec{}, + NATGWs: map[string]policy.NATGWSpec{}, + } + + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + if _, ok := m.PortGroups["sg_orphan"]; ok { + t.Errorf("orphan port group not removed") + } +} + +// ReconcileApplyOnly must not prune managed sg_* PGs when intent is empty +// (startup race vs. peer subscribers); full Reconcile must still prune. +func TestReconcile_ApplyOnlyKeepsOrphanPortGroup(t *testing.T) { + rec, m := newTestReconciler(t) + ctx := context.Background() + + if err := m.CreatePortGroup(ctx, "sg_orphan", nil); err != nil { + t.Fatalf("seed orphan port group: %v", err) + } + + intent := IntentState{ + VPCs: map[string]topology.VPCSpec{}, + Subnets: map[string]topology.SubnetSpec{}, + Ports: map[string]topology.PortSpec{}, + SGs: map[string]policy.SGSpec{}, + IGWs: map[string]external.IGWSpec{}, + EIPs: map[string]policy.EIPSpec{}, + NATGWs: map[string]policy.NATGWSpec{}, + } + + if err := rec.ReconcileApplyOnly(ctx, intent); err != nil { + t.Fatalf("ReconcileApplyOnly: %v", err) + } + + if _, ok := m.PortGroups["sg_orphan"]; !ok { + t.Errorf("ReconcileApplyOnly pruned port group; startup race fix regressed") + } + + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile: %v", err) + } + if _, ok := m.PortGroups["sg_orphan"]; ok { + t.Errorf("Reconcile failed to prune orphan after ApplyOnly path") + } +} + +func TestReconcile_ChassisRebindOnExistingIGW(t *testing.T) { + m := mock.New() + sg := policy.NewSecurityGroupManager(m) + nat, _ := policy.NewNATManager(m, policy.NATModeDistributed) + routes := policy.NewRouteManager(m) + igw, _ := external.NewIGWManager(external.IGWManagerConfig{ + OVN: m, Routes: routes, NAT: nat, + Allocator: external.LinkLocalAllocator{}, + NATMode: policy.NATModeDistributed, + }) + topo := topology.NewLiveManager(m) + rec, err := New(Config{ + OVN: m, SG: sg, NAT: nat, Routes: routes, IGW: igw, Topology: topo, + LocalAZ: "us-east-1a", NodeHostname: "test-host", + Chassis: []string{"chassis-1", "chassis-2"}, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + ctx := context.Background() + + // Seed VPC router first so AttachIGW can create the gateway LRP on it. + seedIntent := IntentState{ + VPCs: map[string]topology.VPCSpec{ + "vpc-a": {VPCID: "vpc-a", CIDR: netip.MustParsePrefix("10.0.0.0/16"), VNI: 100}, + }, + Subnets: map[string]topology.SubnetSpec{}, + Ports: map[string]topology.PortSpec{}, + SGs: map[string]policy.SGSpec{}, + IGWs: map[string]external.IGWSpec{}, + EIPs: map[string]policy.EIPSpec{}, + NATGWs: map[string]policy.NATGWSpec{}, + } + if err := rec.Reconcile(ctx, seedIntent); err != nil { + t.Fatalf("seed VPC reconcile: %v", err) + } + // Seed external switch + gateway LRP so apply takes the rebind branch. + if err := igw.AttachIGW(ctx, external.IGWSpec{VPCID: "vpc-a", InternetGatewayID: "igw-a"}); err != nil { + t.Fatalf("seed AttachIGW: %v", err) + } + setCallsBefore := m.SetGatewayChassisCalls + + intent := IntentState{ + VPCs: map[string]topology.VPCSpec{}, + Subnets: map[string]topology.SubnetSpec{}, + Ports: map[string]topology.PortSpec{}, + SGs: map[string]policy.SGSpec{}, + IGWs: map[string]external.IGWSpec{"vpc-a": {VPCID: "vpc-a", InternetGatewayID: "igw-a"}}, + EIPs: map[string]policy.EIPSpec{}, + NATGWs: map[string]policy.NATGWSpec{}, + } + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + if m.SetGatewayChassisCalls <= setCallsBefore { + t.Errorf("expected SetGatewayChassis to fire on existing IGW for chassis rebind; calls before=%d after=%d", + setCallsBefore, m.SetGatewayChassisCalls) + } +} + +func TestReconcile_IGWAttachWhenTopologyMissing(t *testing.T) { + m := mock.New() + sg := policy.NewSecurityGroupManager(m) + nat, _ := policy.NewNATManager(m, policy.NATModeDistributed) + routes := policy.NewRouteManager(m) + igw, _ := external.NewIGWManager(external.IGWManagerConfig{ + OVN: m, Routes: routes, NAT: nat, + Allocator: external.LinkLocalAllocator{}, + NATMode: policy.NATModeDistributed, + }) + topo := topology.NewLiveManager(m) + rec, err := New(Config{ + OVN: m, SG: sg, NAT: nat, Routes: routes, IGW: igw, Topology: topo, + LocalAZ: "us-east-1a", NodeHostname: "test-host", + Chassis: []string{"chassis-1"}, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + ctx := context.Background() + + intent := IntentState{ + VPCs: map[string]topology.VPCSpec{ + "vpc-a": {VPCID: "vpc-a", CIDR: netip.MustParsePrefix("10.0.0.0/16"), VNI: 100}, + }, + Subnets: map[string]topology.SubnetSpec{}, + Ports: map[string]topology.PortSpec{}, + SGs: map[string]policy.SGSpec{}, + IGWs: map[string]external.IGWSpec{"vpc-a": {VPCID: "vpc-a", InternetGatewayID: "igw-a"}}, + EIPs: map[string]policy.EIPSpec{}, + NATGWs: map[string]policy.NATGWSpec{}, + } + + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile: %v", err) + } + + if _, ok := m.Switches[topology.ExternalSwitch("vpc-a")]; !ok { + t.Errorf("external switch not created by reconciler AttachIGW path") + } + gwPort := topology.GatewayRouterPort("vpc-a") + if _, ok := m.RouterPorts[gwPort]; !ok { + t.Errorf("gateway LRP %s not created", gwPort) + } + + switchCount := len(m.Switches) + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile #2: %v", err) + } + if len(m.Switches) != switchCount { + t.Errorf("second reconcile created duplicate switches: %d → %d", switchCount, len(m.Switches)) + } +} + +func TestReconcile_PortMembershipDriftCorrected(t *testing.T) { + rec, m := newTestReconciler(t) + ctx := context.Background() + + intent := freshIntent(t) + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile #1: %v", err) + } + + intent.SGs["sg-b"] = policy.SGSpec{GroupID: "sg-b", VPCID: "vpc-a"} + port := intent.Ports["eni-a"] + port.SGIDs = append(port.SGIDs, "sg-b") + intent.Ports["eni-a"] = port + + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile #2: %v", err) + } + + pgB := m.PortGroups[topology.SecurityGroupPortGroup("sg-b")] + if pgB == nil { + t.Fatal("new SG port group not created") + } + storedPort := m.Ports["port-"+port.PortID] + if !slices.Contains(pgB.Ports, storedPort.UUID) { + t.Errorf("ENI port not joined to new SG port group on drift") + } +} + +func TestDiffSets(t *testing.T) { + add, remove := diffSets([]string{"a", "b", "c"}, []string{"b", "c", "d"}) + if !slices.Equal(sortedCopy(add), []string{"a"}) { + t.Errorf("add = %v, want [a]", add) + } + if !slices.Equal(sortedCopy(remove), []string{"d"}) { + t.Errorf("remove = %v, want [d]", remove) + } +} + +func freshIntent(t *testing.T) IntentState { + t.Helper() + mac, _ := net.ParseMAC("02:00:00:00:00:01") + return IntentState{ + VPCs: map[string]topology.VPCSpec{ + "vpc-a": {VPCID: "vpc-a", CIDR: netip.MustParsePrefix("10.0.0.0/16"), VNI: 100}, + }, + Subnets: map[string]topology.SubnetSpec{ + "subnet-a": {SubnetID: "subnet-a", VPCID: "vpc-a", CIDR: netip.MustParsePrefix("10.0.1.0/24")}, + }, + Ports: map[string]topology.PortSpec{ + "eni-a": {PortID: "eni-a", SubnetID: "subnet-a", VPCID: "vpc-a", + PrivateIP: netip.MustParseAddr("10.0.1.10"), MAC: mac, SGIDs: []string{"sg-a"}}, + }, + SGs: map[string]policy.SGSpec{ + "sg-a": {GroupID: "sg-a", VPCID: "vpc-a"}, + }, + IGWs: map[string]external.IGWSpec{}, + EIPs: map[string]policy.EIPSpec{}, + NATGWs: map[string]policy.NATGWSpec{}, + } +} + +func sortedCopy(in []string) []string { + out := append([]string(nil), in...) + slices.Sort(out) + return out +} diff --git a/spinifex/network/reconcile/drift.go b/spinifex/network/reconcile/drift.go new file mode 100644 index 000000000..26a2cbab3 --- /dev/null +++ b/spinifex/network/reconcile/drift.go @@ -0,0 +1,53 @@ +package reconcile + +import ( + "context" + "log/slog" + "time" + + "github.com/nats-io/nats.go" +) + +// DriftInterval is the gap between drift passes. Var (not const) so +// integration tests can shrink it. +var DriftInterval = 5 * time.Minute + +// DriftLoop runs Reconcile every DriftInterval, gated on AcquireLeader so +// only one vpcd scans at a time. Returns when ctx is cancelled. +func DriftLoop(ctx context.Context, rec Reconciler, nc *nats.Conn, localAZ, holder string) { + js, err := nc.JetStream() + if err != nil { + slog.Error("reconcile/drift: JetStream context unavailable, drift loop disabled", "err", err) + return + } + + ticker := time.NewTicker(DriftInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + runDriftCycle(ctx, rec, nc, js, localAZ, holder) + } + } +} + +// runDriftCycle is one tick body, split out so tests can drive it directly. +func runDriftCycle(ctx context.Context, rec Reconciler, nc *nats.Conn, js nats.JetStreamContext, localAZ, holder string) { + release, elected := AcquireLeader(nc, holder) + if !elected { + return + } + defer release() + + intent, err := LoadIntentFromKV(ctx, js, localAZ) + if err != nil { + slog.Warn("reconcile/drift: load intent failed", "err", err) + return + } + if err := rec.Reconcile(ctx, intent); err != nil { + slog.Warn("reconcile/drift: reconcile failed", "err", err) + } +} diff --git a/spinifex/network/reconcile/intent.go b/spinifex/network/reconcile/intent.go new file mode 100644 index 000000000..feca8b513 --- /dev/null +++ b/spinifex/network/reconcile/intent.go @@ -0,0 +1,508 @@ +package reconcile + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net" + "net/netip" + "strings" + + handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip" + handlers_ec2_igw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/igw" + handlers_ec2_natgw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/natgw" + handlers_ec2_routetable "github.com/mulgadc/spinifex/spinifex/handlers/ec2/routetable" + handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + "github.com/mulgadc/spinifex/spinifex/network/external" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/topology" + "github.com/mulgadc/spinifex/spinifex/utils" + "github.com/nats-io/nats.go" +) + +// IntentState is the desired OVN state derived from the JetStream KV +// snapshot, scoped to the local AZ. Empty maps are valid. +type IntentState struct { + VPCs map[string]topology.VPCSpec + Subnets map[string]topology.SubnetSpec + Ports map[string]topology.PortSpec + SGs map[string]policy.SGSpec + IGWs map[string]external.IGWSpec // attached only + EIPs map[string]policy.EIPSpec // keyed by logicalIP; associated only + NATGWs map[string]policy.NATGWSpec +} + +// LoadIntentFromKV assembles IntentState scoped to localAZ. Missing buckets +// (first boot) are treated as empty. AZ filter: `vpc.AZ == "" || vpc.AZ == +// localAZ`. Children inherit the filter transitively via their parent VPC. +func LoadIntentFromKV(ctx context.Context, js nats.JetStreamContext, localAZ string) (IntentState, error) { + intent := IntentState{ + VPCs: make(map[string]topology.VPCSpec), + Subnets: make(map[string]topology.SubnetSpec), + Ports: make(map[string]topology.PortSpec), + SGs: make(map[string]policy.SGSpec), + IGWs: make(map[string]external.IGWSpec), + EIPs: make(map[string]policy.EIPSpec), + NATGWs: make(map[string]policy.NATGWSpec), + } + + localVPCs, err := loadVPCs(js, localAZ, intent.VPCs) + if err != nil { + return IntentState{}, err + } + if err := loadSubnets(js, localVPCs, intent.Subnets); err != nil { + return IntentState{}, err + } + if err := loadSGs(js, localVPCs, intent.SGs); err != nil { + return IntentState{}, err + } + if err := loadPorts(js, localVPCs, intent.Ports); err != nil { + return IntentState{}, err + } + if err := loadIGWs(js, localVPCs, intent.IGWs); err != nil { + return IntentState{}, err + } + if err := loadEIPs(js, localVPCs, intent.EIPs); err != nil { + return IntentState{}, err + } + routeTables, err := loadRouteTables(js, localVPCs) + if err != nil { + return IntentState{}, err + } + if err := loadNATGWs(js, localVPCs, intent.Subnets, routeTables, intent.NATGWs); err != nil { + return IntentState{}, err + } + + _ = ctx + return intent, nil +} + +// matchesLocalAZ enforces §11.1: empty AZ counts as local (legacy records). +func matchesLocalAZ(vpcAZ, localAZ string) bool { + return vpcAZ == "" || vpcAZ == localAZ +} + +func keyIsVersion(key string) bool { return key == utils.VersionKey } + +func loadVPCs(js nats.JetStreamContext, localAZ string, out map[string]topology.VPCSpec) (map[string]struct{}, error) { + localVPCs := make(map[string]struct{}) + + kv, err := js.KeyValue(handlers_ec2_vpc.KVBucketVPCs) + if err != nil { + slog.Debug("reconcile/intent: VPC bucket not available, skipping", "err", err) + return localVPCs, nil + } + keys, err := kv.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return localVPCs, nil + } + return nil, fmt.Errorf("list VPC keys: %w", err) + } + for _, key := range keys { + if keyIsVersion(key) { + continue + } + entry, err := kv.Get(key) + if err != nil { + slog.Warn("reconcile/intent: VPC read failed", "key", key, "err", err) + continue + } + var rec handlers_ec2_vpc.VPCRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + slog.Warn("reconcile/intent: VPC unmarshal failed", "key", key, "err", err) + continue + } + if !matchesLocalAZ(rec.AZ, localAZ) { + continue + } + prefix, err := netip.ParsePrefix(rec.CidrBlock) + if err != nil { + slog.Warn("reconcile/intent: VPC CIDR parse failed", "vpc_id", rec.VpcId, "cidr", rec.CidrBlock, "err", err) + continue + } + out[rec.VpcId] = topology.VPCSpec{ + VPCID: rec.VpcId, + CIDR: prefix, + VNI: rec.VNI, + } + localVPCs[rec.VpcId] = struct{}{} + } + return localVPCs, nil +} + +func loadSubnets(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[string]topology.SubnetSpec) error { + kv, err := js.KeyValue(handlers_ec2_vpc.KVBucketSubnets) + if err != nil { + slog.Debug("reconcile/intent: subnet bucket not available, skipping", "err", err) + return nil + } + keys, err := kv.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return nil + } + return fmt.Errorf("list subnet keys: %w", err) + } + for _, key := range keys { + if keyIsVersion(key) { + continue + } + entry, err := kv.Get(key) + if err != nil { + slog.Warn("reconcile/intent: subnet read failed", "key", key, "err", err) + continue + } + var rec handlers_ec2_vpc.SubnetRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + slog.Warn("reconcile/intent: subnet unmarshal failed", "key", key, "err", err) + continue + } + if _, ok := localVPCs[rec.VpcId]; !ok { + continue + } + prefix, err := netip.ParsePrefix(rec.CidrBlock) + if err != nil { + slog.Warn("reconcile/intent: subnet CIDR parse failed", "subnet_id", rec.SubnetId, "cidr", rec.CidrBlock, "err", err) + continue + } + out[rec.SubnetId] = topology.SubnetSpec{ + SubnetID: rec.SubnetId, + VPCID: rec.VpcId, + CIDR: prefix, + } + } + return nil +} + +func loadSGs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[string]policy.SGSpec) error { + kv, err := js.KeyValue(handlers_ec2_vpc.KVBucketSecurityGroups) + if err != nil { + slog.Debug("reconcile/intent: SG bucket not available, skipping", "err", err) + return nil + } + keys, err := kv.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return nil + } + return fmt.Errorf("list SG keys: %w", err) + } + for _, key := range keys { + if keyIsVersion(key) { + continue + } + entry, err := kv.Get(key) + if err != nil { + slog.Warn("reconcile/intent: SG read failed", "key", key, "err", err) + continue + } + var rec handlers_ec2_vpc.SecurityGroupRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + slog.Warn("reconcile/intent: SG unmarshal failed", "key", key, "err", err) + continue + } + if _, ok := localVPCs[rec.VpcId]; !ok { + continue + } + out[rec.GroupId] = policy.SGSpec{ + GroupID: rec.GroupId, + VPCID: rec.VpcId, + IngressRules: sgRulesToPolicyRules(rec.IngressRules), + EgressRules: sgRulesToPolicyRules(rec.EgressRules), + } + } + return nil +} + +func loadPorts(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[string]topology.PortSpec) error { + kv, err := js.KeyValue(handlers_ec2_vpc.KVBucketENIs) + if err != nil { + slog.Debug("reconcile/intent: ENI bucket not available, skipping", "err", err) + return nil + } + keys, err := kv.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return nil + } + return fmt.Errorf("list ENI keys: %w", err) + } + for _, key := range keys { + if keyIsVersion(key) { + continue + } + entry, err := kv.Get(key) + if err != nil { + slog.Warn("reconcile/intent: ENI read failed", "key", key, "err", err) + continue + } + var rec handlers_ec2_vpc.ENIRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + slog.Warn("reconcile/intent: ENI unmarshal failed", "key", key, "err", err) + continue + } + if _, ok := localVPCs[rec.VpcId]; !ok { + continue + } + addr, err := netip.ParseAddr(rec.PrivateIpAddress) + if err != nil { + slog.Warn("reconcile/intent: ENI IP parse failed", "eni", rec.NetworkInterfaceId, "ip", rec.PrivateIpAddress, "err", err) + continue + } + mac, err := net.ParseMAC(rec.MacAddress) + if err != nil { + slog.Warn("reconcile/intent: ENI MAC parse failed", "eni", rec.NetworkInterfaceId, "mac", rec.MacAddress, "err", err) + continue + } + out[rec.NetworkInterfaceId] = topology.PortSpec{ + PortID: rec.NetworkInterfaceId, + SubnetID: rec.SubnetId, + VPCID: rec.VpcId, + PrivateIP: addr, + MAC: mac, + SGIDs: append([]string(nil), rec.SecurityGroupIds...), + } + } + return nil +} + +func loadIGWs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[string]external.IGWSpec) error { + kv, err := js.KeyValue(handlers_ec2_igw.KVBucketIGW) + if err != nil { + slog.Debug("reconcile/intent: IGW bucket not available, skipping", "err", err) + return nil + } + keys, err := kv.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return nil + } + return fmt.Errorf("list IGW keys: %w", err) + } + for _, key := range keys { + if keyIsVersion(key) { + continue + } + entry, err := kv.Get(key) + if err != nil { + slog.Warn("reconcile/intent: IGW read failed", "key", key, "err", err) + continue + } + var rec handlers_ec2_igw.IGWRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + slog.Warn("reconcile/intent: IGW unmarshal failed", "key", key, "err", err) + continue + } + if rec.VpcId == "" || !strings.EqualFold(rec.State, "available") { + continue + } + if _, ok := localVPCs[rec.VpcId]; !ok { + continue + } + out[rec.VpcId] = external.IGWSpec{ + VPCID: rec.VpcId, + InternetGatewayID: rec.InternetGatewayId, + } + } + return nil +} + +func loadEIPs(js nats.JetStreamContext, localVPCs map[string]struct{}, out map[string]policy.EIPSpec) error { + kv, err := js.KeyValue(handlers_ec2_eip.KVBucketEIPs) + if err != nil { + slog.Debug("reconcile/intent: EIP bucket not available, skipping", "err", err) + return nil + } + keys, err := kv.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return nil + } + return fmt.Errorf("list EIP keys: %w", err) + } + for _, key := range keys { + if keyIsVersion(key) { + continue + } + entry, err := kv.Get(key) + if err != nil { + slog.Warn("reconcile/intent: EIP read failed", "key", key, "err", err) + continue + } + var rec handlers_ec2_eip.EIPRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + slog.Warn("reconcile/intent: EIP unmarshal failed", "key", key, "err", err) + continue + } + if !strings.EqualFold(rec.State, "associated") || rec.VpcId == "" || rec.PublicIp == "" || rec.PrivateIp == "" { + continue + } + if _, ok := localVPCs[rec.VpcId]; !ok { + continue + } + spec := policy.EIPSpec{ + VPCID: rec.VpcId, + ExternalIP: rec.PublicIp, + LogicalIP: rec.PrivateIp, + } + if rec.ENIId != "" { + spec.PortName = topology.Port(rec.ENIId) + } + out[rec.PrivateIp] = spec + } + return nil +} + +// loadRouteTables snapshots every route table whose VPC is local. Used by +// loadNATGWs to fan out NATGW SNAT specs over associated subnets — mirrors the +// event-driven publisher in handlers/ec2/routetable. +func loadRouteTables(js nats.JetStreamContext, localVPCs map[string]struct{}) ([]handlers_ec2_routetable.RouteTableRecord, error) { + kv, err := js.KeyValue(handlers_ec2_routetable.KVBucketRouteTables) + if err != nil { + slog.Debug("reconcile/intent: route table bucket not available, skipping", "err", err) + return nil, nil + } + keys, err := kv.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return nil, nil + } + return nil, fmt.Errorf("list route table keys: %w", err) + } + var out []handlers_ec2_routetable.RouteTableRecord + for _, key := range keys { + if keyIsVersion(key) { + continue + } + entry, err := kv.Get(key) + if err != nil { + slog.Warn("reconcile/intent: route table read failed", "key", key, "err", err) + continue + } + var rec handlers_ec2_routetable.RouteTableRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + slog.Warn("reconcile/intent: route table unmarshal failed", "key", key, "err", err) + continue + } + if _, ok := localVPCs[rec.VpcId]; !ok { + continue + } + out = append(out, rec) + } + return out, nil +} + +// natgwSpecKey is the intent-map key for NATGWSpec. The reconciler needs one +// spec per (natgwID, subnetCIDR) because a NATGW may serve multiple subnets +// (route-table associations) and OVN stores one snat row per subnet CIDR. +func natgwSpecKey(natgwID, subnetCIDR string) string { + return natgwID + "|" + subnetCIDR +} + +// loadNATGWs emits one NATGWSpec per (NATGW, associated subnet) pair. Mirrors +// publishNatGatewayEventsForAssociation in handlers/ec2/routetable: a NATGW's +// SNAT rule rewrites traffic from the *private* subnets routed through it, +// not from the NATGW's own public subnet. +func loadNATGWs( + js nats.JetStreamContext, + localVPCs map[string]struct{}, + subnets map[string]topology.SubnetSpec, + routeTables []handlers_ec2_routetable.RouteTableRecord, + out map[string]policy.NATGWSpec, +) error { + kv, err := js.KeyValue(handlers_ec2_natgw.KVBucketNatGateways) + if err != nil { + slog.Debug("reconcile/intent: NAT GW bucket not available, skipping", "err", err) + return nil + } + keys, err := kv.Keys() + if err != nil { + if errors.Is(err, nats.ErrNoKeysFound) { + return nil + } + return fmt.Errorf("list NAT GW keys: %w", err) + } + for _, key := range keys { + if keyIsVersion(key) { + continue + } + entry, err := kv.Get(key) + if err != nil { + slog.Warn("reconcile/intent: NAT GW read failed", "key", key, "err", err) + continue + } + var rec handlers_ec2_natgw.NatGatewayRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + slog.Warn("reconcile/intent: NAT GW unmarshal failed", "key", key, "err", err) + continue + } + if !strings.EqualFold(rec.State, "available") || rec.VpcId == "" || rec.PublicIp == "" { + continue + } + if _, ok := localVPCs[rec.VpcId]; !ok { + continue + } + + emitted := false + for _, rt := range routeTables { + if rt.VpcId != rec.VpcId { + continue + } + hasNatRoute := false + for _, r := range rt.Routes { + if r.NatGatewayId == rec.NatGatewayId { + hasNatRoute = true + break + } + } + if !hasNatRoute { + continue + } + for _, assoc := range rt.Associations { + if assoc.SubnetId == "" || assoc.Main { + continue + } + subnet, ok := subnets[assoc.SubnetId] + if !ok { + slog.Warn("reconcile/intent: NAT GW associated subnet not in intent, skipping", + "natgw_id", rec.NatGatewayId, "subnet_id", assoc.SubnetId) + continue + } + cidr := subnet.CIDR.String() + out[natgwSpecKey(rec.NatGatewayId, cidr)] = policy.NATGWSpec{ + VPCID: rec.VpcId, + NATGatewayID: rec.NatGatewayId, + PublicIP: rec.PublicIp, + SubnetCIDR: cidr, + } + emitted = true + } + } + if !emitted { + slog.Debug("reconcile/intent: NAT GW has no associated private subnets, skipping", + "natgw_id", rec.NatGatewayId, "vpc_id", rec.VpcId) + } + } + return nil +} + +// sgRulesToPolicyRules adapts handler-side SGRule to policy.Rule. +func sgRulesToPolicyRules(in []handlers_ec2_vpc.SGRule) []policy.Rule { + if len(in) == 0 { + return nil + } + out := make([]policy.Rule, len(in)) + for i, r := range in { + out[i] = policy.Rule{ + IPProtocol: r.IpProtocol, + FromPort: r.FromPort, + ToPort: r.ToPort, + CIDR: r.CidrIp, + SourceSG: r.SourceSG, + } + } + return out +} diff --git a/spinifex/network/reconcile/intent_test.go b/spinifex/network/reconcile/intent_test.go new file mode 100644 index 000000000..1431cb28a --- /dev/null +++ b/spinifex/network/reconcile/intent_test.go @@ -0,0 +1,269 @@ +package reconcile + +import ( + "context" + "encoding/json" + "testing" + "time" + + handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip" + handlers_ec2_igw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/igw" + handlers_ec2_natgw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/natgw" + handlers_ec2_routetable "github.com/mulgadc/spinifex/spinifex/handlers/ec2/routetable" + handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" + "github.com/mulgadc/spinifex/spinifex/testutil" +) + +func TestMatchesLocalAZ(t *testing.T) { + cases := []struct { + vpcAZ, localAZ string + want bool + }{ + {"", "us-east-1a", true}, // legacy record matches every AZ + {"us-east-1a", "us-east-1a", true}, + {"us-east-1b", "us-east-1a", false}, + {"us-east-1a", "", false}, // non-legacy vpc AZ does not match empty local AZ + } + for _, c := range cases { + if got := matchesLocalAZ(c.vpcAZ, c.localAZ); got != c.want { + t.Errorf("matchesLocalAZ(%q, %q) = %v, want %v", c.vpcAZ, c.localAZ, got, c.want) + } + } +} + +func TestLoadIntentFromKV_AZFilter(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + + localVPC := handlers_ec2_vpc.VPCRecord{ + VpcId: "vpc-local", CidrBlock: "10.0.0.0/16", State: "available", VNI: 100, AZ: "us-east-1a", CreatedAt: time.Now(), + } + foreignVPC := handlers_ec2_vpc.VPCRecord{ + VpcId: "vpc-foreign", CidrBlock: "10.1.0.0/16", State: "available", VNI: 101, AZ: "us-east-1b", CreatedAt: time.Now(), + } + legacyVPC := handlers_ec2_vpc.VPCRecord{ + VpcId: "vpc-legacy", CidrBlock: "10.2.0.0/16", State: "available", VNI: 102, CreatedAt: time.Now(), + } + testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ + "acct/" + localVPC.VpcId: mustJSON(t, localVPC), + "acct/" + foreignVPC.VpcId: mustJSON(t, foreignVPC), + "acct/" + legacyVPC.VpcId: mustJSON(t, legacyVPC), + }) + + intent, err := LoadIntentFromKV(context.Background(), js, "us-east-1a") + if err != nil { + t.Fatalf("LoadIntentFromKV: %v", err) + } + + if _, ok := intent.VPCs["vpc-local"]; !ok { + t.Errorf("local VPC missing from intent") + } + if _, ok := intent.VPCs["vpc-legacy"]; !ok { + t.Errorf("legacy (empty-AZ) VPC missing from intent — backwards-compat rule broken") + } + if _, ok := intent.VPCs["vpc-foreign"]; ok { + t.Errorf("foreign-AZ VPC leaked into intent") + } +} + +func TestLoadIntentFromKV_TransitiveSubnetFilter(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + + testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ + "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ + VpcId: "vpc-local", CidrBlock: "10.0.0.0/16", AZ: "us-east-1a", CreatedAt: time.Now(), + }), + "acct/vpc-foreign": mustJSON(t, handlers_ec2_vpc.VPCRecord{ + VpcId: "vpc-foreign", CidrBlock: "10.1.0.0/16", AZ: "us-east-1b", CreatedAt: time.Now(), + }), + }) + + testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketSubnets, map[string][]byte{ + "acct/subnet-local": mustJSON(t, handlers_ec2_vpc.SubnetRecord{ + SubnetId: "subnet-local", VpcId: "vpc-local", CidrBlock: "10.0.1.0/24", + }), + "acct/subnet-foreign": mustJSON(t, handlers_ec2_vpc.SubnetRecord{ + SubnetId: "subnet-foreign", VpcId: "vpc-foreign", CidrBlock: "10.1.1.0/24", + }), + }) + + intent, err := LoadIntentFromKV(context.Background(), js, "us-east-1a") + if err != nil { + t.Fatalf("LoadIntentFromKV: %v", err) + } + + if _, ok := intent.Subnets["subnet-local"]; !ok { + t.Errorf("local subnet missing from intent") + } + if _, ok := intent.Subnets["subnet-foreign"]; ok { + t.Errorf("foreign-AZ subnet leaked into intent — transitive filter broken") + } +} + +func TestLoadIntentFromKV_EIPStateFilter(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + + testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ + "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ + VpcId: "vpc-local", CidrBlock: "10.0.0.0/16", AZ: "us-east-1a", + }), + }) + testutil.SeedKV(t, js, handlers_ec2_eip.KVBucketEIPs, map[string][]byte{ + "acct/eipassoc-a": mustJSON(t, handlers_ec2_eip.EIPRecord{ + AllocationId: "eipalloc-a", PublicIp: "203.0.113.10", PrivateIp: "10.0.1.5", + VpcId: "vpc-local", State: "associated", + }), + "acct/eipassoc-b": mustJSON(t, handlers_ec2_eip.EIPRecord{ + AllocationId: "eipalloc-b", PublicIp: "203.0.113.11", PrivateIp: "10.0.1.6", + VpcId: "vpc-local", State: "allocated", // not associated → excluded + }), + }) + + intent, err := LoadIntentFromKV(context.Background(), js, "us-east-1a") + if err != nil { + t.Fatalf("LoadIntentFromKV: %v", err) + } + + if _, ok := intent.EIPs["10.0.1.5"]; !ok { + t.Errorf("associated EIP missing from intent") + } + if _, ok := intent.EIPs["10.0.1.6"]; ok { + t.Errorf("non-associated EIP leaked into intent") + } +} + +func TestLoadIntentFromKV_IGWAttachedFilter(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + + testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ + "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ + VpcId: "vpc-local", CidrBlock: "10.0.0.0/16", AZ: "us-east-1a", + }), + }) + testutil.SeedKV(t, js, handlers_ec2_igw.KVBucketIGW, map[string][]byte{ + "acct/igw-attached": mustJSON(t, handlers_ec2_igw.IGWRecord{ + InternetGatewayId: "igw-attached", VpcId: "vpc-local", State: "available", + }), + "acct/igw-detached": mustJSON(t, handlers_ec2_igw.IGWRecord{ + InternetGatewayId: "igw-detached", VpcId: "", State: "available", + }), + }) + + intent, err := LoadIntentFromKV(context.Background(), js, "us-east-1a") + if err != nil { + t.Fatalf("LoadIntentFromKV: %v", err) + } + + if _, ok := intent.IGWs["vpc-local"]; !ok { + t.Errorf("attached IGW missing from intent") + } + if len(intent.IGWs) != 1 { + t.Errorf("got %d IGWs, want 1 (detached should be excluded)", len(intent.IGWs)) + } +} + +// TestLoadIntentFromKV_NATGWUsesAssociatedSubnet enforces that loadNATGWs +// emits one NATGWSpec per *associated* private subnet (via route-table +// associations) — NOT the NATGW's own public home subnet. Bug observed in CI +// Phase 8d: reconciler installed snat for the NATGW's public-subnet CIDR +// (172.31.0.0/20) instead of the routed private-subnet CIDR (172.31.16.0/20), +// corrupting conntrack reverse-NAT and producing 100% packet loss. +func TestLoadIntentFromKV_NATGWUsesAssociatedSubnet(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + + testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ + "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ + VpcId: "vpc-local", CidrBlock: "172.31.0.0/16", AZ: "us-east-1a", + }), + }) + testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketSubnets, map[string][]byte{ + "acct/subnet-pub": mustJSON(t, handlers_ec2_vpc.SubnetRecord{SubnetId: "subnet-pub", VpcId: "vpc-local", CidrBlock: "172.31.0.0/20"}), + "acct/subnet-priv": mustJSON(t, handlers_ec2_vpc.SubnetRecord{SubnetId: "subnet-priv", VpcId: "vpc-local", CidrBlock: "172.31.16.0/20"}), + }) + testutil.SeedKV(t, js, handlers_ec2_natgw.KVBucketNatGateways, map[string][]byte{ + "acct/nat-1": mustJSON(t, handlers_ec2_natgw.NatGatewayRecord{ + NatGatewayId: "nat-1", VpcId: "vpc-local", SubnetId: "subnet-pub", + PublicIp: "203.0.113.50", State: "available", + }), + }) + testutil.SeedKV(t, js, handlers_ec2_routetable.KVBucketRouteTables, map[string][]byte{ + "acct/rtb-priv": mustJSON(t, handlers_ec2_routetable.RouteTableRecord{ + RouteTableId: "rtb-priv", VpcId: "vpc-local", + Routes: []handlers_ec2_routetable.RouteRecord{ + {DestinationCidrBlock: "0.0.0.0/0", NatGatewayId: "nat-1", State: "active"}, + }, + Associations: []handlers_ec2_routetable.AssociationRecord{ + {AssociationId: "rtbassoc-x", SubnetId: "subnet-priv"}, + }, + }), + }) + + intent, err := LoadIntentFromKV(context.Background(), js, "us-east-1a") + if err != nil { + t.Fatalf("LoadIntentFromKV: %v", err) + } + + if len(intent.NATGWs) != 1 { + t.Fatalf("got %d NATGW specs, want 1; intent=%#v", len(intent.NATGWs), intent.NATGWs) + } + for _, spec := range intent.NATGWs { + if spec.SubnetCIDR != "172.31.16.0/20" { + t.Errorf("SubnetCIDR=%q, want %q (private subnet routed via NATGW, not NATGW's home subnet)", + spec.SubnetCIDR, "172.31.16.0/20") + } + if spec.NATGatewayID != "nat-1" || spec.PublicIP != "203.0.113.50" { + t.Errorf("spec mismatch: %#v", spec) + } + } +} + +// TestLoadIntentFromKV_NATGWNoAssociationSkips guards against installing a +// SNAT for a NATGW that no route table points to. Without an associated +// private subnet there is no traffic to translate, and emitting the home +// subnet's CIDR would corrupt the SNAT table. +func TestLoadIntentFromKV_NATGWNoAssociationSkips(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + + testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketVPCs, map[string][]byte{ + "acct/vpc-local": mustJSON(t, handlers_ec2_vpc.VPCRecord{ + VpcId: "vpc-local", CidrBlock: "172.31.0.0/16", AZ: "us-east-1a", + }), + }) + testutil.SeedKV(t, js, handlers_ec2_vpc.KVBucketSubnets, map[string][]byte{ + "acct/subnet-pub": mustJSON(t, handlers_ec2_vpc.SubnetRecord{SubnetId: "subnet-pub", VpcId: "vpc-local", CidrBlock: "172.31.0.0/20"}), + }) + testutil.SeedKV(t, js, handlers_ec2_natgw.KVBucketNatGateways, map[string][]byte{ + "acct/nat-orphan": mustJSON(t, handlers_ec2_natgw.NatGatewayRecord{ + NatGatewayId: "nat-orphan", VpcId: "vpc-local", SubnetId: "subnet-pub", + PublicIp: "203.0.113.51", State: "available", + }), + }) + + intent, err := LoadIntentFromKV(context.Background(), js, "us-east-1a") + if err != nil { + t.Fatalf("LoadIntentFromKV: %v", err) + } + if len(intent.NATGWs) != 0 { + t.Errorf("NATGW with no associated subnets must not produce specs, got %#v", intent.NATGWs) + } +} + +func TestLoadIntentFromKV_NoBucketsTolerated(t *testing.T) { + _, _, js := testutil.StartTestJetStream(t) + + intent, err := LoadIntentFromKV(context.Background(), js, "us-east-1a") + if err != nil { + t.Fatalf("LoadIntentFromKV on empty cluster: %v", err) + } + if len(intent.VPCs)+len(intent.Subnets)+len(intent.Ports)+len(intent.SGs)+len(intent.IGWs)+len(intent.EIPs)+len(intent.NATGWs) != 0 { + t.Errorf("expected empty intent on fresh cluster, got %#v", intent) + } +} + +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return b +} diff --git a/spinifex/network/reconcile/liveness_test.go b/spinifex/network/reconcile/liveness_test.go new file mode 100644 index 000000000..c6abc1585 --- /dev/null +++ b/spinifex/network/reconcile/liveness_test.go @@ -0,0 +1,128 @@ +package reconcile + +import ( + "context" + "fmt" + "net" + "net/netip" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/network/external" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +// TestL1_ReconcilerConverges enforces ADR-0006 liveness clause L1. +// +// "While both the NATS KV and OVN NB DB are reachable, the reconciler +// eventually drives the delta to zero. Idempotent EnsureX methods make +// repeated runs safe." +// +// Two reconcile passes over IntentStates of N={1,3,8} VPCs: state on pass 2 +// must equal state on pass 1. +func TestL1_ReconcilerConverges(t *testing.T) { + for _, n := range []int{1, 3, 8} { + t.Run(fmt.Sprintf("N=%d", n), func(t *testing.T) { + rec, m := newTestReconciler(t) + ctx := context.Background() + intent := scaledIntent(t, n) + + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile #1: %v", err) + } + routers1 := len(m.Routers) + switches1 := len(m.Switches) + ports1 := len(m.Ports) + pgs1 := len(m.PortGroups) + + if err := rec.Reconcile(ctx, intent); err != nil { + t.Fatalf("Reconcile #2: %v", err) + } + if len(m.Routers) != routers1 { + t.Errorf("ADR-0006 L1: convergence violated — routers %d → %d on second pass", routers1, len(m.Routers)) + } + if len(m.Switches) != switches1 { + t.Errorf("ADR-0006 L1: convergence violated — switches %d → %d on second pass", switches1, len(m.Switches)) + } + if len(m.Ports) != ports1 { + t.Errorf("ADR-0006 L1: convergence violated — ports %d → %d on second pass", ports1, len(m.Ports)) + } + if len(m.PortGroups) != pgs1 { + t.Errorf("ADR-0006 L1: convergence violated — port groups %d → %d on second pass", pgs1, len(m.PortGroups)) + } + }) + } +} + +// TestL2_DriftIntervalBounded enforces ADR-0006 liveness clause L2. +// +// "While the leader is active, the periodic drift-detection loop +// eventually re-runs and re-applies any divergence introduced since +// the last reconciler run." +// +// Structural-only: bounds DriftInterval to (0, 30m]. Full behavioural +// coverage blocked on a clock-injectable DriftLoop signature. +func TestL2_DriftIntervalBounded(t *testing.T) { + if DriftInterval <= 0 { + t.Fatalf("ADR-0006 L2: DriftInterval must be > 0; got %v", DriftInterval) + } + if DriftInterval > 30*time.Minute { + t.Fatalf("ADR-0006 L2: DriftInterval %v exceeds 30m support-window upper bound", DriftInterval) + } + // TODO: inject a fake clock + fake JetStream into DriftLoop and assert + // runDriftCycle is invoked at least twice within ctx.Deadline(). + // Blocked on a clock-injectable DriftLoop signature. +} + +// TestL3_FederationReEnablesOnLinkRecovery is a placeholder for ADR-0006 L3. +// +// "Once an inter-AZ link reaches ≥ Degraded class (ADR-0003), L4 +// eventually calls BringUpLink and establishes the OVN-IC tunnel to +// the peer AZ." +func TestL3_FederationReEnablesOnLinkRecovery(t *testing.T) { + t.Skip(`ADR-0006 L3 deferred: network/federation/ has not yet been built. ` + + `Once it lands, replace this stub with a test that drives a fake ` + + `LinkObserver from Down → Degraded and asserts BringUpLink fires.`) +} + +// scaledIntent produces an IntentState with n VPCs, each holding one +// subnet, one ENI port, and one SG. +func scaledIntent(t *testing.T, n int) IntentState { + t.Helper() + intent := IntentState{ + VPCs: make(map[string]topology.VPCSpec, n), + Subnets: make(map[string]topology.SubnetSpec, n), + Ports: make(map[string]topology.PortSpec, n), + SGs: make(map[string]policy.SGSpec, n), + IGWs: map[string]external.IGWSpec{}, + EIPs: map[string]policy.EIPSpec{}, + NATGWs: map[string]policy.NATGWSpec{}, + } + for i := range n { + vpcID := fmt.Sprintf("vpc-l1-%d", i) + subnetID := fmt.Sprintf("subnet-l1-%d", i) + eniID := fmt.Sprintf("eni-l1-%d", i) + sgID := fmt.Sprintf("sg-l1-%d", i) + + base := byte(i) + mac, _ := net.ParseMAC(fmt.Sprintf("02:00:00:00:00:%02x", base+1)) + + intent.VPCs[vpcID] = topology.VPCSpec{ + VPCID: vpcID, + CIDR: netip.MustParsePrefix(fmt.Sprintf("10.%d.0.0/16", base)), + VNI: int64(100 + i), + } + intent.Subnets[subnetID] = topology.SubnetSpec{ + SubnetID: subnetID, VPCID: vpcID, + CIDR: netip.MustParsePrefix(fmt.Sprintf("10.%d.1.0/24", base)), + } + intent.Ports[eniID] = topology.PortSpec{ + PortID: eniID, SubnetID: subnetID, VPCID: vpcID, + PrivateIP: netip.MustParseAddr(fmt.Sprintf("10.%d.1.10", base)), + MAC: mac, SGIDs: []string{sgID}, + } + intent.SGs[sgID] = policy.SGSpec{GroupID: sgID, VPCID: vpcID} + } + return intent +} diff --git a/spinifex/network/reconcile/lock.go b/spinifex/network/reconcile/lock.go new file mode 100644 index 000000000..07da0f5c1 --- /dev/null +++ b/spinifex/network/reconcile/lock.go @@ -0,0 +1,63 @@ +package reconcile + +import ( + "log/slog" + "time" + + "github.com/nats-io/nats.go" +) + +// Single CAS-elected leader key; TTL bounds crash-recovery. +const ( + reconcileLeaderBucket = "spinifex-vpcd-reconcile" + reconcileLeaderKey = "leader" + reconcileLeaderTTL = 60 * time.Second +) + +// Bounded wait for JetStream quorum on cold multi-node start. Vars (not +// consts) so tests can shrink them. +var ( + leaderRetryFor = 60 * time.Second + leaderRetryStep = 1 * time.Second +) + +// AcquireLeader returns (release, true) exactly once across all vpcds, or +// (nil, false) for losers and JetStream-unreachable cases. +func AcquireLeader(nc *nats.Conn, holder string) (func(), bool) { + js, _ := nc.JetStream() + + var ( + kv nats.KeyValue + err error + ) + deadline := time.Now().Add(leaderRetryFor) + for { + kv, err = js.CreateKeyValue(&nats.KeyValueConfig{ + Bucket: reconcileLeaderBucket, + History: 1, + TTL: reconcileLeaderTTL, + }) + if err == nil { + break + } + if time.Now().After(deadline) { + slog.Error("reconcile/lock: JetStream KV unreachable after retry, skipping reconcile", + "holder", holder, "waited", leaderRetryFor, "err", err) + return nil, false + } + slog.Debug("reconcile/lock: JetStream KV not ready, retrying", "holder", holder, "err", err) + time.Sleep(leaderRetryStep) + } + + if _, err := kv.Create(reconcileLeaderKey, []byte(holder)); err != nil { + slog.Info("reconcile/lock: another vpcd is leader, skipping reconcile", "holder", holder, "err", err) + return nil, false + } + + slog.Info("reconcile/lock: elected", "holder", holder) + return func() { + if err := kv.Delete(reconcileLeaderKey); err != nil { + slog.Warn("reconcile/lock: failed to release lock (TTL will reap)", "holder", holder, "err", err) + } + }, true +} diff --git a/spinifex/network/reconcile/reconcile.go b/spinifex/network/reconcile/reconcile.go new file mode 100644 index 000000000..192e44091 --- /dev/null +++ b/spinifex/network/reconcile/reconcile.go @@ -0,0 +1,134 @@ +// Package reconcile is the network stack's intent-actual reconciliation +// layer. It loads desired state from the JetStream KV snapshot (scoped to +// the local AZ) and applies the diff against OVN NB in a single +// topologically-sorted pass (VPC → Subnet → SG → Port → IGW → EIP → +// NATGW). Deletes run reverse order. Drift fires every 5 minutes. +package reconcile + +import ( + "context" + "errors" + "fmt" + "log/slog" + + "github.com/mulgadc/spinifex/spinifex/network/external" + "github.com/mulgadc/spinifex/spinifex/network/ovn" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/topology" +) + +// Reconciler converges OVN NB DB to a declared IntentState. Implementations +// are idempotent: a second call with the same IntentState is a no-op. +type Reconciler interface { + Reconcile(ctx context.Context, intent IntentState) error + // ReconcileApplyOnly skips orphan-pruning. Startup uses this to avoid + // racing peer subscribers that haven't processed in-flight create events + // yet; legitimate orphans are pruned on the next drift tick. + ReconcileApplyOnly(ctx context.Context, intent IntentState) error +} + +// Config is the construction-time bag for the reconciler. All fields except +// Chassis are required. +type Config struct { + OVN ovn.Client + SG policy.SecurityGroupManager + NAT policy.NATManager + Routes policy.RouteManager + IGW external.IGWManager + Topology topology.Manager + LocalAZ string + // NodeHostname is the holder identity for leader-election CAS. + NodeHostname string + // Chassis is the SBDB-discovered chassis list for gateway LRP rebinding. + Chassis []string +} + +type reconciler struct { + ovn ovn.Client + sg policy.SecurityGroupManager + nat policy.NATManager + routes policy.RouteManager + igw external.IGWManager + topology topology.Manager + localAZ string + host string + chassis []string +} + +var _ Reconciler = (*reconciler)(nil) + +// New constructs a Reconciler from cfg. Returns an error when any required +// field is missing. +func New(cfg Config) (Reconciler, error) { + switch { + case cfg.OVN == nil: + return nil, errors.New("reconcile: OVN client required") + case cfg.SG == nil: + return nil, errors.New("reconcile: SecurityGroupManager required") + case cfg.NAT == nil: + return nil, errors.New("reconcile: NATManager required") + case cfg.Routes == nil: + return nil, errors.New("reconcile: RouteManager required") + case cfg.IGW == nil: + return nil, errors.New("reconcile: IGWManager required") + case cfg.Topology == nil: + return nil, errors.New("reconcile: Topology manager required") + case cfg.NodeHostname == "": + return nil, errors.New("reconcile: NodeHostname required") + } + return &reconciler{ + ovn: cfg.OVN, + sg: cfg.SG, + nat: cfg.NAT, + routes: cfg.Routes, + igw: cfg.IGW, + topology: cfg.Topology, + localAZ: cfg.LocalAZ, + host: cfg.NodeHostname, + chassis: cfg.Chassis, + }, nil +} + +// Reconcile diffs intent vs. actual OVN state and applies create/delete +// per-resource: VPC → Subnet → SG → Port → IGW → EIP → NATGW (reverse for +// deletes). SG precedes Port so ENI LSPs join their PGs atomically via +// CreateLogicalSwitchPortInGroups. Per-stage errors are logged and the next +// stage runs; only an actual-state scan failure is returned. +func (r *reconciler) Reconcile(ctx context.Context, intent IntentState) error { + return r.reconcile(ctx, intent, true) +} + +// ReconcileApplyOnly is documented on the Reconciler interface. +func (r *reconciler) ReconcileApplyOnly(ctx context.Context, intent IntentState) error { + return r.reconcile(ctx, intent, false) +} + +func (r *reconciler) reconcile(ctx context.Context, intent IntentState, pruneOrphans bool) error { + actual, err := scanActual(ctx, r.ovn) + if err != nil { + return fmt.Errorf("scan actual OVN state: %w", err) + } + + slog.Info("reconcile: starting", + "local_az", r.localAZ, + "prune_orphans", pruneOrphans, + "intent_vpcs", len(intent.VPCs), + "intent_subnets", len(intent.Subnets), + "intent_ports", len(intent.Ports), + "intent_sgs", len(intent.SGs), + "intent_igws", len(intent.IGWs), + "intent_eips", len(intent.EIPs), + "intent_natgws", len(intent.NATGWs), + ) + + r.applyVPCs(ctx, intent, actual) + r.applySubnets(ctx, intent, actual) + r.applySGs(ctx, intent, actual, pruneOrphans) + r.applyPorts(ctx, intent, actual) + r.applyIGWs(ctx, intent, actual) + r.applyEIPs(ctx, intent, actual) + r.applyNATGWs(ctx, intent, actual) + + slog.Info("reconcile: complete") + return nil +} diff --git a/spinifex/network/subscribers/events.go b/spinifex/network/subscribers/events.go new file mode 100644 index 000000000..e5add8d4c --- /dev/null +++ b/spinifex/network/subscribers/events.go @@ -0,0 +1,90 @@ +package subscribers + +// NATS topic names for VPC lifecycle events published by daemon. +const ( + TopicVPCCreate = "vpc.create" + TopicVPCDelete = "vpc.delete" + TopicSubnetCreate = "vpc.create-subnet" + TopicSubnetDelete = "vpc.delete-subnet" + TopicCreatePort = "vpc.create-port" + TopicDeletePort = "vpc.delete-port" + TopicUpdatePortSGs = "vpc.update-port-sgs" + TopicIGWAttach = "vpc.igw-attach" + TopicIGWDetach = "vpc.igw-detach" + TopicAddNAT = "vpc.add-nat" + TopicDeleteNAT = "vpc.delete-nat" + TopicAddNATGateway = "vpc.add-nat-gateway" + TopicDeleteNATGateway = "vpc.delete-nat-gateway" + TopicCreateSG = "vpc.create-sg" + TopicDeleteSG = "vpc.delete-sg" + TopicUpdateSG = "vpc.update-sg" +) + +// VPCEvent is published on vpc.create / vpc.delete. +type VPCEvent struct { + VpcId string `json:"vpc_id"` + CidrBlock string `json:"cidr_block"` + VNI int64 `json:"vni"` +} + +// SubnetEvent is published on vpc.create-subnet / vpc.delete-subnet. +type SubnetEvent struct { + SubnetId string `json:"subnet_id"` + VpcId string `json:"vpc_id"` + CidrBlock string `json:"cidr_block"` +} + +// PortEvent: vpc.create-port / vpc.delete-port. SecurityGroupIds set on +// create so OVN PG membership is wired atomically with the LSP; empty on +// delete (handleDeletePort reads current memberships from the cache). +type PortEvent struct { + NetworkInterfaceId string `json:"network_interface_id"` + SubnetId string `json:"subnet_id"` + VpcId string `json:"vpc_id"` + PrivateIpAddress string `json:"private_ip_address"` + MacAddress string `json:"mac_address"` + SecurityGroupIds []string `json:"security_group_ids,omitempty"` +} + +// UpdatePortSGsEvent: vpc.update-port-sgs. Declarative — vpcd diffs +// SecurityGroupIds against the current cache memberships. +type UpdatePortSGsEvent struct { + NetworkInterfaceId string `json:"network_interface_id"` + PrivateIpAddress string `json:"private_ip_address"` + SecurityGroupIds []string `json:"security_group_ids"` +} + +// NATEvent is published on vpc.add-nat / vpc.delete-nat for 1:1 public IP NAT. +type NATEvent struct { + VpcId string `json:"vpc_id"` + ExternalIP string `json:"external_ip"` + LogicalIP string `json:"logical_ip"` + PortName string `json:"port_name"` // logical port for distributed NAT + MAC string `json:"mac"` // external MAC for distributed NAT +} + +// NATGatewayEvent is published on vpc.add-nat-gateway / vpc.delete-nat-gateway. +type NATGatewayEvent struct { + VpcId string `json:"vpc_id"` + NatGatewayId string `json:"nat_gateway_id"` + PublicIp string `json:"public_ip"` + SubnetCidr string `json:"subnet_cidr"` // private subnet CIDR for SNAT rule +} + +// SGRule mirrors handlers/ec2/vpc.SGRule on the wire (kept local to avoid +// the handler import). +type SGRule struct { + IpProtocol string `json:"ip_protocol"` + FromPort int64 `json:"from_port"` + ToPort int64 `json:"to_port"` + CidrIp string `json:"cidr_ip,omitempty"` + SourceSG string `json:"source_sg,omitempty"` +} + +// SGEvent mirrors handlers/ec2/vpc.SGEvent. +type SGEvent struct { + GroupId string `json:"group_id"` + VpcId string `json:"vpc_id"` + IngressRules []SGRule `json:"ingress_rules,omitempty"` + EgressRules []SGRule `json:"egress_rules,omitempty"` +} diff --git a/spinifex/network/subscribers/respond.go b/spinifex/network/subscribers/respond.go new file mode 100644 index 000000000..21839b43d --- /dev/null +++ b/spinifex/network/subscribers/respond.go @@ -0,0 +1,34 @@ +package subscribers + +import ( + "encoding/json" + "log/slog" + + "github.com/nats-io/nats.go" +) + +type respondResponse struct { + Success bool `json:"success"` + Error string `json:"error,omitempty"` +} + +// respond sends a JSON success/error envelope to a NATS request. Fire-and- +// forget when no reply is set on msg. +func respond(msg *nats.Msg, err error) { + if msg.Reply == "" { + return + } + resp := respondResponse{Success: true} + if err != nil { + resp.Success = false + resp.Error = err.Error() + } + data, marshalErr := json.Marshal(resp) + if marshalErr != nil { + slog.Error("subscribers: failed to marshal NATS response", "err", marshalErr) + data = []byte(`{"success":false,"error":"internal marshal failure"}`) + } + if err := msg.Respond(data); err != nil { + slog.Error("subscribers: failed to respond to NATS request", "err", err) + } +} diff --git a/spinifex/network/subscribers/security.go b/spinifex/network/subscribers/security.go new file mode 100644 index 000000000..1e33a679e --- /dev/null +++ b/spinifex/network/subscribers/security.go @@ -0,0 +1,100 @@ +package subscribers + +import ( + "context" + "encoding/json" + "log/slog" + + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/nats-io/nats.go" +) + +func (e SGEvent) toSpec() policy.SGSpec { + return policy.SGSpec{ + GroupID: e.GroupId, + VPCID: e.VpcId, + IngressRules: toPolicyRules(e.IngressRules), + EgressRules: toPolicyRules(e.EgressRules), + } +} + +func toPolicyRules(in []SGRule) []policy.Rule { + out := make([]policy.Rule, len(in)) + for i, r := range in { + out[i] = policy.Rule{ + IPProtocol: r.IpProtocol, + FromPort: r.FromPort, + ToPort: r.ToPort, + CIDR: r.CidrIp, + SourceSG: r.SourceSG, + } + } + return out +} + +// handleCreateSG ensures the PG (L2) then applies the ACL set (L3). +func (s *Subscriber) handleCreateSG(msg *nats.Msg) { + var evt SGEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.create-sg event", "err", err) + respond(msg, err) + return + } + ctx := context.Background() + if err := s.topology.EnsureSGPortGroup(ctx, evt.GroupId); err != nil { + slog.Error("subscribers: EnsureSGPortGroup failed", "group_id", evt.GroupId, "err", err) + respond(msg, err) + return + } + if err := s.sg.EnsureSG(ctx, evt.toSpec()); err != nil { + slog.Error("subscribers: EnsureSG failed", "group_id", evt.GroupId, "err", err) + respond(msg, err) + return + } + slog.Info("subscribers: created security group", + "group_id", evt.GroupId, + "vpc_id", evt.VpcId, + "ingress_rules", len(evt.IngressRules), + "egress_rules", len(evt.EgressRules), + ) + respond(msg, nil) +} + +// handleDeleteSG removes the SG's PG and its ACLs; idempotent. +func (s *Subscriber) handleDeleteSG(msg *nats.Msg) { + var evt SGEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.delete-sg event", "err", err) + respond(msg, err) + return + } + if err := s.topology.DeleteSGPortGroup(context.Background(), evt.GroupId); err != nil { + slog.Error("subscribers: DeleteSGPortGroup failed", "group_id", evt.GroupId, "err", err) + respond(msg, err) + return + } + slog.Info("subscribers: deleted security group", "group_id", evt.GroupId, "vpc_id", evt.VpcId) + respond(msg, nil) +} + +// handleUpdateSG replaces the ACL set; the PG is unaffected. +func (s *Subscriber) handleUpdateSG(msg *nats.Msg) { + var evt SGEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.update-sg event", "err", err) + respond(msg, err) + return + } + if err := s.sg.UpdateSG(context.Background(), evt.toSpec()); err != nil { + slog.Error("subscribers: UpdateSG failed", "group_id", evt.GroupId, "err", err) + respond(msg, err) + return + } + slog.Info("subscribers: updated security group", + "group_id", evt.GroupId, + "vpc_id", evt.VpcId, + "ingress_rules", len(evt.IngressRules), + "egress_rules", len(evt.EgressRules), + ) + respond(msg, nil) +} diff --git a/spinifex/network/subscribers/subscribers.go b/spinifex/network/subscribers/subscribers.go new file mode 100644 index 000000000..be5089a39 --- /dev/null +++ b/spinifex/network/subscribers/subscribers.go @@ -0,0 +1,99 @@ +// Package subscribers translates VPC lifecycle NATS events into calls +// against the network/{topology,policy,external} managers. +package subscribers + +import ( + "errors" + "fmt" + "log/slog" + + "github.com/mulgadc/spinifex/spinifex/network/external" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/topology" + "github.com/nats-io/nats.go" +) + +// QueueGroup ensures one vpcd processes each event (shared OVN NB DB). +const QueueGroup = "vpcd-workers" + +// Subscriber wires VPC lifecycle NATS topics to the network managers. +type Subscriber struct { + topology topology.Manager + sg policy.SecurityGroupManager + eip external.EIPManager + natgw external.NATGWManager + igw external.IGWManager +} + +// Config: all fields required. +type Config struct { + Topology topology.Manager + SG policy.SecurityGroupManager + EIP external.EIPManager + NATGW external.NATGWManager + IGW external.IGWManager +} + +// New constructs a Subscriber, returning an error when any manager is nil. +func New(cfg Config) (*Subscriber, error) { + switch { + case cfg.Topology == nil: + return nil, errors.New("subscribers: Topology manager required") + case cfg.SG == nil: + return nil, errors.New("subscribers: SecurityGroupManager required") + case cfg.EIP == nil: + return nil, errors.New("subscribers: EIPManager required") + case cfg.NATGW == nil: + return nil, errors.New("subscribers: NATGWManager required") + case cfg.IGW == nil: + return nil, errors.New("subscribers: IGWManager required") + } + return &Subscriber{ + topology: cfg.Topology, + sg: cfg.SG, + eip: cfg.EIP, + natgw: cfg.NATGW, + igw: cfg.IGW, + }, nil +} + +// Subscribe registers queue subs for every VPC lifecycle topic. On +// partial failure, unsubscribes prior subs before returning the error. +func (s *Subscriber) Subscribe(nc *nats.Conn) ([]*nats.Subscription, error) { + type sub struct { + topic string + handler nats.MsgHandler + } + subs := []sub{ + {TopicVPCCreate, s.handleVPCCreate}, + {TopicVPCDelete, s.handleVPCDelete}, + {TopicSubnetCreate, s.handleSubnetCreate}, + {TopicSubnetDelete, s.handleSubnetDelete}, + {TopicCreatePort, s.handleCreatePort}, + {TopicDeletePort, s.handleDeletePort}, + {TopicUpdatePortSGs, s.handleUpdatePortSGs}, + {TopicIGWAttach, s.handleIGWAttach}, + {TopicIGWDetach, s.handleIGWDetach}, + {TopicAddNAT, s.handleAddNAT}, + {TopicDeleteNAT, s.handleDeleteNAT}, + {TopicAddNATGateway, s.handleAddNATGateway}, + {TopicDeleteNATGateway, s.handleDeleteNATGateway}, + {TopicCreateSG, s.handleCreateSG}, + {TopicDeleteSG, s.handleDeleteSG}, + {TopicUpdateSG, s.handleUpdateSG}, + } + + var result []*nats.Subscription + for _, item := range subs { + natsSub, err := nc.QueueSubscribe(item.topic, QueueGroup, item.handler) + if err != nil { + for _, r := range result { + _ = r.Unsubscribe() + } + return nil, fmt.Errorf("subscribe %s: %w", item.topic, err) + } + result = append(result, natsSub) + slog.Info("Subscribed to VPC topic", "topic", item.topic) + } + return result, nil +} diff --git a/spinifex/network/subscribers/subscribers_test.go b/spinifex/network/subscribers/subscribers_test.go new file mode 100644 index 000000000..595553c82 --- /dev/null +++ b/spinifex/network/subscribers/subscribers_test.go @@ -0,0 +1,149 @@ +package subscribers + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/mulgadc/spinifex/spinifex/network/external" + "github.com/mulgadc/spinifex/spinifex/network/ovn/mock" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/topology" + "github.com/mulgadc/spinifex/spinifex/testutil" +) + +const requestTimeout = 2 * time.Second + +// newTestSubscriber wires every manager against one mock OVN client. +func newTestSubscriber(t *testing.T) (*Subscriber, *mock.Client) { + t.Helper() + m := mock.New() + _ = m.Connect(context.Background()) + + topo := topology.NewLiveManager(m) + sg := policy.NewSecurityGroupManager(m) + nat, err := policy.NewNATManager(m, policy.NATModeDistributed) + if err != nil { + t.Fatalf("NewNATManager: %v", err) + } + routes := policy.NewRouteManager(m) + igw, err := external.NewIGWManager(external.IGWManagerConfig{ + OVN: m, + Routes: routes, + NAT: nat, + Allocator: external.NewStaticRangeAllocator(m), + Chassis: []string{"hv1"}, + NATMode: policy.NATModeDistributed, + }) + if err != nil { + t.Fatalf("NewIGWManager: %v", err) + } + eip, err := external.NewEIPManager(nat, nil) + if err != nil { + t.Fatalf("NewEIPManager: %v", err) + } + natgw, err := external.NewNATGWManager(nat) + if err != nil { + t.Fatalf("NewNATGWManager: %v", err) + } + s, err := New(Config{Topology: topo, SG: sg, EIP: eip, NATGW: natgw, IGW: igw}) + if err != nil { + t.Fatalf("New: %v", err) + } + return s, m +} + +func TestNew_RejectsMissingDeps(t *testing.T) { + cases := []struct { + name string + cfg Config + }{ + {"no topology", Config{SG: policy.NewSecurityGroupManager(mock.New())}}, + {"no sg", Config{Topology: topology.NewLiveManager(mock.New())}}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if _, err := New(c.cfg); err == nil { + t.Fatal("expected error, got nil") + } + }) + } +} + +func TestSubscribe_RegistersAllTopics(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + sub, _ := newTestSubscriber(t) + + subs, err := sub.Subscribe(nc) + if err != nil { + t.Fatalf("Subscribe: %v", err) + } + defer func() { + for _, s := range subs { + _ = s.Unsubscribe() + } + }() + + wantTopics := []string{ + TopicVPCCreate, TopicVPCDelete, + TopicSubnetCreate, TopicSubnetDelete, + TopicCreatePort, TopicDeletePort, TopicUpdatePortSGs, + TopicIGWAttach, TopicIGWDetach, + TopicAddNAT, TopicDeleteNAT, + TopicAddNATGateway, TopicDeleteNATGateway, + TopicCreateSG, TopicDeleteSG, TopicUpdateSG, + } + if len(subs) != len(wantTopics) { + t.Fatalf("subscription count: got %d, want %d", len(subs), len(wantTopics)) + } + got := map[string]bool{} + for _, s := range subs { + got[s.Subject] = true + } + for _, topic := range wantTopics { + if !got[topic] { + t.Errorf("missing subscription for topic %q", topic) + } + } +} + +// Every request/reply topic must return a structured error envelope on bad +// JSON. Fire-and-forget topics (NAT gateway) just must not panic. +func TestBadJSON_AllRequestReplies(t *testing.T) { + _, nc := testutil.StartTestNATS(t) + sub, _ := newTestSubscriber(t) + subs, err := sub.Subscribe(nc) + if err != nil { + t.Fatalf("Subscribe: %v", err) + } + defer func() { + for _, s := range subs { + _ = s.Unsubscribe() + } + }() + + topics := []string{ + TopicVPCCreate, TopicVPCDelete, + TopicSubnetCreate, TopicSubnetDelete, + TopicCreatePort, TopicDeletePort, TopicUpdatePortSGs, + TopicIGWAttach, TopicIGWDetach, + TopicAddNAT, TopicDeleteNAT, + TopicCreateSG, TopicDeleteSG, TopicUpdateSG, + } + for _, topic := range topics { + t.Run(topic, func(t *testing.T) { + resp, err := nc.Request(topic, []byte("not json"), requestTimeout) + if err != nil { + t.Fatalf("request %s: %v", topic, err) + } + var env respondResponse + if err := json.Unmarshal(resp.Data, &env); err != nil { + t.Fatalf("unmarshal envelope: %v", err) + } + if env.Success { + t.Errorf("topic %s: expected success=false on bad JSON, got success=true", topic) + } + }) + } +} diff --git a/spinifex/network/subscribers/topology.go b/spinifex/network/subscribers/topology.go new file mode 100644 index 000000000..9446b6170 --- /dev/null +++ b/spinifex/network/subscribers/topology.go @@ -0,0 +1,298 @@ +package subscribers + +import ( + "context" + "encoding/json" + "log/slog" + "net" + "net/netip" + + "github.com/mulgadc/spinifex/spinifex/network/external" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/topology" + "github.com/mulgadc/spinifex/spinifex/types" + "github.com/nats-io/nats.go" +) + +func (s *Subscriber) handleVPCCreate(msg *nats.Msg) { + var evt VPCEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.create event", "err", err) + respond(msg, err) + return + } + spec := topology.VPCSpec{VPCID: evt.VpcId, VNI: evt.VNI} + if evt.CidrBlock != "" { + cidr, err := netip.ParsePrefix(evt.CidrBlock) + if err != nil { + slog.Error("subscribers: invalid CIDR in vpc.create event", "cidr", evt.CidrBlock, "err", err) + respond(msg, err) + return + } + spec.CIDR = cidr + } + if err := s.topology.EnsureVPC(context.Background(), spec); err != nil { + slog.Error("subscribers: EnsureVPC failed", "vpc_id", evt.VpcId, "err", err) + respond(msg, err) + return + } + respond(msg, nil) +} + +func (s *Subscriber) handleVPCDelete(msg *nats.Msg) { + var evt VPCEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.delete event", "err", err) + respond(msg, err) + return + } + if err := s.topology.DeleteVPC(context.Background(), evt.VpcId); err != nil { + slog.Error("subscribers: DeleteVPC failed", "vpc_id", evt.VpcId, "err", err) + respond(msg, err) + return + } + respond(msg, nil) +} + +func (s *Subscriber) handleSubnetCreate(msg *nats.Msg) { + var evt SubnetEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.create-subnet event", "err", err) + respond(msg, err) + return + } + cidr, err := netip.ParsePrefix(evt.CidrBlock) + if err != nil { + slog.Error("subscribers: invalid CIDR in vpc.create-subnet event", "cidr", evt.CidrBlock, "err", err) + respond(msg, err) + return + } + ctx := context.Background() + // vpc.create and vpc.create-subnet have no ordering guarantee; pre-ensure + // the VPC router so a tight bootstrap doesn't fail with "router not + // found". EnsureVPC is idempotent. + if err := s.topology.EnsureVPC(ctx, topology.VPCSpec{VPCID: evt.VpcId}); err != nil { + slog.Error("subscribers: EnsureVPC (subnet pre-create) failed", "vpc_id", evt.VpcId, "err", err) + respond(msg, err) + return + } + if err := s.topology.EnsureSubnet(ctx, topology.SubnetSpec{ + SubnetID: evt.SubnetId, + VPCID: evt.VpcId, + CIDR: cidr, + }); err != nil { + slog.Error("subscribers: EnsureSubnet failed", "subnet_id", evt.SubnetId, "err", err) + respond(msg, err) + return + } + respond(msg, nil) +} + +func (s *Subscriber) handleSubnetDelete(msg *nats.Msg) { + var evt SubnetEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.delete-subnet event", "err", err) + respond(msg, err) + return + } + spec := topology.SubnetSpec{SubnetID: evt.SubnetId, VPCID: evt.VpcId} + if evt.CidrBlock != "" { + if cidr, perr := netip.ParsePrefix(evt.CidrBlock); perr == nil { + spec.CIDR = cidr + } + } + if err := s.topology.DeleteSubnet(context.Background(), spec); err != nil { + slog.Error("subscribers: DeleteSubnet failed", "subnet_id", evt.SubnetId, "err", err) + respond(msg, err) + return + } + respond(msg, nil) +} + +func (s *Subscriber) handleCreatePort(msg *nats.Msg) { + var evt PortEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.create-port event", "err", err) + respond(msg, err) + return + } + ip, err := netip.ParseAddr(evt.PrivateIpAddress) + if err != nil { + slog.Error("subscribers: invalid private IP in vpc.create-port event", "ip", evt.PrivateIpAddress, "err", err) + respond(msg, err) + return + } + mac, err := net.ParseMAC(evt.MacAddress) + if err != nil { + slog.Error("subscribers: invalid MAC in vpc.create-port event", "mac", evt.MacAddress, "err", err) + respond(msg, err) + return + } + if err := s.topology.EnsurePort(context.Background(), topology.PortSpec{ + PortID: evt.NetworkInterfaceId, + SubnetID: evt.SubnetId, + VPCID: evt.VpcId, + PrivateIP: ip, + MAC: mac, + SGIDs: evt.SecurityGroupIds, + }); err != nil { + slog.Error("subscribers: EnsurePort failed", "eni_id", evt.NetworkInterfaceId, "err", err) + respond(msg, err) + return + } + respond(msg, nil) +} + +func (s *Subscriber) handleDeletePort(msg *nats.Msg) { + var evt PortEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.delete-port event", "err", err) + respond(msg, err) + return + } + if err := s.topology.DeletePort(context.Background(), topology.PortSpec{ + PortID: evt.NetworkInterfaceId, + SubnetID: evt.SubnetId, + VPCID: evt.VpcId, + }); err != nil { + slog.Error("subscribers: DeletePort failed", "eni_id", evt.NetworkInterfaceId, "err", err) + respond(msg, err) + return + } + respond(msg, nil) +} + +// handleUpdatePortSGs reconciles the LSP's PG memberships against the +// declarative SG list; the manager computes the diff. +func (s *Subscriber) handleUpdatePortSGs(msg *nats.Msg) { + var evt UpdatePortSGsEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.update-port-sgs event", "err", err) + respond(msg, err) + return + } + if err := s.topology.SetPortSecurityGroups(context.Background(), evt.NetworkInterfaceId, evt.SecurityGroupIds); err != nil { + slog.Error("subscribers: SetPortSecurityGroups failed", + "eni_id", evt.NetworkInterfaceId, "sgs", evt.SecurityGroupIds, "err", err) + respond(msg, err) + return + } + respond(msg, nil) +} + +func (s *Subscriber) handleIGWAttach(msg *nats.Msg) { + var evt types.IGWEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.igw-attach event", "err", err) + respond(msg, err) + return + } + if err := s.igw.AttachIGW(context.Background(), external.IGWSpec{ + VPCID: evt.VpcId, + InternetGatewayID: evt.InternetGatewayId, + }); err != nil { + slog.Error("subscribers: AttachIGW failed", + "vpc_id", evt.VpcId, "igw_id", evt.InternetGatewayId, "err", err) + respond(msg, err) + return + } + respond(msg, nil) +} + +func (s *Subscriber) handleIGWDetach(msg *nats.Msg) { + var evt types.IGWEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.igw-detach event", "err", err) + respond(msg, err) + return + } + if err := s.igw.DetachIGW(context.Background(), evt.VpcId); err != nil { + slog.Error("subscribers: DetachIGW failed", "vpc_id", evt.VpcId, "err", err) + respond(msg, err) + return + } + slog.Info("subscribers: detached internet gateway from VPC", + "igw_id", evt.InternetGatewayId, "vpc_id", evt.VpcId) + respond(msg, nil) +} + +func (s *Subscriber) handleAddNAT(msg *nats.Msg) { + var evt NATEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.add-nat event", "err", err) + respond(msg, err) + return + } + if err := s.eip.AttachEIP(context.Background(), policy.EIPSpec{ + VPCID: evt.VpcId, + ExternalIP: evt.ExternalIP, + LogicalIP: evt.LogicalIP, + PortName: evt.PortName, + MAC: evt.MAC, + }); err != nil { + slog.Error("subscribers: AddEIP failed", + "vpc_id", evt.VpcId, "external_ip", evt.ExternalIP, + "logical_ip", evt.LogicalIP, "err", err) + respond(msg, err) + return + } + slog.Info("subscribers: added dnat_and_snat rule", + "vpc_id", evt.VpcId, "external_ip", evt.ExternalIP, + "logical_ip", evt.LogicalIP, "port", evt.PortName) + respond(msg, nil) +} + +func (s *Subscriber) handleDeleteNAT(msg *nats.Msg) { + var evt NATEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.delete-nat event", "err", err) + respond(msg, err) + return + } + if err := s.eip.DetachEIP(context.Background(), evt.VpcId, evt.LogicalIP); err != nil { + slog.Error("subscribers: DeleteEIP failed", + "vpc_id", evt.VpcId, "logical_ip", evt.LogicalIP, "err", err) + respond(msg, err) + return + } + slog.Info("subscribers: deleted dnat_and_snat rule", + "vpc_id", evt.VpcId, "logical_ip", evt.LogicalIP) + respond(msg, nil) +} + +func (s *Subscriber) handleAddNATGateway(msg *nats.Msg) { + var evt NATGatewayEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.add-nat-gateway event", "err", err) + return + } + if err := s.natgw.AttachNATGateway(context.Background(), policy.NATGWSpec{ + VPCID: evt.VpcId, + NATGatewayID: evt.NatGatewayId, + PublicIP: evt.PublicIp, + SubnetCIDR: evt.SubnetCidr, + }); err != nil { + slog.Error("subscribers: AddNATGateway failed", + "vpc_id", evt.VpcId, "natgw_id", evt.NatGatewayId, + "public_ip", evt.PublicIp, "subnet_cidr", evt.SubnetCidr, "err", err) + return + } + slog.Info("subscribers: NAT Gateway SNAT rule added", + "vpc_id", evt.VpcId, "natgw_id", evt.NatGatewayId, + "public_ip", evt.PublicIp, "subnet_cidr", evt.SubnetCidr) +} + +func (s *Subscriber) handleDeleteNATGateway(msg *nats.Msg) { + var evt NATGatewayEvent + if err := json.Unmarshal(msg.Data, &evt); err != nil { + slog.Error("subscribers: failed to unmarshal vpc.delete-nat-gateway event", "err", err) + return + } + if err := s.natgw.DetachNATGateway(context.Background(), evt.VpcId, evt.SubnetCidr); err != nil { + slog.Warn("subscribers: DeleteNATGateway failed", + "vpc_id", evt.VpcId, "subnet_cidr", evt.SubnetCidr, "err", err) + return + } + slog.Info("subscribers: NAT Gateway SNAT rule removed", + "vpc_id", evt.VpcId, "natgw_id", evt.NatGatewayId, "subnet_cidr", evt.SubnetCidr) +} diff --git a/spinifex/network/topology/manager.go b/spinifex/network/topology/manager.go new file mode 100644 index 000000000..7c487a1bb --- /dev/null +++ b/spinifex/network/topology/manager.go @@ -0,0 +1,63 @@ +// Package topology is L2 of the spinifex network stack: the logical network +// model. It translates VPC/subnet/ENI/SG API objects into OVN logical network +// objects via L1 (network/ovn). Higher layers (policy, external, federation) +// build on this; lower layers must not call here. +package topology + +import ( + "context" + "net" + "net/netip" +) + +// Manager owns the lifecycle of OVN logical objects for AWS-style API +// resources. Implementations must be idempotent. +type Manager interface { + EnsureVPC(ctx context.Context, vpc VPCSpec) error + DeleteVPC(ctx context.Context, vpcID string) error + + EnsureSubnet(ctx context.Context, subnet SubnetSpec) error + DeleteSubnet(ctx context.Context, subnet SubnetSpec) error + + EnsurePort(ctx context.Context, port PortSpec) error + DeletePort(ctx context.Context, port PortSpec) error + + // SetPortSecurityGroups applies the declarative set of port-group + // memberships, computing the add/remove diff against current OVN state. + SetPortSecurityGroups(ctx context.Context, portID string, sgIDs []string) error + + // SG port-group lifecycle. ACL programming lives in network/policy. + EnsureSGPortGroup(ctx context.Context, groupID string) error + DeleteSGPortGroup(ctx context.Context, groupID string) error + + // DeleteSGPortGroupByName tears down a port group by raw OVN name (e.g. + // "sg_abc"). Used by the reconciler's orphan-removal path. + DeleteSGPortGroupByName(ctx context.Context, pgName string) error +} + +// VPCSpec describes a VPC at L2. +type VPCSpec struct { + VPCID string + CIDR netip.Prefix + VNI int64 +} + +// SubnetSpec describes a subnet at L2. CIDR must be contained in the parent +// VPC's CIDR (API-layer validation). +type SubnetSpec struct { + SubnetID string + VPCID string + CIDR netip.Prefix +} + +// PortSpec describes an ENI / VM port at L2. PrivateIP+MAC bind to LSP +// Addresses/PortSecurity at create. SGIDs is initial membership; +// changes go through SetPortSecurityGroups. +type PortSpec struct { + PortID string + SubnetID string + VPCID string + PrivateIP netip.Addr + MAC net.HardwareAddr + SGIDs []string +} diff --git a/spinifex/network/topology/manager_live.go b/spinifex/network/topology/manager_live.go new file mode 100644 index 000000000..8d8c080ce --- /dev/null +++ b/spinifex/network/topology/manager_live.go @@ -0,0 +1,485 @@ +package topology + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/netip" + "strconv" + "strings" + + "github.com/mulgadc/spinifex/spinifex/network/ovn" + "github.com/mulgadc/spinifex/spinifex/network/ovn/nbdb" + "github.com/mulgadc/spinifex/spinifex/utils" +) + +// Option configures a liveManager at construction. +type Option func(*liveManager) + +// WithDNSServer overrides the DHCP dns_server option emitted with each subnet's DHCPOptions row. +func WithDNSServer(fn func() string) Option { + return func(m *liveManager) { m.dnsServer = fn } +} + +// NewLiveManager returns a topology.Manager driving OVN via the given client. +func NewLiveManager(client ovn.Client, opts ...Option) Manager { + m := &liveManager{ + ovn: client, + dnsServer: defaultDNSServer, + } + for _, opt := range opts { + opt(m) + } + return m +} + +type liveManager struct { + ovn ovn.Client + dnsServer func() string +} + +var _ Manager = (*liveManager)(nil) + +func defaultDNSServer() string { return "{8.8.8.8, 1.1.1.1}" } + +// EnsureVPC idempotently creates the OVN logical router for the VPC via the +// wait-op-protected EnsureLogicalRouter primitive (cross-node race-safe). +// Backfills ExternalIDs if the surviving row was created with empty metadata. +func (m *liveManager) EnsureVPC(ctx context.Context, spec VPCSpec) error { + if m.ovn == nil { + return fmt.Errorf("OVN client not connected") + } + if spec.VPCID == "" { + return fmt.Errorf("EnsureVPC: empty VPCID") + } + routerName := VPCRouter(spec.VPCID) + cidr := "" + if spec.CIDR.IsValid() { + cidr = spec.CIDR.String() + } + lr := &nbdb.LogicalRouter{ + Name: routerName, + ExternalIDs: map[string]string{ + "spinifex:vpc_id": spec.VPCID, + "spinifex:vni": strconv.FormatInt(spec.VNI, 10), + "spinifex:cidr": cidr, + }, + } + existing, err := m.ovn.EnsureLogicalRouter(ctx, lr) + if err != nil { + return fmt.Errorf("ensure logical router %q: %w", routerName, err) + } + if existing.UUID == lr.UUID { + slog.Info("topology: EnsureVPC created router", "router", routerName, "vpc_id", spec.VPCID, "cidr", cidr) + return nil + } + if err := m.backfillRouterMetadata(ctx, existing, lr.ExternalIDs); err != nil { + slog.Warn("topology: EnsureVPC metadata backfill failed", + "router", routerName, "vpc_id", spec.VPCID, "err", err) + } + return nil +} + +// backfillRouterMetadata fills empty ExternalIDs on the existing row from spec +// so the race-loser's richer metadata isn't lost (EnsureLogicalRouter is single-shot). +func (m *liveManager) backfillRouterMetadata(ctx context.Context, existing *nbdb.LogicalRouter, spec map[string]string) error { + if existing.ExternalIDs == nil { + existing.ExternalIDs = map[string]string{} + } + changed := false + for k, v := range spec { + if v == "" { + continue + } + if cur := existing.ExternalIDs[k]; cur == "" { + existing.ExternalIDs[k] = v + changed = true + } + } + if !changed { + return nil + } + return m.ovn.UpdateLogicalRouterExternalIDs(ctx, existing.Name, existing.ExternalIDs) +} + +// DeleteVPC removes the OVN logical router for the VPC and cascades through +// any subnets that belong to it. +func (m *liveManager) DeleteVPC(ctx context.Context, vpcID string) error { + if m.ovn == nil { + return fmt.Errorf("OVN client not connected") + } + routerName := VPCRouter(vpcID) + + switches, err := m.ovn.ListLogicalSwitches(ctx) + if err != nil { + slog.Warn("topology: DeleteVPC list switches", "err", err) + } else { + for _, ls := range switches { + if ls.ExternalIDs["spinifex:vpc_id"] != vpcID { + continue + } + if err := m.ovn.DeleteLogicalSwitch(ctx, ls.Name); err != nil { + slog.Warn("topology: DeleteVPC cascade switch", "switch", ls.Name, "err", err) + } + } + } + + dhcpOpts, err := m.ovn.ListDHCPOptions(ctx) + if err != nil { + slog.Warn("topology: DeleteVPC list DHCP options", "err", err) + } else { + for _, opts := range dhcpOpts { + if opts.ExternalIDs["spinifex:vpc_id"] != vpcID { + continue + } + if err := m.ovn.DeleteDHCPOptions(ctx, opts.UUID); err != nil { + slog.Warn("topology: DeleteVPC cascade DHCP", "uuid", opts.UUID, "err", err) + } + } + } + + if err := m.ovn.DeleteLogicalRouter(ctx, routerName); err != nil { + return fmt.Errorf("delete logical router %q: %w", routerName, err) + } + slog.Info("topology: DeleteVPC removed router", "router", routerName, "vpc_id", vpcID) + return nil +} + +// EnsureSubnet creates the OVN logical switch, subnet→VPC router port pair, +// and DHCP options for the subnet. +func (m *liveManager) EnsureSubnet(ctx context.Context, spec SubnetSpec) error { + if m.ovn == nil { + return fmt.Errorf("OVN client not connected") + } + if spec.SubnetID == "" || spec.VPCID == "" { + return fmt.Errorf("EnsureSubnet: SubnetID/VPCID required") + } + if !spec.CIDR.IsValid() { + return fmt.Errorf("EnsureSubnet: invalid CIDR for subnet %q", spec.SubnetID) + } + gwIP, prefixBits, err := SubnetGatewayCIDR(spec.CIDR) + if err != nil { + return fmt.Errorf("invalid subnet CIDR %q: %w", spec.CIDR, err) + } + cidr := spec.CIDR.String() + gwCIDR := fmt.Sprintf("%s/%d", gwIP, prefixBits) + routerMAC := utils.HashMAC(spec.SubnetID) + switchName := SubnetSwitch(spec.SubnetID) + routerName := VPCRouter(spec.VPCID) + routerPortName := SubnetRouterPort(spec.SubnetID) + switchRouterPortName := SubnetSwitchRouterPort(spec.SubnetID) + + ls := &nbdb.LogicalSwitch{ + Name: switchName, + ExternalIDs: map[string]string{ + "spinifex:subnet_id": spec.SubnetID, + "spinifex:vpc_id": spec.VPCID, + }, + } + existingSwitch, err := m.ovn.EnsureLogicalSwitch(ctx, ls) + if err != nil { + return fmt.Errorf("ensure logical switch %q: %w", switchName, err) + } + if existingSwitch.UUID != ls.UUID { + return nil + } + + lrp := &nbdb.LogicalRouterPort{ + Name: routerPortName, + MAC: routerMAC, + Networks: []string{gwCIDR}, + ExternalIDs: map[string]string{ + "spinifex:subnet_id": spec.SubnetID, + "spinifex:vpc_id": spec.VPCID, + }, + } + if err := m.ovn.CreateLogicalRouterPort(ctx, routerName, lrp); err != nil { + _ = m.ovn.DeleteLogicalSwitch(ctx, switchName) + return fmt.Errorf("create router port %q: %w", routerPortName, err) + } + + lsp := &nbdb.LogicalSwitchPort{ + Name: switchRouterPortName, + Type: "router", + Addresses: []string{"router"}, + Options: map[string]string{ + "router-port": routerPortName, + }, + ExternalIDs: map[string]string{ + "spinifex:subnet_id": spec.SubnetID, + "spinifex:vpc_id": spec.VPCID, + }, + } + if err := m.ovn.CreateLogicalSwitchPort(ctx, switchName, lsp); err != nil { + _ = m.ovn.DeleteLogicalRouterPort(ctx, routerName, routerPortName) + _ = m.ovn.DeleteLogicalSwitch(ctx, switchName) + return fmt.Errorf("create switch router port %q: %w", switchRouterPortName, err) + } + + dhcpOpts := &nbdb.DHCPOptions{ + CIDR: cidr, + Options: map[string]string{ + "server_id": gwIP, + "server_mac": routerMAC, + "lease_time": "3600", + "router": gwIP, + "dns_server": m.dnsServer(), + "mtu": "1442", + }, + ExternalIDs: map[string]string{ + "spinifex:subnet_id": spec.SubnetID, + "spinifex:vpc_id": spec.VPCID, + }, + } + if _, err := m.ovn.CreateDHCPOptions(ctx, dhcpOpts); err != nil { + slog.Warn("topology: EnsureSubnet DHCP options create failed", "cidr", cidr, "err", err) + } + + slog.Info("topology: EnsureSubnet created topology", + "switch", switchName, + "router_port", routerPortName, + "gateway", gwCIDR, + "subnet_id", spec.SubnetID, + ) + return nil +} + +// DeleteSubnet tears down the subnet's logical switch, router port, switch +// router port, and DHCP options. +func (m *liveManager) DeleteSubnet(ctx context.Context, spec SubnetSpec) error { + if m.ovn == nil { + return fmt.Errorf("OVN client not connected") + } + switchName := SubnetSwitch(spec.SubnetID) + routerName := VPCRouter(spec.VPCID) + routerPortName := SubnetRouterPort(spec.SubnetID) + switchRouterPortName := SubnetSwitchRouterPort(spec.SubnetID) + + if err := m.ovn.DeleteLogicalSwitchPort(ctx, switchName, switchRouterPortName); err != nil { + slog.Warn("topology: DeleteSubnet switch router port", "port", switchRouterPortName, "err", err) + } + if err := m.ovn.DeleteLogicalRouterPort(ctx, routerName, routerPortName); err != nil { + slog.Warn("topology: DeleteSubnet router port", "port", routerPortName, "err", err) + } + if spec.CIDR.IsValid() { + if dhcpOpts, err := m.ovn.FindDHCPOptionsByCIDR(ctx, spec.CIDR.String()); err == nil { + if err := m.ovn.DeleteDHCPOptions(ctx, dhcpOpts.UUID); err != nil { + slog.Warn("topology: DeleteSubnet DHCP options", "cidr", spec.CIDR.String(), "err", err) + } + } + } + if err := m.ovn.DeleteLogicalSwitch(ctx, switchName); err != nil { + return fmt.Errorf("delete logical switch %q: %w", switchName, err) + } + slog.Info("topology: DeleteSubnet removed topology", "switch", switchName, "subnet_id", spec.SubnetID) + return nil +} + +// EnsurePort creates the ENI's LSP and joins initial SG port groups atomically. +// If the LSP exists from a prior crashed attempt, converges SG memberships in +// place — the gap would leave a port with zero ACLs (OVN default = unrestricted). +func (m *liveManager) EnsurePort(ctx context.Context, spec PortSpec) error { + if m.ovn == nil { + return fmt.Errorf("OVN client not connected") + } + if spec.PortID == "" || spec.SubnetID == "" { + return fmt.Errorf("EnsurePort: PortID/SubnetID required") + } + portName := Port(spec.PortID) + switchName := SubnetSwitch(spec.SubnetID) + + if _, err := m.ovn.GetLogicalSwitchPort(ctx, portName); err == nil { + if _, err := m.reconcilePortSGs(ctx, portName, spec.SGIDs); err != nil { + return fmt.Errorf("reconcile SGs for existing port %q: %w", portName, err) + } + return nil + } + + addrStr := fmt.Sprintf("%s %s", spec.MAC.String(), spec.PrivateIP.String()) + lsp := &nbdb.LogicalSwitchPort{ + Name: portName, + Addresses: []string{addrStr}, + PortSecurity: []string{addrStr}, + ExternalIDs: map[string]string{ + "spinifex:eni_id": spec.PortID, + "spinifex:subnet_id": spec.SubnetID, + "spinifex:vpc_id": spec.VPCID, + }, + } + if dhcpOpts, err := m.ovn.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", spec.SubnetID); err == nil { + lsp.DHCPv4Options = &dhcpOpts.UUID + } + + pgNames := make([]string, 0, len(spec.SGIDs)) + for _, sgID := range spec.SGIDs { + pgNames = append(pgNames, SecurityGroupPortGroup(sgID)) + } + if err := m.ovn.CreateLogicalSwitchPortInGroups(ctx, switchName, lsp, pgNames); err != nil { + return fmt.Errorf("create logical switch port %q on %q: %w", portName, switchName, err) + } + slog.Info("topology: EnsurePort created LSP", + "port", portName, + "switch", switchName, + "eni_id", spec.PortID, + "ip", spec.PrivateIP.String(), + "mac", spec.MAC.String(), + "addr_str", addrStr, + "sgs", spec.SGIDs, + "port_groups", pgNames, + ) + return nil +} + +// DeletePort clears the ENI's port-group memberships and removes the LSP. +func (m *liveManager) DeletePort(ctx context.Context, spec PortSpec) error { + if m.ovn == nil { + return fmt.Errorf("OVN client not connected") + } + portName := Port(spec.PortID) + switchName := SubnetSwitch(spec.SubnetID) + + if _, err := m.reconcilePortSGs(ctx, portName, nil); err != nil { + return fmt.Errorf("clear port group memberships for %q: %w", portName, err) + } + if err := m.ovn.DeleteLogicalSwitchPort(ctx, switchName, portName); err != nil { + return fmt.Errorf("delete logical switch port %q on %q: %w", portName, switchName, err) + } + slog.Info("topology: DeletePort removed LSP", + "port", portName, + "switch", switchName, + "eni_id", spec.PortID, + ) + return nil +} + +// EnsureSGPortGroup idempotently creates the empty OVN port-group row for an SG. +// ACL programming is owned by network/policy.SecurityGroupManager. +func (m *liveManager) EnsureSGPortGroup(ctx context.Context, groupID string) error { + if m.ovn == nil { + return fmt.Errorf("OVN client not connected") + } + if groupID == "" { + return fmt.Errorf("EnsureSGPortGroup: empty groupID") + } + pgName := SecurityGroupPortGroup(groupID) + if _, err := m.ovn.EnsurePortGroup(ctx, pgName, nil); err != nil { + return fmt.Errorf("ensure port group %s: %w", pgName, err) + } + slog.Info("topology: ensured SG port group", "pg", pgName, "group_id", groupID) + return nil +} + +// DeleteSGPortGroup clears ACLs then removes the port-group row. Idempotent. +// ClearACLs must precede DeletePortGroup — libovsdb rejects dangling refs. +func (m *liveManager) DeleteSGPortGroup(ctx context.Context, groupID string) error { + if groupID == "" { + return fmt.Errorf("DeleteSGPortGroup: empty groupID") + } + return m.DeleteSGPortGroupByName(ctx, SecurityGroupPortGroup(groupID)) +} + +// DeleteSGPortGroupByName is the raw-name variant of DeleteSGPortGroup. +func (m *liveManager) DeleteSGPortGroupByName(ctx context.Context, pgName string) error { + if m.ovn == nil { + return fmt.Errorf("OVN client not connected") + } + if pgName == "" { + return fmt.Errorf("DeleteSGPortGroupByName: empty pgName") + } + if _, err := m.ovn.GetPortGroup(ctx, pgName); err != nil { + if errors.Is(err, ovn.ErrPortGroupNotFound) { + return nil + } + return fmt.Errorf("get port group %s: %w", pgName, err) + } + if err := m.ovn.ClearACLs(ctx, pgName); err != nil { + return fmt.Errorf("clear ACLs on %s: %w", pgName, err) + } + if err := m.ovn.DeletePortGroup(ctx, pgName); err != nil { + return fmt.Errorf("delete port group %s: %w", pgName, err) + } + slog.Info("topology: deleted SG port group", "pg", pgName) + return nil +} + +// SetPortSecurityGroups reconciles the port's port-group memberships against +// the declared list. +func (m *liveManager) SetPortSecurityGroups(ctx context.Context, portID string, sgIDs []string) error { + if m.ovn == nil { + return fmt.Errorf("OVN client not connected") + } + portName := Port(portID) + if _, err := m.reconcilePortSGs(ctx, portName, sgIDs); err != nil { + return fmt.Errorf("reconcile SGs for port %q: %w", portName, err) + } + return nil +} + +// reconcilePortSGs applies the SG diff atomically — an N→M modify must never +// expose an intermediate gap (OVN default = unrestricted). Returns true if changed. +func (m *liveManager) reconcilePortSGs(ctx context.Context, lspName string, desiredSGs []string) (bool, error) { + desired := make(map[string]struct{}, len(desiredSGs)) + for _, sgID := range desiredSGs { + desired[SecurityGroupPortGroup(sgID)] = struct{}{} + } + + currentNames, err := m.ovn.ListPortGroupsForPort(ctx, lspName) + if err != nil { + return false, fmt.Errorf("list current port groups for %s: %w", lspName, err) + } + current := make(map[string]struct{}, len(currentNames)) + for _, name := range currentNames { + current[name] = struct{}{} + } + + addPGs := make([]string, 0) + for name := range desired { + if _, ok := current[name]; !ok { + addPGs = append(addPGs, name) + } + } + removePGs := make([]string, 0) + for name := range current { + if _, ok := desired[name]; !ok { + removePGs = append(removePGs, name) + } + } + + if len(addPGs) == 0 && len(removePGs) == 0 { + return false, nil + } + + if err := m.ovn.UpdatePortGroupMemberships(ctx, lspName, addPGs, removePGs); err != nil { + return false, err + } + slog.Info("topology: reconciled port group memberships", + "port", lspName, + "added", addPGs, + "removed", removePGs, + "desired", desiredSGs, + ) + return true, nil +} + +// SubnetGatewayCIDR returns the .1 host IP and prefix length for an IPv4 subnet. +func SubnetGatewayCIDR(prefix netip.Prefix) (string, int, error) { + if !prefix.IsValid() { + return "", 0, fmt.Errorf("invalid prefix") + } + addr := prefix.Masked().Addr() + if !addr.Is4() { + return "", 0, fmt.Errorf("only IPv4 supported: %s", prefix) + } + bytes := addr.As4() + bytes[3]++ + return netip.AddrFrom4(bytes).String(), prefix.Bits(), nil +} + +// FormatDNSServerList renders an OVN dns_server value: "{ip1, ip2}". +func FormatDNSServerList(ips []string) string { + if len(ips) == 0 { + return defaultDNSServer() + } + return "{" + strings.Join(ips, ", ") + "}" +} diff --git a/spinifex/network/topology/manager_live_test.go b/spinifex/network/topology/manager_live_test.go new file mode 100644 index 000000000..21658019e --- /dev/null +++ b/spinifex/network/topology/manager_live_test.go @@ -0,0 +1,311 @@ +package topology + +import ( + "context" + "net" + "net/netip" + "testing" + + "github.com/mulgadc/spinifex/spinifex/network/ovn/mock" +) + +func newLiveManagerForTest(t *testing.T) (Manager, *mock.Client) { + t.Helper() + m := mock.New() + _ = m.Connect(context.Background()) + return NewLiveManager(m), m +} + +func TestLiveManager_EnsureVPC(t *testing.T) { + mgr, mockClient := newLiveManagerForTest(t) + ctx := context.Background() + spec := VPCSpec{ + VPCID: "vpc-mgr1", + CIDR: netip.MustParsePrefix("10.0.0.0/16"), + VNI: 42, + } + if err := mgr.EnsureVPC(ctx, spec); err != nil { + t.Fatalf("EnsureVPC: %v", err) + } + got, err := mockClient.GetLogicalRouter(ctx, VPCRouter(spec.VPCID)) + if err != nil { + t.Fatalf("router not present: %v", err) + } + if got.ExternalIDs["spinifex:vpc_id"] != spec.VPCID { + t.Errorf("vpc_id mismatch: %q", got.ExternalIDs["spinifex:vpc_id"]) + } + if got.ExternalIDs["spinifex:cidr"] != "10.0.0.0/16" { + t.Errorf("cidr mismatch: %q", got.ExternalIDs["spinifex:cidr"]) + } + + // Second call is a no-op (idempotent). + if err := mgr.EnsureVPC(ctx, spec); err != nil { + t.Fatalf("EnsureVPC second call: %v", err) + } +} + +func TestLiveManager_EnsureSubnet(t *testing.T) { + mgr, mockClient := newLiveManagerForTest(t) + ctx := context.Background() + vpc := VPCSpec{VPCID: "vpc-sub", CIDR: netip.MustParsePrefix("10.1.0.0/16")} + if err := mgr.EnsureVPC(ctx, vpc); err != nil { + t.Fatalf("EnsureVPC: %v", err) + } + sub := SubnetSpec{ + SubnetID: "subnet-A", + VPCID: vpc.VPCID, + CIDR: netip.MustParsePrefix("10.1.1.0/24"), + } + if err := mgr.EnsureSubnet(ctx, sub); err != nil { + t.Fatalf("EnsureSubnet: %v", err) + } + if _, err := mockClient.GetLogicalSwitch(ctx, SubnetSwitch(sub.SubnetID)); err != nil { + t.Errorf("subnet switch missing: %v", err) + } + if _, err := mockClient.GetLogicalRouterPort(ctx, SubnetRouterPort(sub.SubnetID)); err != nil { + t.Errorf("subnet router port missing: %v", err) + } + if _, err := mockClient.GetLogicalSwitchPort(ctx, SubnetSwitchRouterPort(sub.SubnetID)); err != nil { + t.Errorf("subnet switch-side router port missing: %v", err) + } + + if err := mgr.EnsureSubnet(ctx, sub); err != nil { + t.Fatalf("EnsureSubnet second call: %v", err) + } +} + +func TestLiveManager_EnsurePort(t *testing.T) { + mgr, mockClient := newLiveManagerForTest(t) + ctx := context.Background() + + vpc := VPCSpec{VPCID: "vpc-port", CIDR: netip.MustParsePrefix("10.2.0.0/16")} + sub := SubnetSpec{SubnetID: "subnet-P", VPCID: vpc.VPCID, CIDR: netip.MustParsePrefix("10.2.1.0/24")} + if err := mgr.EnsureVPC(ctx, vpc); err != nil { + t.Fatalf("EnsureVPC: %v", err) + } + if err := mgr.EnsureSubnet(ctx, sub); err != nil { + t.Fatalf("EnsureSubnet: %v", err) + } + + mac, _ := net.ParseMAC("02:00:00:00:00:01") + port := PortSpec{ + PortID: "eni-1", + SubnetID: sub.SubnetID, + VPCID: vpc.VPCID, + PrivateIP: netip.MustParseAddr("10.2.1.10"), + MAC: mac, + } + if err := mgr.EnsurePort(ctx, port); err != nil { + t.Fatalf("EnsurePort: %v", err) + } + got, err := mockClient.GetLogicalSwitchPort(ctx, Port(port.PortID)) + if err != nil { + t.Fatalf("port LSP missing: %v", err) + } + if got.ExternalIDs["spinifex:eni_id"] != port.PortID { + t.Errorf("eni_id mismatch: %q", got.ExternalIDs["spinifex:eni_id"]) + } + if len(got.Addresses) != 1 || got.Addresses[0] != "02:00:00:00:00:01 10.2.1.10" { + t.Errorf("addresses mismatch: %v", got.Addresses) + } + + if err := mgr.EnsurePort(ctx, port); err != nil { + t.Fatalf("EnsurePort second call: %v", err) + } +} + +func TestLiveManager_DeletePort(t *testing.T) { + mgr, mockClient := newLiveManagerForTest(t) + ctx := context.Background() + vpc := VPCSpec{VPCID: "vpc-dp", CIDR: netip.MustParsePrefix("10.3.0.0/16")} + sub := SubnetSpec{SubnetID: "subnet-DP", VPCID: vpc.VPCID, CIDR: netip.MustParsePrefix("10.3.1.0/24")} + _ = mgr.EnsureVPC(ctx, vpc) + _ = mgr.EnsureSubnet(ctx, sub) + mac, _ := net.ParseMAC("02:00:00:00:00:02") + port := PortSpec{ + PortID: "eni-DP", SubnetID: sub.SubnetID, VPCID: vpc.VPCID, + PrivateIP: netip.MustParseAddr("10.3.1.5"), MAC: mac, + } + if err := mgr.EnsurePort(ctx, port); err != nil { + t.Fatalf("EnsurePort: %v", err) + } + if err := mgr.DeletePort(ctx, port); err != nil { + t.Fatalf("DeletePort: %v", err) + } + if _, err := mockClient.GetLogicalSwitchPort(ctx, Port(port.PortID)); err == nil { + t.Fatal("expected LSP to be gone after DeletePort") + } +} + +func TestLiveManager_DeleteSubnetAndVPC(t *testing.T) { + mgr, mockClient := newLiveManagerForTest(t) + ctx := context.Background() + vpc := VPCSpec{VPCID: "vpc-del", CIDR: netip.MustParsePrefix("10.4.0.0/16")} + sub := SubnetSpec{SubnetID: "subnet-del", VPCID: vpc.VPCID, CIDR: netip.MustParsePrefix("10.4.1.0/24")} + _ = mgr.EnsureVPC(ctx, vpc) + _ = mgr.EnsureSubnet(ctx, sub) + + if err := mgr.DeleteSubnet(ctx, sub); err != nil { + t.Fatalf("DeleteSubnet: %v", err) + } + if _, err := mockClient.GetLogicalSwitch(ctx, SubnetSwitch(sub.SubnetID)); err == nil { + t.Fatal("expected subnet switch to be gone") + } + + if err := mgr.DeleteVPC(ctx, vpc.VPCID); err != nil { + t.Fatalf("DeleteVPC: %v", err) + } + if _, err := mockClient.GetLogicalRouter(ctx, VPCRouter(vpc.VPCID)); err == nil { + t.Fatal("expected VPC router to be gone") + } +} + +func TestLiveManager_EnsureSGPortGroup(t *testing.T) { + mgr, mockClient := newLiveManagerForTest(t) + ctx := context.Background() + + if err := mgr.EnsureSGPortGroup(ctx, "sg-pg1"); err != nil { + t.Fatalf("EnsureSGPortGroup: %v", err) + } + pg, err := mockClient.GetPortGroup(ctx, SecurityGroupPortGroup("sg-pg1")) + if err != nil { + t.Fatalf("port group not present: %v", err) + } + if pg.Name != SecurityGroupPortGroup("sg-pg1") { + t.Errorf("port group name mismatch: %q", pg.Name) + } + + // Idempotent: second call must not fail or churn state. + if err := mgr.EnsureSGPortGroup(ctx, "sg-pg1"); err != nil { + t.Fatalf("EnsureSGPortGroup second call: %v", err) + } +} + +func TestLiveManager_DeleteSGPortGroup(t *testing.T) { + mgr, mockClient := newLiveManagerForTest(t) + ctx := context.Background() + + if err := mgr.EnsureSGPortGroup(ctx, "sg-del1"); err != nil { + t.Fatalf("EnsureSGPortGroup: %v", err) + } + if err := mgr.DeleteSGPortGroup(ctx, "sg-del1"); err != nil { + t.Fatalf("DeleteSGPortGroup: %v", err) + } + if _, err := mockClient.GetPortGroup(ctx, SecurityGroupPortGroup("sg-del1")); err == nil { + t.Fatalf("port group still present after delete") + } + + // Idempotent on already-absent. + if err := mgr.DeleteSGPortGroup(ctx, "sg-del1"); err != nil { + t.Fatalf("DeleteSGPortGroup second call: %v", err) + } +} + +func TestLiveManager_DeleteSGPortGroupByName(t *testing.T) { + mgr, mockClient := newLiveManagerForTest(t) + ctx := context.Background() + + pgName := SecurityGroupPortGroup("sg-del-byname") + if err := mockClient.CreatePortGroup(ctx, pgName, nil); err != nil { + t.Fatalf("seed port group: %v", err) + } + if err := mgr.DeleteSGPortGroupByName(ctx, pgName); err != nil { + t.Fatalf("DeleteSGPortGroupByName: %v", err) + } + if _, err := mockClient.GetPortGroup(ctx, pgName); err == nil { + t.Fatalf("port group still present after delete-by-name") + } + + // Idempotent on already-absent. + if err := mgr.DeleteSGPortGroupByName(ctx, pgName); err != nil { + t.Fatalf("DeleteSGPortGroupByName second call: %v", err) + } + + // Empty pgName rejected. + if err := mgr.DeleteSGPortGroupByName(ctx, ""); err == nil { + t.Fatalf("DeleteSGPortGroupByName: empty pgName must error") + } +} + +func TestLiveManager_SetPortSecurityGroups(t *testing.T) { + mgr, mockClient := newLiveManagerForTest(t) + ctx := context.Background() + vpc := VPCSpec{VPCID: "vpc-sg", CIDR: netip.MustParsePrefix("10.5.0.0/16")} + sub := SubnetSpec{SubnetID: "subnet-sg", VPCID: vpc.VPCID, CIDR: netip.MustParsePrefix("10.5.1.0/24")} + _ = mgr.EnsureVPC(ctx, vpc) + _ = mgr.EnsureSubnet(ctx, sub) + + if err := mockClient.CreatePortGroup(ctx, SecurityGroupPortGroup("sg-A"), nil); err != nil { + t.Fatalf("seed port group: %v", err) + } + + mac, _ := net.ParseMAC("02:00:00:00:00:03") + port := PortSpec{ + PortID: "eni-sg", SubnetID: sub.SubnetID, VPCID: vpc.VPCID, + PrivateIP: netip.MustParseAddr("10.5.1.7"), MAC: mac, + SGIDs: []string{"sg-A"}, + } + if err := mgr.EnsurePort(ctx, port); err != nil { + t.Fatalf("EnsurePort: %v", err) + } + + if err := mockClient.CreatePortGroup(ctx, SecurityGroupPortGroup("sg-B"), nil); err != nil { + t.Fatalf("seed port group B: %v", err) + } + if err := mgr.SetPortSecurityGroups(ctx, port.PortID, []string{"sg-B"}); err != nil { + t.Fatalf("SetPortSecurityGroups: %v", err) + } + names, err := mockClient.ListPortGroupsForPort(ctx, Port(port.PortID)) + if err != nil { + t.Fatalf("ListPortGroupsForPort: %v", err) + } + if len(names) != 1 || names[0] != SecurityGroupPortGroup("sg-B") { + t.Errorf("expected only sg-B membership, got %v", names) + } +} + +func TestLiveManager_WithDNSServer(t *testing.T) { + mockClient := mock.New() + _ = mockClient.Connect(context.Background()) + mgr := NewLiveManager(mockClient, WithDNSServer(func() string { return "{10.0.0.2}" })) + ctx := context.Background() + _ = mgr.EnsureVPC(ctx, VPCSpec{VPCID: "vpc-dns", CIDR: netip.MustParsePrefix("10.6.0.0/16")}) + _ = mgr.EnsureSubnet(ctx, SubnetSpec{SubnetID: "subnet-dns", VPCID: "vpc-dns", CIDR: netip.MustParsePrefix("10.6.1.0/24")}) + opts, err := mockClient.FindDHCPOptionsByCIDR(ctx, "10.6.1.0/24") + if err != nil { + t.Fatalf("DHCPOptions missing: %v", err) + } + if opts.Options["dns_server"] != "{10.0.0.2}" { + t.Errorf("dns_server = %q, want {10.0.0.2}", opts.Options["dns_server"]) + } +} + +func TestSubnetGatewayCIDR(t *testing.T) { + cases := []struct { + cidr string + wantIP string + bits int + }{ + {"10.0.1.0/24", "10.0.1.1", 24}, + {"192.168.0.0/16", "192.168.0.1", 16}, + {"172.16.0.0/20", "172.16.0.1", 20}, + {"172.31.0.0/28", "172.31.0.1", 28}, + } + for _, tc := range cases { + t.Run(tc.cidr, func(t *testing.T) { + gw, bits, err := SubnetGatewayCIDR(netip.MustParsePrefix(tc.cidr)) + if err != nil { + t.Fatalf("SubnetGatewayCIDR: %v", err) + } + if gw != tc.wantIP || bits != tc.bits { + t.Errorf("got (%q, %d), want (%q, %d)", gw, bits, tc.wantIP, tc.bits) + } + }) + } +} + +func TestSubnetGatewayCIDR_IPv6Rejected(t *testing.T) { + if _, _, err := SubnetGatewayCIDR(netip.MustParsePrefix("2001:db8::/32")); err == nil { + t.Fatal("expected error for IPv6 prefix") + } +} diff --git a/spinifex/network/topology/names.go b/spinifex/network/topology/names.go new file mode 100644 index 000000000..ce9e8aa84 --- /dev/null +++ b/spinifex/network/topology/names.go @@ -0,0 +1,49 @@ +package topology + +import "strings" + +// OVN object names form a stable contract; changes require a reconciliation migration. + +// VPCRouter is the OVN logical router name for a VPC. +func VPCRouter(vpcID string) string { return "vpc-" + vpcID } + +// SubnetSwitch is the OVN logical switch name for a subnet. +func SubnetSwitch(subnetID string) string { return "subnet-" + subnetID } + +// SubnetRouterPort is the OVN LRP name on the VPC router that terminates the +// subnet's default gateway. +func SubnetRouterPort(subnetID string) string { return "rtr-" + subnetID } + +// SubnetSwitchRouterPort is the OVN LSP (type=router) on the subnet switch +// peered with SubnetRouterPort. +func SubnetSwitchRouterPort(subnetID string) string { return "rtr-port-" + subnetID } + +// Port is the OVN logical switch port name for an ENI. +func Port(portID string) string { return "port-" + portID } + +// GatewayRouterPort is the OVN LRP name for the VPC's uplink/gateway port. +func GatewayRouterPort(vpcID string) string { return "gw-" + vpcID } + +// GatewaySwitchPort is the OVN LSP (type=router) on the external switch +// peered with GatewayRouterPort. +func GatewaySwitchPort(vpcID string) string { return "gw-port-" + vpcID } + +// ExternalSwitch is the OVN logical switch name for the per-VPC external +// switch bridging the gateway LRP to the localnet. +func ExternalSwitch(vpcID string) string { return "ext-" + vpcID } + +// ExternalLocalnetPort is the OVN LSP name for the localnet bridging the +// external switch onto the host uplink. +func ExternalLocalnetPort(vpcID string) string { return "ext-port-" + vpcID } + +// SecurityGroupPortGroup is the OVN port group name for an SG. +// OVN names match [a-zA-Z_][a-zA-Z0-9_]*, so hyphens become underscores. +func SecurityGroupPortGroup(sgID string) string { + return strings.ReplaceAll(sgID, "-", "_") +} + +// TransitSwitch is the OVN-IC transit switch name used for federation. +func TransitSwitch(azID, vpcID string) string { return "ts-" + azID + "-" + vpcID } + +// TransitRouterPort is the OVN-IC transit router port name. +func TransitRouterPort(azID, vpcID string) string { return "trp-" + azID + "-" + vpcID } diff --git a/spinifex/network/topology/names_test.go b/spinifex/network/topology/names_test.go new file mode 100644 index 000000000..f6df0fedb --- /dev/null +++ b/spinifex/network/topology/names_test.go @@ -0,0 +1,31 @@ +package topology + +import "testing" + +func TestNameHelpers(t *testing.T) { + tests := []struct { + name string + got string + want string + }{ + {"VPCRouter", VPCRouter("vpc-abc"), "vpc-vpc-abc"}, + {"SubnetSwitch", SubnetSwitch("subnet-1"), "subnet-subnet-1"}, + {"SubnetRouterPort", SubnetRouterPort("subnet-1"), "rtr-subnet-1"}, + {"SubnetSwitchRouterPort", SubnetSwitchRouterPort("subnet-1"), "rtr-port-subnet-1"}, + {"Port", Port("eni-1"), "port-eni-1"}, + {"GatewayRouterPort", GatewayRouterPort("vpc-abc"), "gw-vpc-abc"}, + {"GatewaySwitchPort", GatewaySwitchPort("vpc-abc"), "gw-port-vpc-abc"}, + {"ExternalSwitch", ExternalSwitch("vpc-abc"), "ext-vpc-abc"}, + {"ExternalLocalnetPort", ExternalLocalnetPort("vpc-abc"), "ext-port-vpc-abc"}, + {"SecurityGroupPortGroup", SecurityGroupPortGroup("sg-abc-123"), "sg_abc_123"}, + {"TransitSwitch", TransitSwitch("az-1", "vpc-abc"), "ts-az-1-vpc-abc"}, + {"TransitRouterPort", TransitRouterPort("az-1", "vpc-abc"), "trp-az-1-vpc-abc"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.got != tt.want { + t.Errorf("got %q, want %q", tt.got, tt.want) + } + }) + } +} diff --git a/spinifex/service/service.go b/spinifex/service/service.go index 66fd13926..716651448 100644 --- a/spinifex/service/service.go +++ b/spinifex/service/service.go @@ -9,7 +9,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/services/spinifex" "github.com/mulgadc/spinifex/spinifex/services/spinifexui" "github.com/mulgadc/spinifex/spinifex/services/viperblockd" - "github.com/mulgadc/spinifex/spinifex/services/vpcd" + "github.com/mulgadc/spinifex/spinifex/vpcd" ) type Service interface { diff --git a/spinifex/service/service_test.go b/spinifex/service/service_test.go index fb496e2f3..3696fe3a8 100644 --- a/spinifex/service/service_test.go +++ b/spinifex/service/service_test.go @@ -8,7 +8,7 @@ import ( "github.com/mulgadc/spinifex/spinifex/services/predastore" "github.com/mulgadc/spinifex/spinifex/services/spinifexui" "github.com/mulgadc/spinifex/spinifex/services/viperblockd" - "github.com/mulgadc/spinifex/spinifex/services/vpcd" + "github.com/mulgadc/spinifex/spinifex/vpcd" "github.com/stretchr/testify/assert" ) diff --git a/spinifex/services/vpcd/acl.go b/spinifex/services/vpcd/acl.go deleted file mode 100644 index 5391688c7..000000000 --- a/spinifex/services/vpcd/acl.go +++ /dev/null @@ -1,110 +0,0 @@ -package vpcd - -import ( - "fmt" - "strings" -) - -// SGRuleForACL mirrors the SGRule from handlers for vpcd's use. -type SGRuleForACL struct { - IpProtocol string `json:"ip_protocol"` // "tcp", "udp", "icmp", "-1" (all) - FromPort int64 `json:"from_port"` - ToPort int64 `json:"to_port"` - CidrIp string `json:"cidr_ip,omitempty"` - SourceSG string `json:"source_sg,omitempty"` -} - -// portGroupName converts a security group ID to an OVN port group name. -// OVN port group names must match [a-zA-Z_][a-zA-Z0-9_]*, so hyphens -// in sg-xxx IDs are replaced with underscores. -func portGroupName(groupId string) string { - return strings.ReplaceAll(groupId, "-", "_") -} - -// addressSetName is the OVN address set holding the IPv4 addresses of every -// port that belongs to the given port group. SG-to-SG ACL match expressions -// reference it via "ip4.src == $" / "ip4.dst == $". Lifecycle is -// tied to the port group (created in handleCreateSG, deleted in handleDeleteSG); -// reconcilePortSGs maintains its membership in step with port-group joins. -func addressSetName(pgName string) string { - return pgName + "_ip4" -} - -// BuildIngressACLMatch generates an OVN ACL match expression for an ingress rule. -// Ingress rules use "outport == @{pgName}" because OVN ACLs with direction "to-lport" -// match on the destination port (outport from the pipeline's perspective). -// -// Tenant-supplied CidrIp/SourceSG values MUST be validated by the handler before -// reaching this builder (see ec2/vpc/security_group.go ipPermissionsToSGRules). This -// builder interpolates them verbatim. -func BuildIngressACLMatch(pgName string, rule SGRuleForACL) string { - parts := []string{fmt.Sprintf("outport == @%s", pgName), "ip4"} - - // Protocol filter - parts = appendProtocolMatch(parts, rule) - - // Source filter - if rule.CidrIp != "" && rule.CidrIp != "0.0.0.0/0" { - parts = append(parts, fmt.Sprintf("ip4.src == %s", rule.CidrIp)) - } - if rule.SourceSG != "" { - parts = append(parts, fmt.Sprintf("ip4.src == $%s", addressSetName(portGroupName(rule.SourceSG)))) - } - - return strings.Join(parts, " && ") -} - -// BuildEgressACLMatch generates an OVN ACL match expression for an egress rule. -// Egress rules use "inport == @{pgName}" because OVN ACLs with direction "from-lport" -// match on the source port (inport from the pipeline's perspective). -func BuildEgressACLMatch(pgName string, rule SGRuleForACL) string { - parts := []string{fmt.Sprintf("inport == @%s", pgName), "ip4"} - - // Protocol filter - parts = appendProtocolMatch(parts, rule) - - // Destination filter - if rule.CidrIp != "" && rule.CidrIp != "0.0.0.0/0" { - parts = append(parts, fmt.Sprintf("ip4.dst == %s", rule.CidrIp)) - } - if rule.SourceSG != "" { - parts = append(parts, fmt.Sprintf("ip4.dst == $%s", addressSetName(portGroupName(rule.SourceSG)))) - } - - return strings.Join(parts, " && ") -} - -// appendProtocolMatch adds protocol-specific match clauses to the parts slice. -func appendProtocolMatch(parts []string, rule SGRuleForACL) []string { - switch rule.IpProtocol { - case "tcp": - parts = appendPortMatch(parts, "tcp", rule.FromPort, rule.ToPort) - case "udp": - parts = appendPortMatch(parts, "udp", rule.FromPort, rule.ToPort) - case "icmp": - parts = append(parts, "icmp4") - case "-1", "": - // All protocols — no additional filter needed beyond "ip4" - default: - // Unknown protocol — treat as all traffic - } - return parts -} - -// appendPortMatch adds TCP/UDP port match clauses. -func appendPortMatch(parts []string, proto string, fromPort, toPort int64) []string { - if fromPort == 0 && toPort == 0 { - // All ports for this protocol - parts = append(parts, proto) - return parts - } - if fromPort == toPort { - // Single port - parts = append(parts, fmt.Sprintf("%s.dst == %d", proto, fromPort)) - } else { - // Port range - parts = append(parts, fmt.Sprintf("%s.dst >= %d", proto, fromPort)) - parts = append(parts, fmt.Sprintf("%s.dst <= %d", proto, toPort)) - } - return parts -} diff --git a/spinifex/services/vpcd/acl_test.go b/spinifex/services/vpcd/acl_test.go deleted file mode 100644 index 6348c4d4d..000000000 --- a/spinifex/services/vpcd/acl_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package vpcd - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestACL_TCPPortFromCIDR(t *testing.T) { - rule := SGRuleForACL{ - IpProtocol: "tcp", - FromPort: 22, - ToPort: 22, - CidrIp: "10.0.0.0/8", - } - match := BuildIngressACLMatch("sg_test", rule) - assert.Contains(t, match, "tcp.dst == 22") - assert.Contains(t, match, "ip4.src == 10.0.0.0/8") - assert.Contains(t, match, "outport == @sg_test") - assert.Contains(t, match, "ip4") -} - -func TestACL_AllTrafficFromSG(t *testing.T) { - rule := SGRuleForACL{ - IpProtocol: "-1", - SourceSG: "sg-abc123", - } - match := BuildIngressACLMatch("sg_test", rule) - assert.Contains(t, match, "ip4.src == $sg_abc123_ip4") - assert.Contains(t, match, "outport == @sg_test") - assert.Contains(t, match, "ip4") -} - -func TestACL_PortRange(t *testing.T) { - rule := SGRuleForACL{ - IpProtocol: "udp", - FromPort: 1024, - ToPort: 65535, - } - match := BuildIngressACLMatch("sg_test", rule) - assert.Contains(t, match, "udp.dst >= 1024") - assert.Contains(t, match, "udp.dst <= 65535") -} - -func TestACL_ICMP(t *testing.T) { - rule := SGRuleForACL{ - IpProtocol: "icmp", - CidrIp: "0.0.0.0/0", - } - match := BuildIngressACLMatch("sg_test", rule) - assert.Contains(t, match, "icmp4") - assert.NotContains(t, match, "tcp.dst") - assert.NotContains(t, match, "udp.dst") -} - -func TestACL_AllProtocols(t *testing.T) { - rule := SGRuleForACL{ - IpProtocol: "-1", - CidrIp: "10.0.0.0/16", - } - match := BuildIngressACLMatch("sg_test", rule) - assert.Contains(t, match, "ip4") - assert.Contains(t, match, "ip4.src == 10.0.0.0/16") - assert.NotContains(t, match, "tcp") - assert.NotContains(t, match, "udp") - assert.NotContains(t, match, "icmp") -} - -func TestACL_EgressAll(t *testing.T) { - rule := SGRuleForACL{ - IpProtocol: "-1", - CidrIp: "0.0.0.0/0", - } - match := BuildEgressACLMatch("sg_test", rule) - assert.Contains(t, match, "inport == @sg_test") - assert.NotContains(t, match, "outport") - assert.Contains(t, match, "ip4") -} - -func TestACL_TCPSinglePort(t *testing.T) { - rule := SGRuleForACL{ - IpProtocol: "tcp", - FromPort: 443, - ToPort: 443, - CidrIp: "10.0.0.0/8", - } - match := BuildIngressACLMatch("sg_test", rule) - assert.Contains(t, match, "tcp.dst == 443") - assert.NotContains(t, match, "tcp.dst >=") - assert.NotContains(t, match, "tcp.dst <=") -} - -func TestACL_NoSource(t *testing.T) { - rule := SGRuleForACL{ - IpProtocol: "tcp", - FromPort: 80, - ToPort: 80, - CidrIp: "0.0.0.0/0", - } - match := BuildIngressACLMatch("sg_test", rule) - assert.Contains(t, match, "tcp.dst == 80") - assert.NotContains(t, match, "ip4.src") -} diff --git a/spinifex/services/vpcd/dhcp/client.go b/spinifex/services/vpcd/dhcp/client.go deleted file mode 100644 index f0d3559ee..000000000 --- a/spinifex/services/vpcd/dhcp/client.go +++ /dev/null @@ -1,88 +0,0 @@ -// Package dhcp provides an in-process DHCPv4 client used by spinifex-vpcd -// to obtain, renew and release real server-bound leases on behalf of the -// external IPAM pool. It replaces the historical "shell out to sudo dhcpcd" -// path which had no renewal, phantom leases under -T, and a release channel -// that collided with a host-side dhcpcd daemon managing the same bridge. -// -// The real implementation (NewNClient4) opens an AF_PACKET socket per DORA -// on the target bridge via github.com/insomniacslk/dhcp/dhcpv4/nclient4. -// Tests inject Fake instead. -package dhcp - -import ( - "context" - "net" - "time" -) - -// Client is the per-handshake DHCP surface. Backed by NewNClient4 in -// production; Fake satisfies this for unit tests. -type Client interface { - // Acquire performs a full DORA on req.Bridge and returns the - // server-bound lease. Blocks until ACK or ctx cancellation. - Acquire(ctx context.Context, req AcquireRequest) (*Lease, error) - // Renew refreshes an existing lease with its recorded server-id. - // Returns a new Lease reflecting the refreshed timers. - Renew(ctx context.Context, lease *Lease) (*Lease, error) - // Release sends DHCPRELEASE to the server-id in the lease. - Release(ctx context.Context, lease *Lease) error -} - -// AcquireRequest identifies a new DHCP lease. Fields map 1:1 to DHCP -// options and the payload chaddr. -type AcquireRequest struct { - Bridge string // OVS bridge carrying the AF_PACKET socket - ClientID string // option 61 — client-identifier - Hostname string // option 12 — host-name - VendorClass string // option 60 — vendor-class-identifier - HWAddr net.HardwareAddr // chaddr in the DHCP payload -} - -// Lease is a server-bound DHCPv4 lease. Carries enough state for Renew -// and Release, plus the raw Offer/ACK packets so that the Manager can -// resume after a daemon restart. -type Lease struct { - // Request context — echoed for audit; also used to rebuild the - // underlying nclient4 client on Renew/Release. - Bridge string - ClientID string - Hostname string - VendorClass string - HWAddr net.HardwareAddr - - // Wire state extracted from the server's ACK. - IP net.IP - SubnetMask net.IPMask - Routers []net.IP - DNS []net.IP - ServerID net.IP - - AcquiredAt time.Time - LeaseDuration time.Duration - T1 time.Duration // renew timer (option 58, or lease/2) - T2 time.Duration // rebind timer (option 59, or lease*7/8) - - // Raw wire packets. Opaque to callers; nclient4 needs them to build - // RENEW / RELEASE messages that hit the same server binding. Persisted - // in KV for crash recovery. - RawOffer []byte - RawACK []byte -} - -// ExpiresAt returns the absolute deadline at which the upstream server is -// free to reassign this IP. -func (l *Lease) ExpiresAt() time.Time { - return l.AcquiredAt.Add(l.LeaseDuration) -} - -// RenewAt returns the absolute time the Manager should attempt a RENEW -// (option 58 / T1). -func (l *Lease) RenewAt() time.Time { - return l.AcquiredAt.Add(l.T1) -} - -// RebindAt returns the absolute time the Manager should attempt a REBIND -// (option 59 / T2). -func (l *Lease) RebindAt() time.Time { - return l.AcquiredAt.Add(l.T2) -} diff --git a/spinifex/services/vpcd/dhcp/client_test.go b/spinifex/services/vpcd/dhcp/client_test.go deleted file mode 100644 index 7097c7c3d..000000000 --- a/spinifex/services/vpcd/dhcp/client_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package dhcp_test - -import ( - "context" - "errors" - "net" - "testing" - "time" - - "github.com/mulgadc/spinifex/spinifex/services/vpcd/dhcp" -) - -func TestLeaseTimers(t *testing.T) { - acq := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) - l := &dhcp.Lease{ - AcquiredAt: acq, - LeaseDuration: time.Hour, - T1: 30 * time.Minute, - T2: 52*time.Minute + 30*time.Second, - } - - if got := l.RenewAt(); !got.Equal(acq.Add(30 * time.Minute)) { - t.Errorf("RenewAt: got %v, want %v", got, acq.Add(30*time.Minute)) - } - if got := l.RebindAt(); !got.Equal(acq.Add(52*time.Minute + 30*time.Second)) { - t.Errorf("RebindAt: got %v, want %v", got, acq.Add(52*time.Minute+30*time.Second)) - } - if got := l.ExpiresAt(); !got.Equal(acq.Add(time.Hour)) { - t.Errorf("ExpiresAt: got %v, want %v", got, acq.Add(time.Hour)) - } -} - -func TestFakeAcquireReturnsLeaseAndTracksCount(t *testing.T) { - f := dhcp.NewFake() - mac, _ := net.ParseMAC("02:00:00:aa:bb:cc") - - lease, err := f.Acquire(context.Background(), dhcp.AcquireRequest{ - Bridge: "br-wan", ClientID: "eni-1", HWAddr: mac, - }) - if err != nil { - t.Fatalf("Acquire: %v", err) - } - if lease == nil || lease.IP == nil { - t.Fatalf("expected lease with IP, got %+v", lease) - } - if f.AcquireCount() != 1 { - t.Errorf("acquire count = %d, want 1", f.AcquireCount()) - } - if held, ok := f.HeldLease("eni-1"); !ok || held.IP.String() != lease.IP.String() { - t.Errorf("held lease = %v/%v, want match", held, ok) - } -} - -func TestFakeAcquireHookOverridesDefault(t *testing.T) { - f := dhcp.NewFake() - f.AcquireHook = func(req dhcp.AcquireRequest) (*dhcp.Lease, error) { - return nil, errors.New("injected") - } - _, err := f.Acquire(context.Background(), dhcp.AcquireRequest{Bridge: "br-wan", ClientID: "x"}) - if err == nil || err.Error() != "injected" { - t.Fatalf("expected injected error, got %v", err) - } -} - -func TestFakeRenewRefreshesAcquiredAt(t *testing.T) { - f := dhcp.NewFake() - lease, err := f.Acquire(context.Background(), dhcp.AcquireRequest{ - Bridge: "br-wan", ClientID: "eni-2", - }) - if err != nil { - t.Fatalf("Acquire: %v", err) - } - firstAcq := lease.AcquiredAt - - time.Sleep(10 * time.Millisecond) - renewed, err := f.Renew(context.Background(), lease) - if err != nil { - t.Fatalf("Renew: %v", err) - } - if !renewed.AcquiredAt.After(firstAcq) { - t.Errorf("renewed AcquiredAt %v not after original %v", renewed.AcquiredAt, firstAcq) - } - if f.RenewCount() != 1 { - t.Errorf("renew count = %d, want 1", f.RenewCount()) - } -} - -func TestFakeReleaseClearsTrackedLease(t *testing.T) { - f := dhcp.NewFake() - lease, _ := f.Acquire(context.Background(), dhcp.AcquireRequest{ - Bridge: "br-wan", ClientID: "eni-3", - }) - if err := f.Release(context.Background(), lease); err != nil { - t.Fatalf("Release: %v", err) - } - if _, ok := f.HeldLease("eni-3"); ok { - t.Errorf("expected lease removed after Release") - } - if f.ReleaseCount() != 1 { - t.Errorf("release count = %d, want 1", f.ReleaseCount()) - } -} - -func TestFakeRenewHookSurfacesError(t *testing.T) { - f := dhcp.NewFake() - f.RenewHook = func(l *dhcp.Lease) (*dhcp.Lease, error) { - return nil, errors.New("server NAK") - } - lease, _ := f.Acquire(context.Background(), dhcp.AcquireRequest{Bridge: "br-wan", ClientID: "eni-4"}) - if _, err := f.Renew(context.Background(), lease); err == nil { - t.Fatal("expected hook error") - } -} - -func TestFakeReleaseHookSurfacesError(t *testing.T) { - f := dhcp.NewFake() - f.ReleaseHook = func(*dhcp.Lease) error { return errors.New("server unreachable") } - lease, _ := f.Acquire(context.Background(), dhcp.AcquireRequest{Bridge: "br-wan", ClientID: "eni-rel"}) - if err := f.Release(context.Background(), lease); err == nil { - t.Fatal("expected hook error") - } -} - -func TestFakeRenewNilLease(t *testing.T) { - f := dhcp.NewFake() - if _, err := f.Renew(context.Background(), nil); err == nil { - t.Fatal("expected error for nil lease") - } -} - -func TestFakeReleaseNilIsNoop(t *testing.T) { - f := dhcp.NewFake() - if err := f.Release(context.Background(), nil); err != nil { - t.Fatalf("expected nil, got %v", err) - } -} - -func TestNClient4ValidatesInputs(t *testing.T) { - c := dhcp.NewNClient4(0, 0) // defaults applied - if _, err := c.Acquire(context.Background(), dhcp.AcquireRequest{}); err == nil { - t.Fatal("expected bridge-required error") - } - mac, _ := net.ParseMAC("02:00:00:aa:bb:cc") - if _, err := c.Acquire(context.Background(), dhcp.AcquireRequest{Bridge: "br-wan"}); err == nil { - t.Fatal("expected hw_addr-required error") - } - _ = mac -} - -func TestNClient4RenewRejectsNilAndMissingRaw(t *testing.T) { - c := dhcp.NewNClient4(0, 0) - if _, err := c.Renew(context.Background(), nil); err == nil { - t.Fatal("expected nil-lease error") - } - lease := &dhcp.Lease{Bridge: "br-wan", ClientID: "x"} // no Raw bytes - if _, err := c.Renew(context.Background(), lease); err == nil { - t.Fatal("expected missing-raw-bytes error") - } -} - -func TestNClient4ReleaseNilIsNoop(t *testing.T) { - c := dhcp.NewNClient4(0, 0) - if err := c.Release(context.Background(), nil); err != nil { - t.Fatalf("expected nil, got %v", err) - } -} - -func TestNClient4ReleaseRejectsMissingRaw(t *testing.T) { - c := dhcp.NewNClient4(0, 0) - lease := &dhcp.Lease{Bridge: "br-wan", ClientID: "x"} - if err := c.Release(context.Background(), lease); err == nil { - t.Fatal("expected missing-raw-bytes error") - } -} diff --git a/spinifex/services/vpcd/dhcp/fake.go b/spinifex/services/vpcd/dhcp/fake.go deleted file mode 100644 index c75304c4b..000000000 --- a/spinifex/services/vpcd/dhcp/fake.go +++ /dev/null @@ -1,179 +0,0 @@ -package dhcp - -import ( - "context" - "fmt" - "net" - "sync" - "time" -) - -// Fake is a configurable in-memory Client for unit tests. It mimics the -// DORA / RENEW / RELEASE lifecycle well enough to exercise the Manager -// without opening real sockets. Hooks let tests inject errors or custom -// responses at each step. -type Fake struct { - mu sync.Mutex - - // Default template for synthesised leases. Tests can override via - // SetDefaultLease. - defaultLease LeaseTemplate - - // Optional hooks — if non-nil the fake calls the hook instead of - // synthesising a response. Useful for fault injection. - AcquireHook func(AcquireRequest) (*Lease, error) - RenewHook func(*Lease) (*Lease, error) - ReleaseHook func(*Lease) error - - // Tracked state. - leases map[string]*Lease // keyed by ClientID - acquireCount int - renewCount int - releaseCount int -} - -var _ Client = (*Fake)(nil) - -// LeaseTemplate configures the synthetic lease Fake returns from Acquire -// when no AcquireHook is set. -type LeaseTemplate struct { - IP net.IP - SubnetMask net.IPMask - Routers []net.IP - DNS []net.IP - ServerID net.IP - LeaseDuration time.Duration -} - -// DefaultLeaseTemplate returns a reasonable template for tests (RFC 5737 -// TEST-NET-1, 1 h lease). -func DefaultLeaseTemplate() LeaseTemplate { - return LeaseTemplate{ - IP: net.IPv4(192, 0, 2, 100), - SubnetMask: net.CIDRMask(24, 32), - Routers: []net.IP{net.IPv4(192, 0, 2, 1)}, - DNS: []net.IP{net.IPv4(192, 0, 2, 1)}, - ServerID: net.IPv4(192, 0, 2, 1), - LeaseDuration: time.Hour, - } -} - -// NewFake creates a Fake pre-loaded with DefaultLeaseTemplate. -func NewFake() *Fake { - return &Fake{ - defaultLease: DefaultLeaseTemplate(), - leases: map[string]*Lease{}, - } -} - -// SetDefaultLease overrides the template used for subsequent Acquire calls. -func (f *Fake) SetDefaultLease(tmpl LeaseTemplate) { - f.mu.Lock() - defer f.mu.Unlock() - f.defaultLease = tmpl -} - -// AcquireCount returns the number of Acquire calls observed. -func (f *Fake) AcquireCount() int { f.mu.Lock(); defer f.mu.Unlock(); return f.acquireCount } - -// RenewCount returns the number of Renew calls observed. -func (f *Fake) RenewCount() int { f.mu.Lock(); defer f.mu.Unlock(); return f.renewCount } - -// ReleaseCount returns the number of Release calls observed. -func (f *Fake) ReleaseCount() int { f.mu.Lock(); defer f.mu.Unlock(); return f.releaseCount } - -// HeldLease returns the last lease Fake synthesised for the given ClientID. -func (f *Fake) HeldLease(clientID string) (*Lease, bool) { - f.mu.Lock() - defer f.mu.Unlock() - l, ok := f.leases[clientID] - return l, ok -} - -func (f *Fake) Acquire(_ context.Context, req AcquireRequest) (*Lease, error) { - f.mu.Lock() - f.acquireCount++ - hook := f.AcquireHook - tmpl := f.defaultLease - f.mu.Unlock() - - if hook != nil { - lease, err := hook(req) - if err == nil && lease != nil { - f.mu.Lock() - f.leases[req.ClientID] = lease - f.mu.Unlock() - } - return lease, err - } - - lease := &Lease{ - Bridge: req.Bridge, - ClientID: req.ClientID, - Hostname: req.Hostname, - VendorClass: req.VendorClass, - HWAddr: append(net.HardwareAddr(nil), req.HWAddr...), - IP: tmpl.IP, - SubnetMask: tmpl.SubnetMask, - Routers: tmpl.Routers, - DNS: tmpl.DNS, - ServerID: tmpl.ServerID, - AcquiredAt: time.Now(), - LeaseDuration: tmpl.LeaseDuration, - T1: tmpl.LeaseDuration / 2, - T2: tmpl.LeaseDuration * 7 / 8, - // Synthetic raw packets — opaque marker bytes so Renew/Release - // distinguish a fake lease from a nil-bytes one. - RawOffer: []byte("fake-offer-" + req.ClientID), - RawACK: []byte("fake-ack-" + req.ClientID), - } - - f.mu.Lock() - f.leases[req.ClientID] = lease - f.mu.Unlock() - return lease, nil -} - -func (f *Fake) Renew(_ context.Context, lease *Lease) (*Lease, error) { - if lease == nil { - return nil, fmt.Errorf("fake renew: lease is nil") - } - f.mu.Lock() - f.renewCount++ - hook := f.RenewHook - f.mu.Unlock() - - if hook != nil { - renewed, err := hook(lease) - if err == nil && renewed != nil { - f.mu.Lock() - f.leases[lease.ClientID] = renewed - f.mu.Unlock() - } - return renewed, err - } - - renewed := *lease - renewed.AcquiredAt = time.Now() - f.mu.Lock() - f.leases[lease.ClientID] = &renewed - f.mu.Unlock() - return &renewed, nil -} - -func (f *Fake) Release(_ context.Context, lease *Lease) error { - if lease == nil { - return nil - } - f.mu.Lock() - f.releaseCount++ - hook := f.ReleaseHook - f.mu.Unlock() - if hook != nil { - return hook(lease) - } - f.mu.Lock() - delete(f.leases, lease.ClientID) - f.mu.Unlock() - return nil -} diff --git a/spinifex/services/vpcd/dhcp/nats.go b/spinifex/services/vpcd/dhcp/nats.go deleted file mode 100644 index 758a1d39e..000000000 --- a/spinifex/services/vpcd/dhcp/nats.go +++ /dev/null @@ -1,73 +0,0 @@ -package dhcp - -// NATS subjects that spinifex-vpcd's DHCP manager subscribes to and -// publishes on. The daemon-side handlers in handlers/ec2/vpc send -// requests against the request/reply subjects. -const ( - // TopicAcquire is the request/reply subject for obtaining a lease. - // Request: AcquireRequestMsg. Reply: AcquireReplyMsg. - TopicAcquire = "vpc.dhcp.acquire" - // TopicRelease is the request/reply subject for releasing a lease. - // Request: ReleaseRequestMsg. Reply: ReleaseReplyMsg. - TopicRelease = "vpc.dhcp.release" - // TopicLeaseExpired is a fan-out event published by vpcd when a - // renewal permanently fails and the lease has passed its expiry. - // Consumers (EIP / instance / VPC reconcilers) decide whether to - // re-request a lease, reschedule, or fail the pending API call. - TopicLeaseExpired = "vpc.dhcp.lease-expired" -) - -// KV bucket for persisted lease state. Keyed by ClientID. Separate from -// the external IPAM allocation ledger by design: IPAM records who owns -// which IP, this bucket records when a lease expires and how to renew it. -const ( - KVBucketDHCPLeases = "spinifex-dhcp-leases" - KVBucketDHCPLeasesVersion = 1 -) - -// AcquireRequestMsg is the NATS payload for TopicAcquire. -type AcquireRequestMsg struct { - Bridge string `json:"bridge"` - ClientID string `json:"client_id"` - Hostname string `json:"hostname"` - VendorClass string `json:"vendor_class"` - // HWAddr is the MAC to put in the DHCP payload chaddr, formatted as - // 02:aa:bb:cc:dd:ee. Empty means the vpcd Manager derives a synthetic - // MAC from ClientID. - HWAddr string `json:"hw_addr,omitempty"` - PoolName string `json:"pool_name,omitempty"` -} - -// AcquireReplyMsg is the NATS reply for TopicAcquire. Error is the only -// field set on failure. -type AcquireReplyMsg struct { - IP string `json:"ip,omitempty"` - SubnetMask string `json:"subnet_mask,omitempty"` - Routers []string `json:"routers,omitempty"` - DNS []string `json:"dns,omitempty"` - ServerID string `json:"server_id,omitempty"` - HWAddr string `json:"hw_addr,omitempty"` - ExpiresUnix int64 `json:"expires_unix,omitempty"` - Error string `json:"error,omitempty"` -} - -// ReleaseRequestMsg is the NATS payload for TopicRelease. ClientID must -// match the identifier the lease was acquired with. -type ReleaseRequestMsg struct { - ClientID string `json:"client_id"` -} - -// ReleaseReplyMsg is the NATS reply for TopicRelease. -type ReleaseReplyMsg struct { - Error string `json:"error,omitempty"` -} - -// LeaseExpiredEvent is the payload fan-out on TopicLeaseExpired. -type LeaseExpiredEvent struct { - ClientID string `json:"client_id"` - PoolName string `json:"pool_name,omitempty"` - IP string `json:"ip,omitempty"` - // Reason is a short machine-readable code (e.g. "renew_failed", - // "nak", "server_unreachable"). - Reason string `json:"reason"` -} diff --git a/spinifex/services/vpcd/dhcp/nats_client.go b/spinifex/services/vpcd/dhcp/nats_client.go deleted file mode 100644 index 5d24d5652..000000000 --- a/spinifex/services/vpcd/dhcp/nats_client.go +++ /dev/null @@ -1,151 +0,0 @@ -package dhcp - -import ( - "encoding/json" - "fmt" - "log/slog" - "time" - - "github.com/nats-io/nats.go" -) - -// NATSTimeout bounds the request/reply round-trip to vpcd's DHCPManager. -// Picked to comfortably exceed the DORA handshake (15 s default + server -// jitter + ~10 s of slack for NAK + fresh DISCOVER fallbacks). A var rather -// than a const so tests can drive the timeout error path without waiting -// 20 s per table entry. -var NATSTimeout = 30 * time.Second - -// AcquireMaxAttempts caps retry of vpc.dhcp.acquire. 60 attempts × 1 s sleep -// covers a cold-boot vpcd start window where NATS is up but the dhcp.Manager -// hasn't yet subscribed (typical: 5–30 s, pathological: 60 s during full -// reconcile). Tests override to keep timeout-path coverage fast. -var AcquireMaxAttempts = 60 - -// AcquireRetryDelay is the wait between retries. Var for the same reason as -// AcquireMaxAttempts. -var AcquireRetryDelay = 1 * time.Second - -// LeaseResult is the data callers need from a successful acquire. Mirrors -// AcquireReplyMsg but exposed as a typed result so callers don't depend on -// the wire struct. -type LeaseResult struct { - IP string - SubnetMask string - Routers []string - DNS []string - ServerID string - HWAddr string - ExpiresUnix int64 -} - -// RequestAcquire asks vpcd's DHCPManager (over NATS) to acquire a DHCP lease -// on the given bridge, identifying the caller with option 61 (client-id), -// option 12 (hostname) and option 60 (vendor class). Blocks until vpcd -// replies or NATSTimeout expires. The Manager-side handler is idempotent on -// clientID: a second call with the same clientID while a live lease exists -// returns the same lease without a fresh DORA, so CAS retry loops on the -// caller are safe. -func RequestAcquire(nc *nats.Conn, bridge, clientID, hostname, vendorClass, poolName, hwAddr string) (LeaseResult, error) { - if nc == nil { - return LeaseResult{}, fmt.Errorf("DHCP lease: NATS connection is required") - } - if bridge == "" { - return LeaseResult{}, fmt.Errorf("DHCP lease: bridge name is required") - } - if clientID == "" { - return LeaseResult{}, fmt.Errorf("DHCP lease: client ID is required") - } - - req := AcquireRequestMsg{ - Bridge: bridge, - ClientID: clientID, - Hostname: hostname, - VendorClass: vendorClass, - PoolName: poolName, - HWAddr: hwAddr, - } - data, err := json.Marshal(req) - if err != nil { - return LeaseResult{}, fmt.Errorf("marshal dhcp acquire request: %w", err) - } - - var reply AcquireReplyMsg - maxAttempts := AcquireMaxAttempts - - for attempt := 1; attempt <= maxAttempts; attempt++ { - msg, err := nc.Request(TopicAcquire, data, NATSTimeout) - if err != nil { - if attempt == maxAttempts { - return LeaseResult{}, fmt.Errorf( - "dhcp acquire NATS request failed after %d attempts (client %s): %w", - maxAttempts, clientID, err, - ) - } - slog.Warn("dhcp acquire NATS request failed, retrying", - "clientID", clientID, - "attempt", attempt, - "maxAttempts", maxAttempts, - "err", err, - ) - time.Sleep(AcquireRetryDelay) - continue - } - - if err := json.Unmarshal(msg.Data, &reply); err != nil { - return LeaseResult{}, fmt.Errorf("unmarshal dhcp acquire reply: %w", err) - } - if reply.Error != "" { - return LeaseResult{}, fmt.Errorf("dhcp acquire client %s: %s", clientID, reply.Error) - } - break - } - - slog.Info("DHCP lease obtained", - "bridge", bridge, - "client_id", clientID, - "ip", reply.IP, - "server_id", reply.ServerID, - "expires_unix", reply.ExpiresUnix, - ) - - return LeaseResult{ - IP: reply.IP, - SubnetMask: reply.SubnetMask, - Routers: reply.Routers, - DNS: reply.DNS, - ServerID: reply.ServerID, - HWAddr: reply.HWAddr, - ExpiresUnix: reply.ExpiresUnix, - }, nil -} - -// RequestRelease asks vpcd to release the lease identified by clientID. -// Returns nil silently when nc or clientID is empty so callers in clean-up -// paths can invoke it unconditionally. -func RequestRelease(nc *nats.Conn, clientID string) error { - if nc == nil || clientID == "" { - return nil - } - - data, err := json.Marshal(ReleaseRequestMsg{ClientID: clientID}) - if err != nil { - return fmt.Errorf("marshal dhcp release request: %w", err) - } - - msg, err := nc.Request(TopicRelease, data, NATSTimeout) - if err != nil { - return fmt.Errorf("dhcp release NATS request (client %s): %w", clientID, err) - } - - var reply ReleaseReplyMsg - if err := json.Unmarshal(msg.Data, &reply); err != nil { - return fmt.Errorf("unmarshal dhcp release reply: %w", err) - } - if reply.Error != "" { - return fmt.Errorf("dhcp release (client %s): %s", clientID, reply.Error) - } - - slog.Info("DHCP lease released", "client_id", clientID) - return nil -} diff --git a/spinifex/services/vpcd/dhcp/nats_client_test.go b/spinifex/services/vpcd/dhcp/nats_client_test.go deleted file mode 100644 index 7c7c1ff49..000000000 --- a/spinifex/services/vpcd/dhcp/nats_client_test.go +++ /dev/null @@ -1,104 +0,0 @@ -package dhcp - -import ( - "encoding/json" - "testing" - "time" - - "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/nats-io/nats.go" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestRequestAcquire_RequestTimeout(t *testing.T) { - _, nc := testutil.StartTestNATS(t) - sub, err := nc.Subscribe(TopicAcquire, func(*nats.Msg) {}) - require.NoError(t, err) - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - prev := NATSTimeout - NATSTimeout = 150 * time.Millisecond - t.Cleanup(func() { NATSTimeout = prev }) - - prevMax := AcquireMaxAttempts - prevDelay := AcquireRetryDelay - AcquireMaxAttempts = 2 - AcquireRetryDelay = 10 * time.Millisecond - t.Cleanup(func() { - AcquireMaxAttempts = prevMax - AcquireRetryDelay = prevDelay - }) - - _, err = RequestAcquire(nc, "br-wan", "eni-timeout", "eni-timeout", "mulga-spinifex", "wan", "") - assert.ErrorContains(t, err, "dhcp acquire NATS request") -} - -func TestRequestRelease_RequestTimeout(t *testing.T) { - _, nc := testutil.StartTestNATS(t) - sub, err := nc.Subscribe(TopicRelease, func(*nats.Msg) {}) - require.NoError(t, err) - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - prev := NATSTimeout - NATSTimeout = 150 * time.Millisecond - t.Cleanup(func() { NATSTimeout = prev }) - - err = RequestRelease(nc, "eni-timeout") - assert.ErrorContains(t, err, "dhcp release NATS request") -} - -func TestRequestAcquire_MalformedReply(t *testing.T) { - _, nc := testutil.StartTestNATS(t) - sub, err := nc.Subscribe(TopicAcquire, func(msg *nats.Msg) { - _ = msg.Respond([]byte("not json")) - }) - require.NoError(t, err) - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - _, err = RequestAcquire(nc, "br-wan", "eni-malformed", "eni-malformed", "mulga-spinifex", "wan", "") - assert.ErrorContains(t, err, "unmarshal dhcp acquire reply") -} - -func TestRequestRelease_MalformedReply(t *testing.T) { - _, nc := testutil.StartTestNATS(t) - sub, err := nc.Subscribe(TopicRelease, func(msg *nats.Msg) { - _ = msg.Respond([]byte("not json")) - }) - require.NoError(t, err) - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - err = RequestRelease(nc, "eni-malformed") - assert.ErrorContains(t, err, "unmarshal dhcp release reply") -} - -func TestRequestRelease_ReplyError(t *testing.T) { - _, nc := testutil.StartTestNATS(t) - sub, err := nc.Subscribe(TopicRelease, func(msg *nats.Msg) { - data, _ := json.Marshal(ReleaseReplyMsg{Error: "vpcd rejected release"}) - _ = msg.Respond(data) - }) - require.NoError(t, err) - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - err = RequestRelease(nc, "eni-rejected") - assert.ErrorContains(t, err, "vpcd rejected release") -} - -func TestRequestAcquire_Guards(t *testing.T) { - _, err := RequestAcquire(nil, "br-wan", "eni-1", "eni-1", "mulga-spinifex", "wan", "") - assert.ErrorContains(t, err, "NATS connection is required") - - _, nc := testutil.StartTestNATS(t) - _, err = RequestAcquire(nc, "", "eni-1", "eni-1", "mulga-spinifex", "wan", "") - assert.ErrorContains(t, err, "bridge name is required") - - _, err = RequestAcquire(nc, "br-wan", "", "eni-1", "mulga-spinifex", "wan", "") - assert.ErrorContains(t, err, "client ID is required") -} - -func TestRequestRelease_NoopWhenNilOrEmpty(t *testing.T) { - assert.NoError(t, RequestRelease(nil, "eni-1")) - _, nc := testutil.StartTestNATS(t) - assert.NoError(t, RequestRelease(nc, "")) -} diff --git a/spinifex/services/vpcd/dhcp/nclient4.go b/spinifex/services/vpcd/dhcp/nclient4.go deleted file mode 100644 index 3f5e9ff5c..000000000 --- a/spinifex/services/vpcd/dhcp/nclient4.go +++ /dev/null @@ -1,188 +0,0 @@ -package dhcp - -import ( - "context" - "fmt" - "time" - - "github.com/insomniacslk/dhcp/dhcpv4" - "github.com/insomniacslk/dhcp/dhcpv4/nclient4" -) - -// NClient4Client is the production DHCP client backed by -// github.com/insomniacslk/dhcp/dhcpv4/nclient4. Each Acquire/Renew/Release -// opens an AF_PACKET socket on the target bridge for the duration of the -// handshake and closes it when done — no long-lived per-lease process. -// -// Retransmission lives one layer up in DHCPManager.acquireWithBackoff -// (RFC 2131 §4.1). This client always does exactly one DISCOVER attempt; -// the per-attempt window is whatever the caller's context allows. -type NClient4Client struct { - timeout time.Duration -} - -var _ Client = (*NClient4Client)(nil) - -// NewNClient4 creates an NClient4Client. timeout is the nclient4-internal -// socket read deadline used as a safety net when the caller supplies no -// context deadline; in production DHCPManager always wraps each call in a -// per-attempt ctx so this is rarely hit. The retry argument is accepted -// for backwards compat and ignored — DHCPManager.acquireWithBackoff owns -// retransmission. -func NewNClient4(timeout time.Duration, _ int) *NClient4Client { - if timeout <= 0 { - timeout = 60 * time.Second - } - return &NClient4Client{timeout: timeout} -} - -func (c *NClient4Client) Acquire(ctx context.Context, req AcquireRequest) (*Lease, error) { - if req.Bridge == "" { - return nil, fmt.Errorf("dhcp acquire: bridge is required") - } - if len(req.HWAddr) == 0 { - return nil, fmt.Errorf("dhcp acquire: hw_addr is required") - } - - client, err := nclient4.New(req.Bridge, - nclient4.WithHWAddr(req.HWAddr), - nclient4.WithTimeout(c.timeout), - // Single attempt per call — DHCPManager.acquireWithBackoff drives - // retransmission per RFC 2131 §4.1. - nclient4.WithRetry(1), - ) - if err != nil { - return nil, fmt.Errorf("open nclient4 on %s: %w", req.Bridge, err) - } - defer func() { _ = client.Close() }() - - // Without the broadcast flag, the server sends a unicast OFFER to the - // generated chaddr MAC. The physical NIC drops it in hardware (not its MAC), - // so the AF_PACKET socket on the bridge never sees the frame. Setting the - // broadcast flag forces the server to respond with ff:ff:ff:ff:ff:ff, which - // all NICs accept unconditionally. - mods := append(identityModifiers(req.ClientID, req.Hostname, req.VendorClass), dhcpv4.WithBroadcast(true)) - lease, err := client.Request(ctx, mods...) - if err != nil { - return nil, fmt.Errorf("dhcp DORA on %s (client=%s): %w", req.Bridge, req.ClientID, err) - } - return leaseFromNClient4(req, lease), nil -} - -func (c *NClient4Client) Renew(ctx context.Context, lease *Lease) (*Lease, error) { - if lease == nil { - return nil, fmt.Errorf("dhcp renew: lease is nil") - } - nclient4Lease, err := reconstructNClient4Lease(lease) - if err != nil { - return nil, fmt.Errorf("dhcp renew: %w", err) - } - - client, err := nclient4.New(lease.Bridge, - nclient4.WithHWAddr(lease.HWAddr), - nclient4.WithTimeout(c.timeout), - nclient4.WithRetry(1), - ) - if err != nil { - return nil, fmt.Errorf("open nclient4 on %s for renew: %w", lease.Bridge, err) - } - defer func() { _ = client.Close() }() - - renewed, err := client.Renew(ctx, nclient4Lease, - identityModifiers(lease.ClientID, lease.Hostname, lease.VendorClass)...) - if err != nil { - return nil, fmt.Errorf("dhcp renew on %s (client=%s): %w", lease.Bridge, lease.ClientID, err) - } - - return leaseFromNClient4(AcquireRequest{ - Bridge: lease.Bridge, - ClientID: lease.ClientID, - Hostname: lease.Hostname, - VendorClass: lease.VendorClass, - HWAddr: lease.HWAddr, - }, renewed), nil -} - -func (c *NClient4Client) Release(ctx context.Context, lease *Lease) error { - if lease == nil { - return nil - } - nclient4Lease, err := reconstructNClient4Lease(lease) - if err != nil { - return fmt.Errorf("dhcp release: %w", err) - } - - client, err := nclient4.New(lease.Bridge, - nclient4.WithHWAddr(lease.HWAddr), - nclient4.WithTimeout(c.timeout), - ) - if err != nil { - return fmt.Errorf("open nclient4 on %s for release: %w", lease.Bridge, err) - } - defer func() { _ = client.Close() }() - - if err := client.Release(nclient4Lease, - dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(lease.ClientID)))); err != nil { - return fmt.Errorf("dhcp release on %s (client=%s): %w", lease.Bridge, lease.ClientID, err) - } - return nil -} - -// identityModifiers builds the three identifying DHCP options we set on -// every outbound message: option 61 (client-id), option 12 (hostname), -// option 60 (vendor class). -func identityModifiers(clientID, hostname, vendorClass string) []dhcpv4.Modifier { - var mods []dhcpv4.Modifier - if clientID != "" { - mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClientIdentifier([]byte(clientID)))) - } - if hostname != "" { - mods = append(mods, dhcpv4.WithOption(dhcpv4.OptHostName(hostname))) - } - if vendorClass != "" { - mods = append(mods, dhcpv4.WithOption(dhcpv4.OptClassIdentifier(vendorClass))) - } - return mods -} - -func leaseFromNClient4(req AcquireRequest, in *nclient4.Lease) *Lease { - ack := in.ACK - leaseTime := ack.IPAddressLeaseTime(24 * time.Hour) - return &Lease{ - Bridge: req.Bridge, - ClientID: req.ClientID, - Hostname: req.Hostname, - VendorClass: req.VendorClass, - HWAddr: req.HWAddr, - IP: ack.YourIPAddr, - SubnetMask: ack.SubnetMask(), - Routers: ack.Router(), - DNS: ack.DNS(), - ServerID: ack.ServerIdentifier(), - AcquiredAt: in.CreationTime, - LeaseDuration: leaseTime, - T1: ack.IPAddressRenewalTime(leaseTime / 2), - T2: ack.IPAddressRebindingTime(leaseTime * 7 / 8), - RawOffer: in.Offer.ToBytes(), - RawACK: ack.ToBytes(), - } -} - -func reconstructNClient4Lease(l *Lease) (*nclient4.Lease, error) { - if len(l.RawOffer) == 0 || len(l.RawACK) == 0 { - return nil, fmt.Errorf("lease is missing raw offer/ack bytes; renewal/release not possible") - } - offer, err := dhcpv4.FromBytes(l.RawOffer) - if err != nil { - return nil, fmt.Errorf("parse stored offer: %w", err) - } - ack, err := dhcpv4.FromBytes(l.RawACK) - if err != nil { - return nil, fmt.Errorf("parse stored ack: %w", err) - } - return &nclient4.Lease{ - Offer: offer, - ACK: ack, - CreationTime: l.AcquiredAt, - }, nil -} diff --git a/spinifex/services/vpcd/dhcp_manager.go b/spinifex/services/vpcd/dhcp_manager.go deleted file mode 100644 index e41121106..000000000 --- a/spinifex/services/vpcd/dhcp_manager.go +++ /dev/null @@ -1,865 +0,0 @@ -package vpcd - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log/slog" - "math/rand/v2" - "net" - "strings" - "sync" - "time" - - "github.com/mulgadc/spinifex/spinifex/migrate" - "github.com/mulgadc/spinifex/spinifex/services/vpcd/dhcp" - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" -) - -// DHCPManager owns the active DHCP leases held by spinifex-vpcd. It -// services vpc.dhcp.acquire / vpc.dhcp.release NATS requests from the -// daemon-side ExternalIPAM handlers, runs a per-lease renewal goroutine, -// and persists each lease in the spinifex-dhcp-leases KV bucket so that -// a vpcd restart does not forget what has already been allocated upstream. -type DHCPManager struct { - client dhcp.Client - kv nats.KeyValue - nc *nats.Conn - - // macForClientID produces the chaddr for a request that came in with - // HWAddr unset. Defaults to generateMAC; tests override. - macForClientID func(clientID string) net.HardwareAddr - - // acquireTimeout is the wallclock budget for a full Acquire — sum of - // every retransmit attempt in acquireBackoff plus slack. Renew/Release - // reuse this as their single-attempt deadline. - acquireTimeout time.Duration - - // acquireBackoff drives the per-DISCOVER timeouts used by - // acquireWithBackoff (RFC 2131 §4.1 retransmission). Each entry is - // the wait-for-OFFER window for one attempt; the next entry doubles. - // Default: 4s, 8s, 16s, 32s. ±1s jitter applied per attempt. - acquireBackoff []time.Duration - - // jitterFraction is the ± range applied to renewal sleeps, expressed - // as a fraction of T1 (0.1 = ±10%). - jitterFraction float64 - rng *rand.Rand - - // promiscOn/Off are called to toggle IFF_PROMISC on the DHCP bridge - // around each Acquire. nil = no-op (non-veth modes, or tests). - // promiscCount tracks in-flight acquires so the bridge is not cleared - // while a concurrent DORA is still waiting for a reply. - promiscOn func() - promiscOff func() - promiscMu sync.Mutex - promiscCount int - - mu sync.Mutex - leases map[string]*managedLease // keyed by ClientID - - // inFlightMu guards inFlight. Tracks concurrent Acquire calls per - // bridge so handleAcquire can surface contention as a diagnostic - // signal (mulga-siv-32). - inFlightMu sync.Mutex - inFlight map[string]int - - stopCtx context.Context - stopCancel context.CancelFunc - wg sync.WaitGroup -} - -type managedLease struct { - lease *dhcp.Lease - renewStop context.CancelFunc - renewDone chan struct{} -} - -// DHCPManagerOption configures DHCPManager at construction. -type DHCPManagerOption func(*DHCPManager) - -// WithDHCPMACFunc overrides the default MAC derivation (generateMAC). -func WithDHCPMACFunc(fn func(clientID string) net.HardwareAddr) DHCPManagerOption { - return func(m *DHCPManager) { m.macForClientID = fn } -} - -// WithDHCPAcquireTimeout overrides the wallclock budget for a full Acquire -// (sum of every retransmit attempt + slack). Renew/Release reuse it. -func WithDHCPAcquireTimeout(d time.Duration) DHCPManagerOption { - return func(m *DHCPManager) { m.acquireTimeout = d } -} - -// WithDHCPAcquireBackoff overrides the per-DISCOVER timeout schedule used -// by acquireWithBackoff. Tests use a tight schedule; production keeps the -// RFC 2131 §4.1 default (4s, 8s, 16s, 32s with ±1s jitter). -func WithDHCPAcquireBackoff(b []time.Duration) DHCPManagerOption { - return func(m *DHCPManager) { m.acquireBackoff = b } -} - -// WithDHCPJitterFraction overrides the ±fraction applied to renewal sleeps. -func WithDHCPJitterFraction(f float64) DHCPManagerOption { - return func(m *DHCPManager) { m.jitterFraction = f } -} - -// WithDHCPRand overrides the random source used for jitter (tests only). -func WithDHCPRand(r *rand.Rand) DHCPManagerOption { - return func(m *DHCPManager) { m.rng = r } -} - -// WithDHCPPromiscFuncs wires IFF_PROMISC toggling around each Acquire. on is -// called before the DORA and off is called after (even on error). Both are -// reference-counted so concurrent acquires keep the bridge promisc until the -// last one completes. Pass nil to disable (default). -func WithDHCPPromiscFuncs(on, off func()) DHCPManagerOption { - return func(m *DHCPManager) { - m.promiscOn = on - m.promiscOff = off - } -} - -// NewDHCPManager creates a Manager wired against js for KV persistence and -// nc for NATS request/reply. The caller still needs to call Bootstrap (to -// rehydrate persisted leases) and Subscribe (to start servicing requests). -func NewDHCPManager(nc *nats.Conn, js nats.JetStreamContext, client dhcp.Client, opts ...DHCPManagerOption) (*DHCPManager, error) { - if nc == nil { - return nil, fmt.Errorf("DHCPManager: nats.Conn is required") - } - if js == nil { - return nil, fmt.Errorf("DHCPManager: JetStreamContext is required") - } - if client == nil { - return nil, fmt.Errorf("DHCPManager: dhcp.Client is required") - } - - kv, err := utils.GetOrCreateKVBucket(js, dhcp.KVBucketDHCPLeases, 3) - if err != nil { - return nil, fmt.Errorf("create DHCP lease KV bucket: %w", err) - } - if err := migrate.DefaultRegistry.RunKV(dhcp.KVBucketDHCPLeases, kv, dhcp.KVBucketDHCPLeasesVersion); err != nil { - return nil, fmt.Errorf("migrate %s: %w", dhcp.KVBucketDHCPLeases, err) - } - - ctx, cancel := context.WithCancel(context.Background()) - m := &DHCPManager{ - client: client, - kv: kv, - nc: nc, - // 90s = sum of default acquireBackoff (4+8+16+32 = 60s) + slack for - // socket open/close on each attempt and the final ACK round-trip. - acquireTimeout: 90 * time.Second, - acquireBackoff: []time.Duration{ - 4 * time.Second, - 8 * time.Second, - 16 * time.Second, - 32 * time.Second, - }, - jitterFraction: 0.1, - rng: rand.New(rand.NewPCG(uint64(time.Now().UnixNano()), 0xdeadbeef)), //nolint:gosec // non-cryptographic jitter - leases: map[string]*managedLease{}, - inFlight: map[string]int{}, - stopCtx: ctx, - stopCancel: cancel, - macForClientID: func(id string) net.HardwareAddr { - mac, err := net.ParseMAC(generateMAC(id)) - if err != nil { - return nil - } - return mac - }, - } - for _, opt := range opts { - opt(m) - } - return m, nil -} - -// Bootstrap reads every persisted lease from KV and schedules the -// renewal goroutine. Leases already past their expiry are removed and -// a TopicLeaseExpired event is published so the reconcilers upstream -// know they need to re-request. -func (m *DHCPManager) Bootstrap(ctx context.Context) error { - keys, err := m.kv.Keys() - if err != nil { - if errors.Is(err, nats.ErrNoKeysFound) { - slog.Info("DHCP Manager bootstrapped", "leases", 0) - return nil - } - return fmt.Errorf("list DHCP lease keys: %w", err) - } - - now := time.Now() - resumed, expired := 0, 0 - for _, key := range keys { - if key == "_version" { - continue - } - entry, err := m.kv.Get(key) - if err != nil { - slog.Warn("DHCP Manager: failed to load lease", "client_id", key, "err", err) - continue - } - lease, err := decodeLease(entry.Value()) - if err != nil { - slog.Warn("DHCP Manager: failed to decode lease", "client_id", key, "err", err) - continue - } - - if now.After(lease.ExpiresAt()) { - _ = m.kv.Delete(key) - m.publishExpired(lease, "", "bootstrap_past_expiry") - expired++ - continue - } - - m.trackAndRenew(lease) - resumed++ - } - slog.Info("DHCP Manager bootstrapped", "leases", resumed, "expired", expired) - return nil -} - -// Subscribe registers queue-subscribed handlers on vpc.dhcp.acquire and -// vpc.dhcp.release. Queue group "vpcd-dhcp-workers" matches the existing -// vpcd-workers convention. -func (m *DHCPManager) Subscribe(nc *nats.Conn) ([]*nats.Subscription, error) { - var subs []*nats.Subscription - - acquire, err := nc.QueueSubscribe(dhcp.TopicAcquire, "vpcd-dhcp-workers", m.handleAcquire) - if err != nil { - return nil, fmt.Errorf("subscribe %s: %w", dhcp.TopicAcquire, err) - } - subs = append(subs, acquire) - slog.Info("DHCP Manager subscribed", "topic", dhcp.TopicAcquire) - - release, err := nc.QueueSubscribe(dhcp.TopicRelease, "vpcd-dhcp-workers", m.handleRelease) - if err != nil { - _ = acquire.Unsubscribe() - return nil, fmt.Errorf("subscribe %s: %w", dhcp.TopicRelease, err) - } - subs = append(subs, release) - slog.Info("DHCP Manager subscribed", "topic", dhcp.TopicRelease) - - return subs, nil -} - -// Close stops all renewal goroutines and waits for them to exit. -func (m *DHCPManager) Close() { - m.stopCancel() - m.wg.Wait() -} - -// handleAcquire processes a vpc.dhcp.acquire request. Idempotent: if we -// already hold a valid lease for the requested ClientID we return it -// without a fresh DORA. Lets the daemon-side CAS retry loop be safe. -func (m *DHCPManager) handleAcquire(msg *nats.Msg) { - var req dhcp.AcquireRequestMsg - if err := json.Unmarshal(msg.Data, &req); err != nil { - m.replyAcquireError(msg, fmt.Sprintf("invalid acquire request: %v", err)) - return - } - if req.Bridge == "" || req.ClientID == "" { - m.replyAcquireError(msg, "acquire: bridge and client_id are required") - return - } - - if existing, ok := m.get(req.ClientID); ok { - if time.Now().Before(existing.ExpiresAt()) { - m.replyAcquire(msg, existing) - return - } - m.forget(req.ClientID, "stale_existing") - } - - hwAddr, err := resolveHWAddr(req.HWAddr, req.ClientID, m.macForClientID) - if err != nil { - m.replyAcquireError(msg, err.Error()) - return - } - - m.enablePromisc() - defer m.disablePromisc() - - ctx, cancel := context.WithTimeout(m.stopCtx, m.acquireTimeout) - defer cancel() - - // Diagnostic for mulga-siv-32: track concurrent Acquire calls per - // bridge and the wallclock cost of each DORA. Concurrent DISCOVERs on - // the same bridge are a suspected contributor to nightly DORA timeouts. - inFlight := m.beginAcquire(req.Bridge) - defer m.endAcquire(req.Bridge) - start := time.Now() - - lease, err := m.acquireWithBackoff(ctx, dhcp.AcquireRequest{ - Bridge: req.Bridge, - ClientID: req.ClientID, - Hostname: req.Hostname, - VendorClass: req.VendorClass, - HWAddr: hwAddr, - }) - if err != nil { - slog.Warn("DHCP Manager: acquire failed", - "client_id", req.ClientID, "bridge", req.Bridge, - "in_flight_at_start", inFlight, "duration", time.Since(start), - "err", err) - m.replyAcquireError(msg, err.Error()) - return - } - slog.Debug("DHCP Manager: acquire ok", - "client_id", req.ClientID, "bridge", req.Bridge, - "in_flight_at_start", inFlight, "duration", time.Since(start)) - - if err := m.persistLease(lease); err != nil { - slog.Warn("DHCP Manager: persist lease failed; lease will not survive restart", "client_id", req.ClientID, "err", err) - } - m.trackAndRenew(lease) - - slog.Info("DHCP Manager acquired lease", - "client_id", lease.ClientID, "ip", lease.IP.String(), - "bridge", lease.Bridge, "server_id", lease.ServerID.String(), - "expires_in", time.Until(lease.ExpiresAt())) - - m.replyAcquire(msg, lease) -} - -// handleRelease processes a vpc.dhcp.release request. -func (m *DHCPManager) handleRelease(msg *nats.Msg) { - var req dhcp.ReleaseRequestMsg - if err := json.Unmarshal(msg.Data, &req); err != nil { - m.replyRelease(msg, fmt.Sprintf("invalid release request: %v", err)) - return - } - if req.ClientID == "" { - m.replyRelease(msg, "release: client_id is required") - return - } - - lease, ok := m.get(req.ClientID) - if !ok { - // No active lease; treat as a no-op so the daemon can call release - // in clean-up paths without worrying about order. - m.replyRelease(msg, "") - return - } - - ctx, cancel := context.WithTimeout(m.stopCtx, m.acquireTimeout) - defer cancel() - if err := m.client.Release(ctx, lease); err != nil { - slog.Warn("DHCP Manager: release failed", "client_id", req.ClientID, "err", err) - // Still forget locally — upstream server may or may not have seen - // the RELEASE, but holding on to a lease we've been told to drop - // serves no purpose. - } - m.forget(req.ClientID, "released") - slog.Info("DHCP Manager released lease", "client_id", req.ClientID) - m.replyRelease(msg, "") -} - -// enablePromisc increments the in-flight acquire count and enables IFF_PROMISC -// on the first call. -func (m *DHCPManager) enablePromisc() { - if m.promiscOn == nil { - return - } - m.promiscMu.Lock() - defer m.promiscMu.Unlock() - if m.promiscCount == 0 { - m.promiscOn() - } - m.promiscCount++ -} - -// disablePromisc decrements the count and disables IFF_PROMISC when it reaches -// zero (i.e. no more in-flight acquires need it). -func (m *DHCPManager) disablePromisc() { - if m.promiscOff == nil { - return - } - m.promiscMu.Lock() - defer m.promiscMu.Unlock() - m.promiscCount-- - if m.promiscCount == 0 { - m.promiscOff() - } -} - -// trackAndRenew registers the lease in the in-memory map and starts the -// per-lease renewal goroutine. Replaces any prior tracker for the same -// ClientID. -func (m *DHCPManager) trackAndRenew(lease *dhcp.Lease) { - m.mu.Lock() - if prev, ok := m.leases[lease.ClientID]; ok && prev.renewStop != nil { - prev.renewStop() - m.mu.Unlock() - <-prev.renewDone - m.mu.Lock() - } - - // cancel is stored in tracker.renewStop and fired from forget() or Close(). - ctx, cancel := context.WithCancel(m.stopCtx) - tracker := &managedLease{ - lease: lease, - renewStop: cancel, - renewDone: make(chan struct{}), - } - m.leases[lease.ClientID] = tracker - m.mu.Unlock() - - m.wg.Add(1) - go m.renewLoop(ctx, tracker) -} - -// renewLoop drives one lease's renewal lifecycle. Sleeps until T1 (with -// ±jitter), attempts Renew; on failure retries at T2; on terminal failure -// past the expiry, publishes TopicLeaseExpired and forgets the lease. -func (m *DHCPManager) renewLoop(ctx context.Context, tracker *managedLease) { - defer m.wg.Done() - defer close(tracker.renewDone) - - for { - lease := m.snapshotLease(tracker) - if lease == nil { - return - } - - // Phase 1 — wait for T1 (± jitter) and try RENEW. - wait := m.jitter(time.Until(lease.RenewAt())) - if !sleep(ctx, wait) { - return - } - - renewed, err := m.tryRenew(ctx, lease) - if err == nil { - tracker.lease = renewed - if perr := m.persistLease(renewed); perr != nil { - slog.Warn("DHCP Manager: persist renewed lease failed", "client_id", renewed.ClientID, "err", perr) - } - continue - } - slog.Warn("DHCP Manager: renew failed, will retry at T2", "client_id", lease.ClientID, "err", err) - - // Phase 2 — retry until T2. - wait = m.jitter(time.Until(lease.RebindAt())) - if !sleep(ctx, wait) { - return - } - renewed, err = m.tryRenew(ctx, lease) - if err == nil { - tracker.lease = renewed - if perr := m.persistLease(renewed); perr != nil { - slog.Warn("DHCP Manager: persist rebound lease failed", "client_id", renewed.ClientID, "err", perr) - } - continue - } - - // Phase 3 — wait out the rest of the lease; on expiry, emit - // lease-expired and drop. - wait = time.Until(lease.ExpiresAt()) - if !sleep(ctx, wait) { - return - } - slog.Warn("DHCP Manager: lease expired without renewal", "client_id", lease.ClientID, "ip", lease.IP.String(), "err", err) - m.publishExpired(lease, "", classifyRenewErr(err)) - m.forget(lease.ClientID, "expired") - return - } -} - -// acquireWithBackoff drives RFC 2131 §4.1 DISCOVER retransmission. Each -// entry in acquireBackoff sets one attempt's per-OFFER wait window with -// ±1s jitter; on timeout the loop tries the next entry. Returns on first -// success, on parent ctx cancellation, or after the schedule is exhausted. -// -// Real DHCP servers (consumer routers, ISC dhcpd, dnsmasq) silently drop -// OFFERs under load — without retransmission a single lost packet kills -// the whole DORA. mulga-siv-39. -func (m *DHCPManager) acquireWithBackoff(parent context.Context, req dhcp.AcquireRequest) (*dhcp.Lease, error) { - schedule := m.acquireBackoff - if len(schedule) == 0 { - // Defensive: caller passed an empty schedule via WithDHCPAcquireBackoff. - // Fall through to a single attempt bounded by parent ctx. - schedule = []time.Duration{0} - } - - var lastErr error - for i, base := range schedule { - if err := parent.Err(); err != nil { - if lastErr != nil { - return nil, fmt.Errorf("dhcp acquire on %s (client=%s): %w (last attempt err: %v)", req.Bridge, req.ClientID, err, lastErr) - } - return nil, fmt.Errorf("dhcp acquire on %s (client=%s): %w", req.Bridge, req.ClientID, err) - } - - attemptTimeout := m.dhcpAttemptTimeout(base) - var ( - ctx context.Context - cancel context.CancelFunc - ) - if attemptTimeout > 0 { - ctx, cancel = context.WithTimeout(parent, attemptTimeout) - } else { - ctx, cancel = context.WithCancel(parent) - } - - lease, err := m.client.Acquire(ctx, req) - cancel() - if err == nil { - return lease, nil - } - lastErr = err - - if i < len(schedule)-1 { - slog.Debug("DHCP Manager: DORA retransmit", - "client_id", req.ClientID, "bridge", req.Bridge, - "attempt", i+1, "next_timeout", m.dhcpAttemptTimeout(schedule[i+1]), - "prev_err", err) - } - } - return nil, lastErr -} - -// dhcpAttemptTimeout applies ±1s jitter to a base DISCOVER timeout. -// Pass 0 to disable timeout (used when schedule is empty fallback). -func (m *DHCPManager) dhcpAttemptTimeout(base time.Duration) time.Duration { - if base <= 0 { - return 0 - } - jitter := time.Duration((m.rng.Float64()*2 - 1) * float64(time.Second)) - out := base + jitter - if out <= 0 { - return base - } - return out -} - -// tryRenew calls the backend Renew under a bounded context. -func (m *DHCPManager) tryRenew(parent context.Context, lease *dhcp.Lease) (*dhcp.Lease, error) { - ctx, cancel := context.WithTimeout(parent, m.acquireTimeout) - defer cancel() - return m.client.Renew(ctx, lease) -} - -// beginAcquire increments the in-flight Acquire counter for bridge and -// returns the count after increment (>= 1). Diagnostic for mulga-siv-32. -func (m *DHCPManager) beginAcquire(bridge string) int { - m.inFlightMu.Lock() - defer m.inFlightMu.Unlock() - m.inFlight[bridge]++ - return m.inFlight[bridge] -} - -// endAcquire decrements the in-flight Acquire counter for bridge. -func (m *DHCPManager) endAcquire(bridge string) { - m.inFlightMu.Lock() - defer m.inFlightMu.Unlock() - if m.inFlight[bridge] > 0 { - m.inFlight[bridge]-- - } -} - -// snapshotLease returns the tracker's current lease under the lock. -func (m *DHCPManager) snapshotLease(tracker *managedLease) *dhcp.Lease { - m.mu.Lock() - defer m.mu.Unlock() - return tracker.lease -} - -// get returns the current lease for a ClientID, if tracked. -func (m *DHCPManager) get(clientID string) (*dhcp.Lease, bool) { - m.mu.Lock() - defer m.mu.Unlock() - t, ok := m.leases[clientID] - if !ok { - return nil, false - } - return t.lease, true -} - -// forget removes a lease from the tracking map, KV and stops its renewal -// goroutine. -func (m *DHCPManager) forget(clientID, reason string) { - m.mu.Lock() - t, ok := m.leases[clientID] - if ok { - delete(m.leases, clientID) - } - m.mu.Unlock() - - if ok && t.renewStop != nil { - t.renewStop() - } - if err := m.kv.Delete(clientID); err != nil && !errors.Is(err, nats.ErrKeyNotFound) { - slog.Debug("DHCP Manager: delete KV key failed", "client_id", clientID, "reason", reason, "err", err) - } -} - -// persistLease writes the lease state to the KV bucket under ClientID. -func (m *DHCPManager) persistLease(lease *dhcp.Lease) error { - data, err := encodeLease(lease) - if err != nil { - return err - } - if _, err := m.kv.Put(lease.ClientID, data); err != nil { - return fmt.Errorf("put lease %s: %w", lease.ClientID, err) - } - return nil -} - -func (m *DHCPManager) publishExpired(lease *dhcp.Lease, poolName, reason string) { - evt := dhcp.LeaseExpiredEvent{ - ClientID: lease.ClientID, - PoolName: poolName, - IP: lease.IP.String(), - Reason: reason, - } - data, err := json.Marshal(evt) - if err != nil { - slog.Warn("DHCP Manager: marshal lease-expired event failed", "client_id", lease.ClientID, "err", err) - return - } - if err := m.nc.Publish(dhcp.TopicLeaseExpired, data); err != nil { - slog.Warn("DHCP Manager: publish lease-expired failed", "client_id", lease.ClientID, "err", err) - } -} - -func (m *DHCPManager) replyAcquire(msg *nats.Msg, lease *dhcp.Lease) { - reply := dhcp.AcquireReplyMsg{ - IP: lease.IP.String(), - SubnetMask: net.IP(lease.SubnetMask).String(), - Routers: ipsToStrings(lease.Routers), - DNS: ipsToStrings(lease.DNS), - ServerID: lease.ServerID.String(), - HWAddr: lease.HWAddr.String(), - ExpiresUnix: lease.ExpiresAt().Unix(), - } - respondJSON(msg, reply) -} - -func (m *DHCPManager) replyAcquireError(msg *nats.Msg, errMsg string) { - respondJSON(msg, dhcp.AcquireReplyMsg{Error: errMsg}) -} - -func (m *DHCPManager) replyRelease(msg *nats.Msg, errMsg string) { - respondJSON(msg, dhcp.ReleaseReplyMsg{Error: errMsg}) -} - -// jitter applies ±jitterFraction of d to produce an actual sleep duration. -// Negative d is clamped to 0. -func (m *DHCPManager) jitter(d time.Duration) time.Duration { - if d <= 0 { - return 0 - } - if m.jitterFraction <= 0 { - return d - } - spread := float64(d) * m.jitterFraction - offset := (m.rng.Float64()*2 - 1) * spread - out := time.Duration(float64(d) + offset) - if out < 0 { - return 0 - } - return out -} - -// resolveHWAddr parses the hw_addr string supplied in the NATS request, -// falling back to a derived MAC when the request carries no value. -func resolveHWAddr(raw, clientID string, derive func(string) net.HardwareAddr) (net.HardwareAddr, error) { - if raw != "" { - mac, err := net.ParseMAC(raw) - if err != nil { - return nil, fmt.Errorf("parse hw_addr %q: %w", raw, err) - } - return mac, nil - } - if derive == nil { - return nil, fmt.Errorf("hw_addr is empty and no derivation function configured") - } - mac := derive(clientID) - if len(mac) == 0 { - return nil, fmt.Errorf("derive hw_addr for client %q: empty result", clientID) - } - return mac, nil -} - -// classifyRenewErr maps a renew error to a short reason code on the -// lease-expired event. Kept coarse; callers that care about specifics -// can re-log from their side. -func classifyRenewErr(err error) string { - if err == nil { - return "unknown" - } - // nclient4 returns *nclient4.ErrNak on NAK. Rather than taking a - // dependency on its private type here, match the known message. - msg := err.Error() - if strings.Contains(msg, "NAK") || strings.Contains(msg, "Nak") { - return "nak" - } - if strings.Contains(msg, "timeout") || strings.Contains(msg, "deadline exceeded") || strings.Contains(msg, "i/o timeout") { - return "server_unreachable" - } - return "renew_failed" -} - -func ipsToStrings(ips []net.IP) []string { - if len(ips) == 0 { - return nil - } - out := make([]string, 0, len(ips)) - for _, ip := range ips { - out = append(out, ip.String()) - } - return out -} - -// respondJSON marshals v and sends it as the reply. If the message has -// no Reply subject (fire-and-forget), drops the reply silently. -func respondJSON(msg *nats.Msg, v any) { - if msg.Reply == "" { - return - } - data, err := json.Marshal(v) - if err != nil { - slog.Warn("DHCP Manager: marshal reply failed", "err", err) - return - } - if err := msg.Respond(data); err != nil { - slog.Warn("DHCP Manager: respond failed", "err", err) - } -} - -// sleep blocks until d elapses or ctx is done. Returns false if ctx was -// cancelled (caller should exit). -func sleep(ctx context.Context, d time.Duration) bool { - if d <= 0 { - select { - case <-ctx.Done(): - return false - default: - return true - } - } - t := time.NewTimer(d) - defer t.Stop() - select { - case <-ctx.Done(): - return false - case <-t.C: - return true - } -} - -// --- KV encoding --- - -// leaseRecord is the JSON representation persisted in KV and used to -// rehydrate on vpcd startup. Explicit format rather than gob-encoding -// *dhcp.Lease so a future Go type change doesn't invalidate every -// in-flight lease. -type leaseRecord struct { - Version int `json:"version"` - Bridge string `json:"bridge"` - ClientID string `json:"client_id"` - Hostname string `json:"hostname,omitempty"` - VendorClass string `json:"vendor_class,omitempty"` - HWAddr string `json:"hw_addr"` - IP string `json:"ip"` - SubnetMask string `json:"subnet_mask,omitempty"` - Routers []string `json:"routers,omitempty"` - DNS []string `json:"dns,omitempty"` - ServerID string `json:"server_id,omitempty"` - AcquiredUnix int64 `json:"acquired_unix"` - LeaseSeconds int64 `json:"lease_seconds"` - T1Seconds int64 `json:"t1_seconds"` - T2Seconds int64 `json:"t2_seconds"` - RawOfferB64 []byte `json:"raw_offer,omitempty"` - RawACKB64 []byte `json:"raw_ack,omitempty"` -} - -const leaseRecordVersion = 1 - -func encodeLease(l *dhcp.Lease) ([]byte, error) { - rec := leaseRecord{ - Version: leaseRecordVersion, - Bridge: l.Bridge, - ClientID: l.ClientID, - Hostname: l.Hostname, - VendorClass: l.VendorClass, - HWAddr: l.HWAddr.String(), - IP: l.IP.String(), - SubnetMask: maskString(l.SubnetMask), - Routers: ipsToStrings(l.Routers), - DNS: ipsToStrings(l.DNS), - ServerID: l.ServerID.String(), - AcquiredUnix: l.AcquiredAt.Unix(), - LeaseSeconds: int64(l.LeaseDuration / time.Second), - T1Seconds: int64(l.T1 / time.Second), - T2Seconds: int64(l.T2 / time.Second), - RawOfferB64: l.RawOffer, - RawACKB64: l.RawACK, - } - return json.Marshal(rec) -} - -func decodeLease(data []byte) (*dhcp.Lease, error) { - var rec leaseRecord - if err := json.Unmarshal(data, &rec); err != nil { - return nil, fmt.Errorf("unmarshal lease: %w", err) - } - if rec.Version != leaseRecordVersion { - return nil, fmt.Errorf("unsupported lease record version %d", rec.Version) - } - mac, err := net.ParseMAC(rec.HWAddr) - if err != nil { - return nil, fmt.Errorf("parse hw_addr %q: %w", rec.HWAddr, err) - } - return &dhcp.Lease{ - Bridge: rec.Bridge, - ClientID: rec.ClientID, - Hostname: rec.Hostname, - VendorClass: rec.VendorClass, - HWAddr: mac, - IP: net.ParseIP(rec.IP), - SubnetMask: parseMask(rec.SubnetMask), - Routers: parseIPs(rec.Routers), - DNS: parseIPs(rec.DNS), - ServerID: net.ParseIP(rec.ServerID), - AcquiredAt: time.Unix(rec.AcquiredUnix, 0), - LeaseDuration: time.Duration(rec.LeaseSeconds) * time.Second, - T1: time.Duration(rec.T1Seconds) * time.Second, - T2: time.Duration(rec.T2Seconds) * time.Second, - RawOffer: rec.RawOfferB64, - RawACK: rec.RawACKB64, - }, nil -} - -func maskString(m net.IPMask) string { - if len(m) == 0 { - return "" - } - return net.IP(m).String() -} - -func parseMask(s string) net.IPMask { - ip := net.ParseIP(s) - if ip == nil { - return nil - } - ip4 := ip.To4() - if ip4 == nil { - return nil - } - return net.IPv4Mask(ip4[0], ip4[1], ip4[2], ip4[3]) -} - -func parseIPs(ss []string) []net.IP { - if len(ss) == 0 { - return nil - } - out := make([]net.IP, 0, len(ss)) - for _, s := range ss { - if ip := net.ParseIP(s); ip != nil { - out = append(out, ip) - } - } - return out -} diff --git a/spinifex/services/vpcd/dhcp_manager_test.go b/spinifex/services/vpcd/dhcp_manager_test.go deleted file mode 100644 index 3b724da1c..000000000 --- a/spinifex/services/vpcd/dhcp_manager_test.go +++ /dev/null @@ -1,555 +0,0 @@ -package vpcd - -import ( - "context" - "encoding/json" - "errors" - "net" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/mulgadc/spinifex/spinifex/services/vpcd/dhcp" - "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/nats-io/nats.go" - "github.com/stretchr/testify/require" -) - -// newTestDHCPManager spins up an embedded JetStream, wires a Fake client, -// and returns the Manager along with the NATS connection. Cleanup of NATS -// is handled by testutil; the Manager is closed via t.Cleanup. -func newTestDHCPManager(t *testing.T, opts ...DHCPManagerOption) (*DHCPManager, *nats.Conn, *dhcp.Fake) { - t.Helper() - _, nc, js := testutil.StartTestJetStream(t) - fake := dhcp.NewFake() - m, err := NewDHCPManager(nc, js, fake, opts...) - require.NoError(t, err) - t.Cleanup(m.Close) - subs, err := m.Subscribe(nc) - require.NoError(t, err) - t.Cleanup(func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }) - return m, nc, fake -} - -func requestAcquire(t *testing.T, nc *nats.Conn, req dhcp.AcquireRequestMsg) dhcp.AcquireReplyMsg { - t.Helper() - data, err := json.Marshal(req) - require.NoError(t, err) - msg, err := nc.Request(dhcp.TopicAcquire, data, 5*time.Second) - require.NoError(t, err) - var reply dhcp.AcquireReplyMsg - require.NoError(t, json.Unmarshal(msg.Data, &reply)) - return reply -} - -func requestRelease(t *testing.T, nc *nats.Conn, clientID string) dhcp.ReleaseReplyMsg { - t.Helper() - data, err := json.Marshal(dhcp.ReleaseRequestMsg{ClientID: clientID}) - require.NoError(t, err) - msg, err := nc.Request(dhcp.TopicRelease, data, 5*time.Second) - require.NoError(t, err) - var reply dhcp.ReleaseReplyMsg - require.NoError(t, json.Unmarshal(msg.Data, &reply)) - return reply -} - -func TestDHCPManager_AcquireReturnsLease(t *testing.T) { - _, nc, fake := newTestDHCPManager(t) - reply := requestAcquire(t, nc, dhcp.AcquireRequestMsg{ - Bridge: "br-wan", ClientID: "eni-1", Hostname: "eni-1", VendorClass: "i-abc", - }) - require.Empty(t, reply.Error, "unexpected error: %s", reply.Error) - require.Equal(t, "192.0.2.100", reply.IP) - require.NotEmpty(t, reply.ServerID) - require.NotZero(t, reply.ExpiresUnix) - require.NotEmpty(t, reply.HWAddr) - require.Equal(t, 1, fake.AcquireCount()) -} - -func TestDHCPManager_AcquireIdempotentWhileLeaseActive(t *testing.T) { - _, nc, fake := newTestDHCPManager(t) - req := dhcp.AcquireRequestMsg{Bridge: "br-wan", ClientID: "eni-idem"} - - first := requestAcquire(t, nc, req) - require.Empty(t, first.Error) - second := requestAcquire(t, nc, req) - require.Empty(t, second.Error) - - require.Equal(t, first.IP, second.IP, "idempotent acquire should return the same IP") - require.Equal(t, 1, fake.AcquireCount(), "second request must not trigger a fresh DORA") -} - -func TestDHCPManager_AcquireFailureSurfacesError(t *testing.T) { - _, nc, fake := newTestDHCPManager(t) - fake.AcquireHook = func(dhcp.AcquireRequest) (*dhcp.Lease, error) { - return nil, errors.New("server unreachable") - } - reply := requestAcquire(t, nc, dhcp.AcquireRequestMsg{Bridge: "br-wan", ClientID: "eni-err"}) - require.Contains(t, reply.Error, "server unreachable") - require.Empty(t, reply.IP) -} - -func TestDHCPManager_ReleaseRemovesLease(t *testing.T) { - m, nc, fake := newTestDHCPManager(t) - req := dhcp.AcquireRequestMsg{Bridge: "br-wan", ClientID: "eni-release"} - require.Empty(t, requestAcquire(t, nc, req).Error) - - reply := requestRelease(t, nc, req.ClientID) - require.Empty(t, reply.Error) - require.Equal(t, 1, fake.ReleaseCount()) - - _, ok := m.get("eni-release") - require.False(t, ok, "lease should be forgotten after Release") -} - -func TestDHCPManager_ReleaseUnknownIsNoop(t *testing.T) { - _, nc, fake := newTestDHCPManager(t) - reply := requestRelease(t, nc, "nonexistent") - require.Empty(t, reply.Error) - require.Equal(t, 0, fake.ReleaseCount()) -} - -func TestDHCPManager_AcquireRejectsMissingFields(t *testing.T) { - _, nc, _ := newTestDHCPManager(t) - reply := requestAcquire(t, nc, dhcp.AcquireRequestMsg{Bridge: "", ClientID: "eni-x"}) - require.Contains(t, reply.Error, "bridge and client_id are required") -} - -func TestDHCPManager_HWAddrDerivedFromClientID(t *testing.T) { - _, nc, fake := newTestDHCPManager(t) - reply := requestAcquire(t, nc, dhcp.AcquireRequestMsg{Bridge: "br-wan", ClientID: "eni-mac"}) - require.Empty(t, reply.Error) - lease, ok := fake.HeldLease("eni-mac") - require.True(t, ok) - // generateMAC produces a locally-administered unicast MAC. First octet - // is hash-derived; assert the IEEE 802 reserved bits. - require.Equal(t, byte(0x02), lease.HWAddr[0]&0x03) -} - -func TestDHCPManager_HWAddrExplicitlySuppliedIsUsed(t *testing.T) { - _, nc, fake := newTestDHCPManager(t) - reply := requestAcquire(t, nc, dhcp.AcquireRequestMsg{ - Bridge: "br-wan", ClientID: "eni-mac-explicit", HWAddr: "02:aa:bb:cc:dd:ee", - }) - require.Empty(t, reply.Error) - lease, _ := fake.HeldLease("eni-mac-explicit") - require.Equal(t, "02:aa:bb:cc:dd:ee", lease.HWAddr.String()) -} - -func TestDHCPManager_RenewalFiresAndUpdatesLease(t *testing.T) { - tmpl := dhcp.DefaultLeaseTemplate() - tmpl.LeaseDuration = 40 * time.Millisecond - fake := dhcp.NewFake() - fake.SetDefaultLease(tmpl) - - _, nc, js := testutil.StartTestJetStream(t) - m, err := NewDHCPManager(nc, js, fake, WithDHCPJitterFraction(0)) - require.NoError(t, err) - t.Cleanup(m.Close) - subs, err := m.Subscribe(nc) - require.NoError(t, err) - t.Cleanup(func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }) - - reply := requestAcquire(t, nc, dhcp.AcquireRequestMsg{Bridge: "br-wan", ClientID: "eni-renew"}) - require.Empty(t, reply.Error) - - // T1 = lease/2 = 20 ms. Wait a few multiples to catch at least one renew. - require.Eventually(t, func() bool { return fake.RenewCount() >= 1 }, 2*time.Second, 10*time.Millisecond) -} - -func TestDHCPManager_RenewFailureEmitsLeaseExpired(t *testing.T) { - tmpl := dhcp.DefaultLeaseTemplate() - tmpl.LeaseDuration = 60 * time.Millisecond - fake := dhcp.NewFake() - fake.SetDefaultLease(tmpl) - fake.RenewHook = func(*dhcp.Lease) (*dhcp.Lease, error) { - return nil, errors.New("server NAK") - } - - _, nc, js := testutil.StartTestJetStream(t) - m, err := NewDHCPManager(nc, js, fake, WithDHCPJitterFraction(0)) - require.NoError(t, err) - t.Cleanup(m.Close) - - var events atomic.Int32 - expiredMsg := make(chan dhcp.LeaseExpiredEvent, 1) - sub, err := nc.Subscribe(dhcp.TopicLeaseExpired, func(msg *nats.Msg) { - events.Add(1) - var evt dhcp.LeaseExpiredEvent - if err := json.Unmarshal(msg.Data, &evt); err == nil { - select { - case expiredMsg <- evt: - default: - } - } - }) - require.NoError(t, err) - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - subs, err := m.Subscribe(nc) - require.NoError(t, err) - t.Cleanup(func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }) - - reply := requestAcquire(t, nc, dhcp.AcquireRequestMsg{Bridge: "br-wan", ClientID: "eni-expire"}) - require.Empty(t, reply.Error) - - // T1=30ms → try, fail. T2=~52ms → try, fail. Expires at 60ms. - select { - case evt := <-expiredMsg: - require.Equal(t, "eni-expire", evt.ClientID) - require.Equal(t, "nak", evt.Reason) - case <-time.After(2 * time.Second): - t.Fatalf("expected lease-expired event (events=%d)", events.Load()) - } -} - -func TestDHCPManager_BootstrapRehydratesLease(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - fake := dhcp.NewFake() - - m1, err := NewDHCPManager(nc, js, fake) - require.NoError(t, err) - subs, err := m1.Subscribe(nc) - require.NoError(t, err) - - reply := requestAcquire(t, nc, dhcp.AcquireRequestMsg{Bridge: "br-wan", ClientID: "eni-boot"}) - require.Empty(t, reply.Error) - - // Close first manager; KV entry should survive. - for _, s := range subs { - _ = s.Unsubscribe() - } - m1.Close() - - // Second manager picks up the lease on Bootstrap. - fake2 := dhcp.NewFake() - m2, err := NewDHCPManager(nc, js, fake2) - require.NoError(t, err) - t.Cleanup(m2.Close) - require.NoError(t, m2.Bootstrap(context.Background())) - - lease, ok := m2.get("eni-boot") - require.True(t, ok, "manager should rehydrate persisted lease") - require.Equal(t, "192.0.2.100", lease.IP.String()) - require.Equal(t, 0, fake2.AcquireCount(), "no fresh DORA on bootstrap") -} - -func TestDHCPManager_BootstrapDropsExpiredLeases(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - fake := dhcp.NewFake() - - m1, err := NewDHCPManager(nc, js, fake) - require.NoError(t, err) - subs, err := m1.Subscribe(nc) - require.NoError(t, err) - - reply := requestAcquire(t, nc, dhcp.AcquireRequestMsg{Bridge: "br-wan", ClientID: "eni-stale"}) - require.Empty(t, reply.Error) - - for _, s := range subs { - _ = s.Unsubscribe() - } - m1.Close() - - // Rewrite the persisted record so AcquiredAt + LeaseDuration is in the past. - kv, err := js.KeyValue(dhcp.KVBucketDHCPLeases) - require.NoError(t, err) - entry, err := kv.Get("eni-stale") - require.NoError(t, err) - var rec map[string]any - require.NoError(t, json.Unmarshal(entry.Value(), &rec)) - rec["acquired_unix"] = time.Now().Add(-2 * time.Hour).Unix() - rec["lease_seconds"] = int64(60) - raw, err := json.Marshal(rec) - require.NoError(t, err) - _, err = kv.Put("eni-stale", raw) - require.NoError(t, err) - - fake2 := dhcp.NewFake() - m2, err := NewDHCPManager(nc, js, fake2) - require.NoError(t, err) - t.Cleanup(m2.Close) - - var expired sync.WaitGroup - expired.Add(1) - sub, err := nc.Subscribe(dhcp.TopicLeaseExpired, func(msg *nats.Msg) { - var evt dhcp.LeaseExpiredEvent - if err := json.Unmarshal(msg.Data, &evt); err == nil && evt.ClientID == "eni-stale" { - expired.Done() - } - }) - require.NoError(t, err) - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - require.NoError(t, m2.Bootstrap(context.Background())) - - done := make(chan struct{}) - go func() { expired.Wait(); close(done) }() - select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatal("expected lease-expired event for stale lease") - } - _, err = kv.Get("eni-stale") - require.ErrorIs(t, err, nats.ErrKeyNotFound) -} - -func TestDHCPManager_JitterWithinBounds(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - m, err := NewDHCPManager(nc, js, dhcp.NewFake(), WithDHCPJitterFraction(0.1)) - require.NoError(t, err) - t.Cleanup(m.Close) - - base := 100 * time.Millisecond - minOK := time.Duration(float64(base) * 0.9) - maxOK := time.Duration(float64(base) * 1.1) - for i := range 100 { - got := m.jitter(base) - if got < minOK || got > maxOK { - t.Fatalf("iteration %d: jitter produced %v, outside [%v,%v]", i, got, minOK, maxOK) - } - } -} - -func TestNewDHCPManager_RejectsNilDeps(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - - _, err := NewDHCPManager(nil, js, dhcp.NewFake()) - require.ErrorContains(t, err, "nats.Conn is required") - - _, err = NewDHCPManager(nc, nil, dhcp.NewFake()) - require.ErrorContains(t, err, "JetStreamContext is required") - - _, err = NewDHCPManager(nc, js, nil) - require.ErrorContains(t, err, "dhcp.Client is required") -} - -func TestDHCPManager_OptionsApply(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - - customMAC := func(string) net.HardwareAddr { - mac, _ := net.ParseMAC("02:de:ad:be:ef:01") - return mac - } - m, err := NewDHCPManager(nc, js, dhcp.NewFake(), - WithDHCPMACFunc(customMAC), - WithDHCPAcquireTimeout(1*time.Second), - WithDHCPJitterFraction(0.05), - ) - require.NoError(t, err) - t.Cleanup(m.Close) - require.Equal(t, time.Second, m.acquireTimeout) - require.InDelta(t, 0.05, m.jitterFraction, 0.0001) - require.Equal(t, "02:de:ad:be:ef:01", m.macForClientID("anything").String()) -} - -func TestDHCPManager_BootstrapEmptyBucket(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - m, err := NewDHCPManager(nc, js, dhcp.NewFake()) - require.NoError(t, err) - t.Cleanup(m.Close) - - // No leases in KV → Bootstrap is a clean no-op. - require.NoError(t, m.Bootstrap(context.Background())) -} - -func TestDHCPManager_JitterWithZeroFractionReturnsInput(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - m, err := NewDHCPManager(nc, js, dhcp.NewFake(), WithDHCPJitterFraction(0)) - require.NoError(t, err) - t.Cleanup(m.Close) - require.Equal(t, 100*time.Millisecond, m.jitter(100*time.Millisecond)) - require.Equal(t, time.Duration(0), m.jitter(0)) -} - -func TestDHCPManager_ClassifyRenewErr(t *testing.T) { - require.Equal(t, "unknown", classifyRenewErr(nil)) - require.Equal(t, "nak", classifyRenewErr(errors.New("got NAK from server"))) - require.Equal(t, "server_unreachable", classifyRenewErr(errors.New("i/o timeout"))) - require.Equal(t, "renew_failed", classifyRenewErr(errors.New("something else"))) -} - -func TestDHCPManager_EncodeDecodeLeaseRoundTrip(t *testing.T) { - mac, _ := net.ParseMAC("02:00:00:de:ad:be") - in := &dhcp.Lease{ - Bridge: "br-wan", ClientID: "eni-rt", Hostname: "eni-rt", VendorClass: "i-xyz", - HWAddr: mac, - IP: net.ParseIP("192.0.2.50"), - SubnetMask: net.CIDRMask(24, 32), - Routers: []net.IP{net.ParseIP("192.0.2.1")}, - DNS: []net.IP{net.ParseIP("1.1.1.1")}, - ServerID: net.ParseIP("192.0.2.1"), - AcquiredAt: time.Unix(1_700_000_000, 0).UTC(), - LeaseDuration: time.Hour, - T1: 30 * time.Minute, - T2: 52 * time.Minute, - RawOffer: []byte{0x01, 0x02}, - RawACK: []byte{0x03, 0x04}, - } - data, err := encodeLease(in) - require.NoError(t, err) - out, err := decodeLease(data) - require.NoError(t, err) - - require.Equal(t, in.ClientID, out.ClientID) - require.Equal(t, in.IP.String(), out.IP.String()) - require.Equal(t, in.LeaseDuration, out.LeaseDuration) - require.Equal(t, in.HWAddr.String(), out.HWAddr.String()) - require.Equal(t, in.RawACK, out.RawACK) - require.Equal(t, in.RawOffer, out.RawOffer) -} - -// --- acquireWithBackoff (mulga-siv-39, RFC 2131 §4.1) --- - -func newBackoffManager(t *testing.T, schedule []time.Duration, fake *dhcp.Fake) *DHCPManager { - t.Helper() - _, nc, js := testutil.StartTestJetStream(t) - m, err := NewDHCPManager(nc, js, fake, WithDHCPAcquireBackoff(schedule)) - require.NoError(t, err) - t.Cleanup(m.Close) - return m -} - -func TestAcquireWithBackoff_FirstAttemptSucceeds(t *testing.T) { - fake := dhcp.NewFake() - m := newBackoffManager(t, []time.Duration{10 * time.Millisecond, 10 * time.Millisecond}, fake) - - lease, err := m.acquireWithBackoff(context.Background(), dhcp.AcquireRequest{ - Bridge: "br-wan", ClientID: "eni-fast", HWAddr: net.HardwareAddr{0x02, 0, 0, 0, 0, 1}, - }) - require.NoError(t, err) - require.NotNil(t, lease) - require.Equal(t, 1, fake.AcquireCount(), "first-attempt success must not retransmit") -} - -func TestAcquireWithBackoff_RetriesThenSucceeds(t *testing.T) { - fake := dhcp.NewFake() - var calls atomic.Int32 - tmpl := dhcp.DefaultLeaseTemplate() - fake.AcquireHook = func(req dhcp.AcquireRequest) (*dhcp.Lease, error) { - n := calls.Add(1) - if n < 3 { - return nil, errors.New("simulated OFFER timeout") - } - return &dhcp.Lease{ - Bridge: req.Bridge, ClientID: req.ClientID, HWAddr: req.HWAddr, - IP: tmpl.IP, ServerID: tmpl.ServerID, LeaseDuration: tmpl.LeaseDuration, - AcquiredAt: time.Now(), - }, nil - } - m := newBackoffManager(t, []time.Duration{ - 5 * time.Millisecond, 5 * time.Millisecond, 5 * time.Millisecond, 5 * time.Millisecond, - }, fake) - - lease, err := m.acquireWithBackoff(context.Background(), dhcp.AcquireRequest{ - Bridge: "br-wan", ClientID: "eni-retry", HWAddr: net.HardwareAddr{0x02, 0, 0, 0, 0, 2}, - }) - require.NoError(t, err) - require.NotNil(t, lease) - require.Equal(t, int32(3), calls.Load()) -} - -func TestAcquireWithBackoff_AllAttemptsFail(t *testing.T) { - fake := dhcp.NewFake() - fake.AcquireHook = func(dhcp.AcquireRequest) (*dhcp.Lease, error) { - return nil, errors.New("server silent") - } - schedule := []time.Duration{ - 2 * time.Millisecond, 2 * time.Millisecond, 2 * time.Millisecond, - } - m := newBackoffManager(t, schedule, fake) - - lease, err := m.acquireWithBackoff(context.Background(), dhcp.AcquireRequest{ - Bridge: "br-wan", ClientID: "eni-fail", HWAddr: net.HardwareAddr{0x02, 0, 0, 0, 0, 3}, - }) - require.Error(t, err) - require.Nil(t, lease) - require.ErrorContains(t, err, "server silent") - require.Equal(t, len(schedule), fake.AcquireCount(), - "every entry in the schedule must run before giving up") -} - -func TestAcquireWithBackoff_ParentCtxCancel(t *testing.T) { - fake := dhcp.NewFake() - fake.AcquireHook = func(dhcp.AcquireRequest) (*dhcp.Lease, error) { - return nil, errors.New("should not be called") - } - m := newBackoffManager(t, []time.Duration{ - time.Second, time.Second, time.Second, - }, fake) - - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - lease, err := m.acquireWithBackoff(ctx, dhcp.AcquireRequest{ - Bridge: "br-wan", ClientID: "eni-cancel", HWAddr: net.HardwareAddr{0x02, 0, 0, 0, 0, 4}, - }) - require.Error(t, err) - require.Nil(t, lease) - require.ErrorIs(t, err, context.Canceled) - require.Equal(t, 0, fake.AcquireCount(), "cancelled parent must short-circuit before any attempt") -} - -func TestDHCPAttemptTimeoutJitterBounds(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - m, err := NewDHCPManager(nc, js, dhcp.NewFake()) - require.NoError(t, err) - t.Cleanup(m.Close) - - base := 4 * time.Second - for i := range 200 { - got := m.dhcpAttemptTimeout(base) - if got < base-time.Second || got > base+time.Second { - t.Fatalf("iter %d: dhcpAttemptTimeout(%v) = %v, outside ±1s", i, base, got) - } - } - require.Zero(t, m.dhcpAttemptTimeout(0)) -} - -func TestNewDHCPManager_DefaultBackoffSchedule(t *testing.T) { - _, nc, js := testutil.StartTestJetStream(t) - m, err := NewDHCPManager(nc, js, dhcp.NewFake()) - require.NoError(t, err) - t.Cleanup(m.Close) - - require.Equal(t, []time.Duration{ - 4 * time.Second, 8 * time.Second, 16 * time.Second, 32 * time.Second, - }, m.acquireBackoff) - require.Equal(t, 90*time.Second, m.acquireTimeout) -} - -func TestResolveHWAddr(t *testing.T) { - t.Run("InvalidRawMACReturnsParseError", func(t *testing.T) { - mac, err := resolveHWAddr("not-a-mac", "client-1", nil) - require.Error(t, err) - require.Nil(t, mac) - require.Contains(t, err.Error(), "parse hw_addr") - }) - - t.Run("EmptyRawWithNilDeriveErrors", func(t *testing.T) { - mac, err := resolveHWAddr("", "client-1", nil) - require.Error(t, err) - require.Nil(t, mac) - require.Contains(t, err.Error(), "no derivation function") - }) - - t.Run("EmptyRawWithDeriveReturningEmptyErrors", func(t *testing.T) { - mac, err := resolveHWAddr("", "client-1", func(string) net.HardwareAddr { return nil }) - require.Error(t, err) - require.Nil(t, mac) - require.Contains(t, err.Error(), "empty result") - }) -} diff --git a/spinifex/services/vpcd/helpers_test.go b/spinifex/services/vpcd/helpers_test.go deleted file mode 100644 index 9c86d18bd..000000000 --- a/spinifex/services/vpcd/helpers_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package vpcd - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// Additional edge-case tests for subnetGateway. -// Core happy-path tests are in topology_test.go. - -func TestSubnetGateway_Slash28(t *testing.T) { - gw, prefix, err := subnetGateway("172.31.0.0/28") - require.NoError(t, err) - assert.Equal(t, "172.31.0.1", gw) - assert.Equal(t, 28, prefix) -} - -func TestSubnetGateway_IPv6(t *testing.T) { - _, _, err := subnetGateway("2001:db8::/32") - require.Error(t, err) - assert.Contains(t, err.Error(), "only IPv4 supported") -} diff --git a/spinifex/services/vpcd/integration_test.go b/spinifex/services/vpcd/integration_test.go deleted file mode 100644 index ae08465b2..000000000 --- a/spinifex/services/vpcd/integration_test.go +++ /dev/null @@ -1,544 +0,0 @@ -package vpcd - -import ( - "context" - "encoding/json" - "testing" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ec2" - handlers_ec2_igw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/igw" - handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" - "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats-server/v2/server" - "github.com/nats-io/nats.go" -) - -const testIGWAccountID = "123456789012" - -// startTestJetStreamNATS starts an embedded NATS server with JetStream for integration tests. -func startTestJetStreamNATS(t *testing.T) (*server.Server, *nats.Conn) { - t.Helper() - ns, nc, _ := testutil.StartTestJetStream(t) - return ns, nc -} - -// TestIntegration_VPCLifecycle tests the full VPC lifecycle: -// VPC create → subnet create → ENI create → IGW attach → IGW detach → cleanup. -// This validates the NATS event flow between the daemon and vpcd topology handler. -func TestIntegration_VPCLifecycle(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - // Set up topology handler subscriptions (simulates vpcd) - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe topology handler: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // === Phase 1: Create VPC === - vpcEvt := VPCEvent{VpcId: "vpc-integ1", CidrBlock: "10.0.0.0/16", VNI: 1000} - vpcData, _ := json.Marshal(vpcEvt) - resp, err := nc.Request(TopicVPCCreate, vpcData, 5*time.Second) - if err != nil { - t.Fatalf("vpc.create: %v", err) - } - assertSuccess(t, resp, "create VPC") - - // Verify OVN: logical router exists - router, err := mock.GetLogicalRouter(ctx, "vpc-vpc-integ1") - if err != nil { - t.Fatalf("expected VPC router: %v", err) - } - if router.ExternalIDs["spinifex:vpc_id"] != "vpc-integ1" { - t.Errorf("router vpc_id = %s, want vpc-integ1", router.ExternalIDs["spinifex:vpc_id"]) - } - - // === Phase 2: Create Subnet === - subEvt := SubnetEvent{SubnetId: "subnet-integ1", VpcId: "vpc-integ1", CidrBlock: "10.0.1.0/24"} - subData, _ := json.Marshal(subEvt) - resp, err = nc.Request(TopicSubnetCreate, subData, 5*time.Second) - if err != nil { - t.Fatalf("vpc.create-subnet: %v", err) - } - assertSuccess(t, resp, "create subnet") - - // Verify OVN: logical switch, router port, DHCP - ls, err := mock.GetLogicalSwitch(ctx, "subnet-subnet-integ1") - if err != nil { - t.Fatalf("expected logical switch: %v", err) - } - if ls.ExternalIDs["spinifex:subnet_id"] != "subnet-integ1" { - t.Errorf("switch subnet_id = %s, want subnet-integ1", ls.ExternalIDs["spinifex:subnet_id"]) - } - dhcp, err := mock.FindDHCPOptionsByCIDR(ctx, "10.0.1.0/24") - if err != nil { - t.Fatalf("expected DHCP options: %v", err) - } - if dhcp.Options["router"] != "10.0.1.1" { - t.Errorf("DHCP router = %s, want 10.0.1.1", dhcp.Options["router"]) - } - - // Verify router has 1 port (subnet port) - router, _ = mock.GetLogicalRouter(ctx, "vpc-vpc-integ1") - if len(router.Ports) != 1 { - t.Errorf("router ports = %d, want 1", len(router.Ports)) - } - - // === Phase 3: Create ENI port === - portEvt := PortEvent{ - NetworkInterfaceId: "eni-integ1", - SubnetId: "subnet-integ1", - VpcId: "vpc-integ1", - PrivateIpAddress: "10.0.1.10", - MacAddress: "02:00:00:aa:bb:01", - } - portData, _ := json.Marshal(portEvt) - resp, err = nc.Request(TopicCreatePort, portData, 5*time.Second) - if err != nil { - t.Fatalf("vpc.create-port: %v", err) - } - assertSuccess(t, resp, "create ENI port") - - // Verify OVN: logical switch port with correct addresses and DHCP - lsp, err := mock.GetLogicalSwitchPort(ctx, "port-eni-integ1") - if err != nil { - t.Fatalf("expected logical switch port: %v", err) - } - if lsp.Addresses[0] != "02:00:00:aa:bb:01 10.0.1.10" { - t.Errorf("port addresses = %v, want [02:00:00:aa:bb:01 10.0.1.10]", lsp.Addresses) - } - if lsp.DHCPv4Options == nil || *lsp.DHCPv4Options != dhcp.UUID { - t.Error("port should have DHCPv4Options set to subnet DHCP UUID") - } - - // Verify switch has 2 ports (router port + ENI port) - ls, _ = mock.GetLogicalSwitch(ctx, "subnet-subnet-integ1") - if len(ls.Ports) != 2 { - t.Errorf("switch ports = %d, want 2 (router + ENI)", len(ls.Ports)) - } - - // === Phase 4: Create and Attach Internet Gateway === - // Create VPC KV bucket and register test VPC so IGW ownership checks pass - js, err := nc.JetStream() - if err != nil { - t.Fatalf("JetStream: %v", err) - } - vpcKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketVPCs, History: 1}) - if err != nil { - t.Fatalf("create VPC KV: %v", err) - } - if _, err := vpcKV.Put(utils.AccountKey(testIGWAccountID, "vpc-integ1"), []byte(`{"vpc_id":"vpc-integ1","state":"available"}`)); err != nil { - t.Fatalf("register test VPC: %v", err) - } - - igwSvc, err := handlers_ec2_igw.NewIGWServiceImplWithNATS(nil, nc) - if err != nil { - t.Fatalf("create IGW service: %v", err) - } - - igwOut, err := igwSvc.CreateInternetGateway(&ec2.CreateInternetGatewayInput{ - TagSpecifications: []*ec2.TagSpecification{ - { - ResourceType: aws.String("internet-gateway"), - Tags: []*ec2.Tag{ - {Key: aws.String("Name"), Value: aws.String("integ-igw")}, - }, - }, - }, - }, testIGWAccountID) - if err != nil { - t.Fatalf("CreateInternetGateway: %v", err) - } - igwID := *igwOut.InternetGateway.InternetGatewayId - - // Attach IGW to VPC - _, err = igwSvc.AttachInternetGateway(&ec2.AttachInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - VpcId: aws.String("vpc-integ1"), - }, testIGWAccountID) - if err != nil { - t.Fatalf("AttachInternetGateway: %v", err) - } - - // Wait briefly for NATS event to be processed - time.Sleep(100 * time.Millisecond) - - // Verify OVN: external switch, SNAT, default route - _, err = mock.GetLogicalSwitch(ctx, "ext-vpc-integ1") - if err != nil { - t.Fatalf("expected external switch after IGW attach: %v", err) - } - - router, _ = mock.GetLogicalRouter(ctx, "vpc-vpc-integ1") - // Router should have 2 ports (subnet + gateway) - if len(router.Ports) != 2 { - t.Errorf("router ports = %d, want 2 (subnet + gateway)", len(router.Ports)) - } - // No blanket SNAT — only per-VM dnat_and_snat rules provide NAT (AWS parity) - if len(router.NAT) != 0 { - t.Errorf("router NAT rules = %d, want 0 (no blanket SNAT)", len(router.NAT)) - } - // Default route should exist - if len(router.StaticRoutes) != 1 { - t.Errorf("router static routes = %d, want 1", len(router.StaticRoutes)) - } - - // Verify IGW state via Describe - descOut, err := igwSvc.DescribeInternetGateways(&ec2.DescribeInternetGatewaysInput{ - InternetGatewayIds: []*string{aws.String(igwID)}, - }, testIGWAccountID) - if err != nil { - t.Fatalf("DescribeInternetGateways: %v", err) - } - if len(descOut.InternetGateways) != 1 { - t.Fatalf("expected 1 IGW, got %d", len(descOut.InternetGateways)) - } - if len(descOut.InternetGateways[0].Attachments) != 1 { - t.Fatal("expected IGW to have 1 attachment") - } - if *descOut.InternetGateways[0].Attachments[0].VpcId != "vpc-integ1" { - t.Errorf("IGW attachment VpcId = %s, want vpc-integ1", *descOut.InternetGateways[0].Attachments[0].VpcId) - } - - // === Phase 5: Verify Full Topology Summary === - switches, _ := mock.ListLogicalSwitches(ctx) - if len(switches) != 2 { - t.Errorf("total switches = %d, want 2 (subnet + external)", len(switches)) - } - routers, _ := mock.ListLogicalRouters(ctx) - if len(routers) != 1 { - t.Errorf("total routers = %d, want 1", len(routers)) - } - dhcpList, _ := mock.ListDHCPOptions(ctx) - if len(dhcpList) != 1 { - t.Errorf("total DHCP options = %d, want 1", len(dhcpList)) - } - - // === Phase 6: Detach IGW === - _, err = igwSvc.DetachInternetGateway(&ec2.DetachInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - VpcId: aws.String("vpc-integ1"), - }, testIGWAccountID) - if err != nil { - t.Fatalf("DetachInternetGateway: %v", err) - } - time.Sleep(100 * time.Millisecond) - - // External switch should be cleaned up - _, err = mock.GetLogicalSwitch(ctx, "ext-vpc-integ1") - if err == nil { - t.Error("expected external switch to be deleted after IGW detach") - } - - // Router should have 1 port (subnet only), no NAT, no routes - router, _ = mock.GetLogicalRouter(ctx, "vpc-vpc-integ1") - if len(router.Ports) != 1 { - t.Errorf("router ports after detach = %d, want 1", len(router.Ports)) - } - if len(router.NAT) != 0 { - t.Errorf("router NAT after detach = %d, want 0", len(router.NAT)) - } - if len(router.StaticRoutes) != 0 { - t.Errorf("router routes after detach = %d, want 0", len(router.StaticRoutes)) - } - - // === Phase 7: Delete IGW === - _, err = igwSvc.DeleteInternetGateway(&ec2.DeleteInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - }, testIGWAccountID) - if err != nil { - t.Fatalf("DeleteInternetGateway: %v", err) - } - - // === Phase 8: Clean up — delete ENI, subnet, VPC === - resp, _ = nc.Request(TopicDeletePort, portData, 5*time.Second) - assertSuccess(t, resp, "delete ENI port") - - resp, _ = nc.Request(TopicSubnetDelete, subData, 5*time.Second) - assertSuccess(t, resp, "delete subnet") - - resp, _ = nc.Request(TopicVPCDelete, vpcData, 5*time.Second) - assertSuccess(t, resp, "delete VPC") - - // === Phase 9: Verify Complete Cleanup === - switches, _ = mock.ListLogicalSwitches(ctx) - if len(switches) != 0 { - t.Errorf("expected 0 switches after cleanup, got %d", len(switches)) - } - routers, _ = mock.ListLogicalRouters(ctx) - if len(routers) != 0 { - t.Errorf("expected 0 routers after cleanup, got %d", len(routers)) - } - dhcpList, _ = mock.ListDHCPOptions(ctx) - if len(dhcpList) != 0 { - t.Errorf("expected 0 DHCP options after cleanup, got %d", len(dhcpList)) - } - - descOut, err = igwSvc.DescribeInternetGateways(&ec2.DescribeInternetGatewaysInput{}, testIGWAccountID) - if err != nil { - t.Fatalf("DescribeInternetGateways: %v", err) - } - if len(descOut.InternetGateways) != 0 { - t.Errorf("expected 0 IGWs after cleanup, got %d", len(descOut.InternetGateways)) - } -} - -// TestIntegration_MultiSubnetWithIGW tests a VPC with multiple subnets and IGW. -func TestIntegration_MultiSubnetWithIGW(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Create VPC - vpcEvt := VPCEvent{VpcId: "vpc-multi", CidrBlock: "10.0.0.0/16", VNI: 2000} - vpcData, _ := json.Marshal(vpcEvt) - resp, _ := nc.Request(TopicVPCCreate, vpcData, 5*time.Second) - assertSuccess(t, resp, "create VPC") - - // Create 3 subnets in the same VPC - subnets := []SubnetEvent{ - {SubnetId: "subnet-a", VpcId: "vpc-multi", CidrBlock: "10.0.1.0/24"}, - {SubnetId: "subnet-b", VpcId: "vpc-multi", CidrBlock: "10.0.2.0/24"}, - {SubnetId: "subnet-c", VpcId: "vpc-multi", CidrBlock: "10.0.3.0/24"}, - } - for _, sub := range subnets { - data, _ := json.Marshal(sub) - resp, _ = nc.Request(TopicSubnetCreate, data, 5*time.Second) - assertSuccess(t, resp, "create subnet "+sub.SubnetId) - } - - // Create ENI in each subnet - ports := []PortEvent{ - {NetworkInterfaceId: "eni-a1", SubnetId: "subnet-a", VpcId: "vpc-multi", PrivateIpAddress: "10.0.1.10", MacAddress: "02:00:00:01:01:01"}, - {NetworkInterfaceId: "eni-b1", SubnetId: "subnet-b", VpcId: "vpc-multi", PrivateIpAddress: "10.0.2.10", MacAddress: "02:00:00:02:02:02"}, - {NetworkInterfaceId: "eni-c1", SubnetId: "subnet-c", VpcId: "vpc-multi", PrivateIpAddress: "10.0.3.10", MacAddress: "02:00:00:03:03:03"}, - } - for _, port := range ports { - data, _ := json.Marshal(port) - resp, _ = nc.Request(TopicCreatePort, data, 5*time.Second) - assertSuccess(t, resp, "create port "+port.NetworkInterfaceId) - } - - // Verify: 1 router, 3 switches, 3 DHCP options - routerList, _ := mock.ListLogicalRouters(ctx) - if len(routerList) != 1 { - t.Errorf("routers = %d, want 1", len(routerList)) - } - switchList, _ := mock.ListLogicalSwitches(ctx) - if len(switchList) != 3 { - t.Errorf("switches = %d, want 3", len(switchList)) - } - dhcpList, _ := mock.ListDHCPOptions(ctx) - if len(dhcpList) != 3 { - t.Errorf("DHCP options = %d, want 3", len(dhcpList)) - } - - // Router should have 3 ports (one per subnet) - router, _ := mock.GetLogicalRouter(ctx, "vpc-vpc-multi") - if len(router.Ports) != 3 { - t.Errorf("router ports = %d, want 3", len(router.Ports)) - } - - // Create VPC KV bucket and register test VPC so IGW ownership checks pass - js, err := nc.JetStream() - if err != nil { - t.Fatalf("JetStream: %v", err) - } - vpcKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketVPCs, History: 1}) - if err != nil { - t.Fatalf("create VPC KV: %v", err) - } - if _, err := vpcKV.Put(utils.AccountKey(testIGWAccountID, "vpc-multi"), []byte(`{"vpc_id":"vpc-multi","state":"available"}`)); err != nil { - t.Fatalf("register test VPC: %v", err) - } - - // Attach IGW - igwSvc, _ := handlers_ec2_igw.NewIGWServiceImplWithNATS(nil, nc) - igwOut, _ := igwSvc.CreateInternetGateway(&ec2.CreateInternetGatewayInput{}, testIGWAccountID) - igwID := *igwOut.InternetGateway.InternetGatewayId - - _, err = igwSvc.AttachInternetGateway(&ec2.AttachInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - VpcId: aws.String("vpc-multi"), - }, testIGWAccountID) - if err != nil { - t.Fatalf("AttachInternetGateway: %v", err) - } - time.Sleep(100 * time.Millisecond) - - // Now: 4 switches (3 subnets + 1 external), router has 4 ports + NAT + route - switchList, _ = mock.ListLogicalSwitches(ctx) - if len(switchList) != 4 { - t.Errorf("switches after IGW = %d, want 4", len(switchList)) - } - router, _ = mock.GetLogicalRouter(ctx, "vpc-vpc-multi") - if len(router.Ports) != 4 { - t.Errorf("router ports after IGW = %d, want 4", len(router.Ports)) - } - if len(router.NAT) != 0 { - t.Errorf("router NAT = %d, want 0 (no blanket SNAT)", len(router.NAT)) - } - if len(router.StaticRoutes) != 1 { - t.Errorf("router routes = %d, want 1", len(router.StaticRoutes)) - } - - // === Cleanup: detach IGW, delete everything === - _, _ = igwSvc.DetachInternetGateway(&ec2.DetachInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - VpcId: aws.String("vpc-multi"), - }, testIGWAccountID) - time.Sleep(100 * time.Millisecond) - _, _ = igwSvc.DeleteInternetGateway(&ec2.DeleteInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - }, testIGWAccountID) - - // Delete ports - for _, port := range ports { - data, _ := json.Marshal(port) - resp, _ = nc.Request(TopicDeletePort, data, 5*time.Second) - assertSuccess(t, resp, "delete port "+port.NetworkInterfaceId) - } - - // Delete subnets - for _, sub := range subnets { - data, _ := json.Marshal(sub) - resp, _ = nc.Request(TopicSubnetDelete, data, 5*time.Second) - assertSuccess(t, resp, "delete subnet "+sub.SubnetId) - } - - // Delete VPC - resp, _ = nc.Request(TopicVPCDelete, vpcData, 5*time.Second) - assertSuccess(t, resp, "delete VPC") - - // Verify complete cleanup - routerList, _ = mock.ListLogicalRouters(ctx) - if len(routerList) != 0 { - t.Errorf("routers after cleanup = %d, want 0", len(routerList)) - } - switchList, _ = mock.ListLogicalSwitches(ctx) - if len(switchList) != 0 { - t.Errorf("switches after cleanup = %d, want 0", len(switchList)) - } -} - -// TestIntegration_IGWErrorPaths tests IGW error handling across the integration boundary. -func TestIntegration_IGWErrorPaths(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Create VPC KV bucket and register test VPCs so IGW ownership checks pass - js, err := nc.JetStream() - if err != nil { - t.Fatalf("JetStream: %v", err) - } - vpcKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketVPCs, History: 1}) - if err != nil { - t.Fatalf("create VPC KV: %v", err) - } - for _, vpcID := range []string{"vpc-err1", "vpc-err2"} { - if _, err := vpcKV.Put(utils.AccountKey(testIGWAccountID, vpcID), []byte(`{"vpc_id":"`+vpcID+`","state":"available"}`)); err != nil { - t.Fatalf("register test VPC: %v", err) - } - } - - igwSvc, err := handlers_ec2_igw.NewIGWServiceImplWithNATS(nil, nc) - if err != nil { - t.Fatalf("create IGW service: %v", err) - } - - // Create and attach IGW - igwOut, _ := igwSvc.CreateInternetGateway(&ec2.CreateInternetGatewayInput{}, testIGWAccountID) - igwID := *igwOut.InternetGateway.InternetGatewayId - - // Can't delete while attached - _, _ = igwSvc.AttachInternetGateway(&ec2.AttachInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - VpcId: aws.String("vpc-err1"), - }, testIGWAccountID) - - _, err = igwSvc.DeleteInternetGateway(&ec2.DeleteInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - }, testIGWAccountID) - if err == nil { - t.Error("expected DependencyViolation error") - } - - // Can't attach twice - _, err = igwSvc.AttachInternetGateway(&ec2.AttachInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - VpcId: aws.String("vpc-err2"), - }, testIGWAccountID) - if err == nil { - t.Error("expected ResourceAlreadyAssociated error") - } - - // Can't detach from wrong VPC - _, err = igwSvc.DetachInternetGateway(&ec2.DetachInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - VpcId: aws.String("vpc-wrong"), - }, testIGWAccountID) - if err == nil { - t.Error("expected GatewayNotAttached error") - } - - // Operations on nonexistent IGW - _, err = igwSvc.AttachInternetGateway(&ec2.AttachInternetGatewayInput{ - InternetGatewayId: aws.String("igw-doesnotexist"), - VpcId: aws.String("vpc-err1"), - }, testIGWAccountID) - if err == nil { - t.Error("expected NotFound error for attach") - } - - _, err = igwSvc.DetachInternetGateway(&ec2.DetachInternetGatewayInput{ - InternetGatewayId: aws.String("igw-doesnotexist"), - VpcId: aws.String("vpc-err1"), - }, testIGWAccountID) - if err == nil { - t.Error("expected NotFound error for detach") - } - - // Cleanup - _, _ = igwSvc.DetachInternetGateway(&ec2.DetachInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - VpcId: aws.String("vpc-err1"), - }, testIGWAccountID) - _, _ = igwSvc.DeleteInternetGateway(&ec2.DeleteInternetGatewayInput{ - InternetGatewayId: aws.String(igwID), - }, testIGWAccountID) -} diff --git a/spinifex/services/vpcd/mock_ovn.go b/spinifex/services/vpcd/mock_ovn.go deleted file mode 100644 index e0a9f0899..000000000 --- a/spinifex/services/vpcd/mock_ovn.go +++ /dev/null @@ -1,878 +0,0 @@ -package vpcd - -import ( - "context" - "fmt" - "slices" - "sync" - - "github.com/mulgadc/spinifex/spinifex/services/vpcd/nbdb" - "github.com/mulgadc/spinifex/spinifex/utils" -) - -// MockOVNClient implements OVNClient with in-memory storage for testing. -type MockOVNClient struct { - mu sync.Mutex - connected bool - - switches map[string]*nbdb.LogicalSwitch - ports map[string]*nbdb.LogicalSwitchPort - routers map[string]*nbdb.LogicalRouter - routerPorts map[string]*nbdb.LogicalRouterPort - dhcpOpts map[string]*nbdb.DHCPOptions - nats map[string]*nbdb.NAT // keyed by UUID - staticRoutes map[string]*nbdb.LogicalRouterStaticRoute // keyed by UUID - portGroups map[string]*nbdb.PortGroup // keyed by name - acls map[string]*nbdb.ACL // keyed by UUID - gatewayChassis map[string]*nbdb.GatewayChassis // keyed by UUID - - // UpdateLogicalSwitchPortCalls counts UpdateLogicalSwitchPort invocations - // so tests can assert idempotent read-before-write paths (e.g. - // ensureLocalnetOptions — mulga-998.b Fix 3). - UpdateLogicalSwitchPortCalls int - - // UpdateLogicalRouterPortCalls counts UpdateLogicalRouterPort invocations - // so tests can assert ensureGatewayPortNetworks only writes on drift - // (mulga-siv-26 D8). - UpdateLogicalRouterPortCalls int - - // SetGatewayChassisCalls / DeleteGatewayChassisCalls / - // UpdateGatewayChassisPriorityCalls let tests distinguish between - // "no-op", "create", "delete", and "priority update" paths through - // reconcileGatewayChassis (mulga-999). - SetGatewayChassisCalls int - DeleteGatewayChassisCalls int - UpdateGatewayChassisPriorityCalls int - - // AddACLErrAfter, when > 0, makes AddACL return an error on the Nth - // call (1-indexed). Lets fail-fast tests inject a transient OVN error - // at a specific point in the create/update flow without rewiring the - // whole client. Counter persists across calls until the mock is - // recreated. - AddACLErrAfter int - addACLCalls int -} - -var _ OVNClient = (*MockOVNClient)(nil) - -// NewMockOVNClient creates a new MockOVNClient for testing. -func NewMockOVNClient() *MockOVNClient { - return &MockOVNClient{ - switches: make(map[string]*nbdb.LogicalSwitch), - ports: make(map[string]*nbdb.LogicalSwitchPort), - routers: make(map[string]*nbdb.LogicalRouter), - routerPorts: make(map[string]*nbdb.LogicalRouterPort), - dhcpOpts: make(map[string]*nbdb.DHCPOptions), - nats: make(map[string]*nbdb.NAT), - staticRoutes: make(map[string]*nbdb.LogicalRouterStaticRoute), - portGroups: make(map[string]*nbdb.PortGroup), - acls: make(map[string]*nbdb.ACL), - gatewayChassis: make(map[string]*nbdb.GatewayChassis), - } -} - -func (m *MockOVNClient) Connect(_ context.Context) error { - m.mu.Lock() - defer m.mu.Unlock() - m.connected = true - return nil -} - -func (m *MockOVNClient) Close() { - m.mu.Lock() - defer m.mu.Unlock() - m.connected = false -} - -func (m *MockOVNClient) Connected() bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.connected -} - -// Logical Switch - -func (m *MockOVNClient) CreateLogicalSwitch(_ context.Context, ls *nbdb.LogicalSwitch) error { - m.mu.Lock() - defer m.mu.Unlock() - if _, exists := m.switches[ls.Name]; exists { - return fmt.Errorf("logical switch %q already exists", ls.Name) - } - if ls.UUID == "" { - ls.UUID = utils.GenerateResourceID("ovn") - } - stored := *ls - m.switches[ls.Name] = &stored - return nil -} - -func (m *MockOVNClient) DeleteLogicalSwitch(_ context.Context, name string) error { - m.mu.Lock() - defer m.mu.Unlock() - if _, exists := m.switches[name]; !exists { - return fmt.Errorf("logical switch %q not found", name) - } - delete(m.switches, name) - return nil -} - -func (m *MockOVNClient) GetLogicalSwitch(_ context.Context, name string) (*nbdb.LogicalSwitch, error) { - m.mu.Lock() - defer m.mu.Unlock() - ls, exists := m.switches[name] - if !exists { - return nil, fmt.Errorf("logical switch %q not found", name) - } - result := *ls - return &result, nil -} - -func (m *MockOVNClient) ListLogicalSwitches(_ context.Context) ([]nbdb.LogicalSwitch, error) { - m.mu.Lock() - defer m.mu.Unlock() - result := make([]nbdb.LogicalSwitch, 0, len(m.switches)) - for _, ls := range m.switches { - result = append(result, *ls) - } - return result, nil -} - -// Logical Switch Port - -func (m *MockOVNClient) CreateLogicalSwitchPort(_ context.Context, switchName string, lsp *nbdb.LogicalSwitchPort) error { - m.mu.Lock() - defer m.mu.Unlock() - ls, exists := m.switches[switchName] - if !exists { - return fmt.Errorf("logical switch %q not found", switchName) - } - if _, exists := m.ports[lsp.Name]; exists { - return fmt.Errorf("logical switch port %q already exists", lsp.Name) - } - if lsp.UUID == "" { - lsp.UUID = utils.GenerateResourceID("ovn") - } - stored := *lsp - m.ports[lsp.Name] = &stored - ls.Ports = append(ls.Ports, lsp.UUID) - return nil -} - -// CreateLogicalSwitchPortInGroups mirrors the live client's atomic create + -// port-group join path. The mock is not transactional, but every step still -// has to succeed up front; on a port-group lookup failure we leave no LSP -// behind so tests observe the same all-or-nothing semantics as production. -// SB Address_Set rows for `_ip4` / `_ip6` are auto-derived by -// ovn-northd in production and intentionally not modelled in the mock. -func (m *MockOVNClient) CreateLogicalSwitchPortInGroups(_ context.Context, switchName string, lsp *nbdb.LogicalSwitchPort, portGroupNames []string) error { - m.mu.Lock() - defer m.mu.Unlock() - ls, exists := m.switches[switchName] - if !exists { - return fmt.Errorf("logical switch %q not found", switchName) - } - if _, exists := m.ports[lsp.Name]; exists { - return fmt.Errorf("logical switch port %q already exists", lsp.Name) - } - for _, pgName := range portGroupNames { - if _, ok := m.portGroups[pgName]; !ok { - return fmt.Errorf("port group %q not found", pgName) - } - } - if lsp.UUID == "" { - lsp.UUID = utils.GenerateResourceID("ovn") - } - stored := *lsp - m.ports[lsp.Name] = &stored - ls.Ports = append(ls.Ports, lsp.UUID) - for _, pgName := range portGroupNames { - pg := m.portGroups[pgName] - if !slices.Contains(pg.Ports, lsp.UUID) { - pg.Ports = append(pg.Ports, lsp.UUID) - } - } - return nil -} - -func (m *MockOVNClient) DeleteLogicalSwitchPort(_ context.Context, switchName string, portName string) error { - m.mu.Lock() - defer m.mu.Unlock() - port, exists := m.ports[portName] - if !exists { - return fmt.Errorf("logical switch port %q not found", portName) - } - ls, exists := m.switches[switchName] - if !exists { - return fmt.Errorf("logical switch %q not found", switchName) - } - // Mirror libovsdb's reference-integrity rejection: an LSP still in any - // port group's Ports set cannot be deleted. Forces every delete-port - // path to clear membership first — handleDeletePort already does this - // via reconcilePortSGs, but locking the invariant in the mock catches - // any reordering that would only break against real OVN. - for pgName, pg := range m.portGroups { - if slices.Contains(pg.Ports, port.UUID) { - return fmt.Errorf("logical switch port %q still in port group %q", portName, pgName) - } - } - // Remove port UUID from switch's ports list - for i, uuid := range ls.Ports { - if uuid == port.UUID { - ls.Ports = append(ls.Ports[:i], ls.Ports[i+1:]...) - break - } - } - delete(m.ports, portName) - return nil -} - -func (m *MockOVNClient) GetLogicalSwitchPort(_ context.Context, name string) (*nbdb.LogicalSwitchPort, error) { - m.mu.Lock() - defer m.mu.Unlock() - lsp, exists := m.ports[name] - if !exists { - return nil, fmt.Errorf("logical switch port %q not found", name) - } - result := *lsp - return &result, nil -} - -func (m *MockOVNClient) UpdateLogicalSwitchPort(_ context.Context, lsp *nbdb.LogicalSwitchPort) error { - m.mu.Lock() - defer m.mu.Unlock() - if _, exists := m.ports[lsp.Name]; !exists { - return fmt.Errorf("logical switch port %q not found", lsp.Name) - } - stored := *lsp - m.ports[lsp.Name] = &stored - m.UpdateLogicalSwitchPortCalls++ - return nil -} - -// Logical Router - -func (m *MockOVNClient) CreateLogicalRouter(_ context.Context, lr *nbdb.LogicalRouter) error { - m.mu.Lock() - defer m.mu.Unlock() - if _, exists := m.routers[lr.Name]; exists { - return fmt.Errorf("logical router %q already exists", lr.Name) - } - if lr.UUID == "" { - lr.UUID = utils.GenerateResourceID("ovn") - } - stored := *lr - m.routers[lr.Name] = &stored - return nil -} - -func (m *MockOVNClient) DeleteLogicalRouter(_ context.Context, name string) error { - m.mu.Lock() - defer m.mu.Unlock() - if _, exists := m.routers[name]; !exists { - return fmt.Errorf("logical router %q not found", name) - } - delete(m.routers, name) - return nil -} - -func (m *MockOVNClient) GetLogicalRouter(_ context.Context, name string) (*nbdb.LogicalRouter, error) { - m.mu.Lock() - defer m.mu.Unlock() - lr, exists := m.routers[name] - if !exists { - return nil, fmt.Errorf("logical router %q not found", name) - } - result := *lr - return &result, nil -} - -func (m *MockOVNClient) ListLogicalRouters(_ context.Context) ([]nbdb.LogicalRouter, error) { - m.mu.Lock() - defer m.mu.Unlock() - result := make([]nbdb.LogicalRouter, 0, len(m.routers)) - for _, lr := range m.routers { - result = append(result, *lr) - } - return result, nil -} - -// Logical Router Port - -func (m *MockOVNClient) CreateLogicalRouterPort(_ context.Context, routerName string, lrp *nbdb.LogicalRouterPort) error { - m.mu.Lock() - defer m.mu.Unlock() - lr, exists := m.routers[routerName] - if !exists { - return fmt.Errorf("logical router %q not found", routerName) - } - if _, exists := m.routerPorts[lrp.Name]; exists { - return fmt.Errorf("logical router port %q already exists", lrp.Name) - } - if lrp.UUID == "" { - lrp.UUID = utils.GenerateResourceID("ovn") - } - stored := *lrp - m.routerPorts[lrp.Name] = &stored - lr.Ports = append(lr.Ports, lrp.UUID) - return nil -} - -func (m *MockOVNClient) DeleteLogicalRouterPort(_ context.Context, routerName string, portName string) error { - m.mu.Lock() - defer m.mu.Unlock() - port, exists := m.routerPorts[portName] - if !exists { - return fmt.Errorf("logical router port %q not found", portName) - } - lr, exists := m.routers[routerName] - if !exists { - return fmt.Errorf("logical router %q not found", routerName) - } - for i, uuid := range lr.Ports { - if uuid == port.UUID { - lr.Ports = append(lr.Ports[:i], lr.Ports[i+1:]...) - break - } - } - delete(m.routerPorts, portName) - return nil -} - -func (m *MockOVNClient) GetLogicalRouterPort(_ context.Context, name string) (*nbdb.LogicalRouterPort, error) { - m.mu.Lock() - defer m.mu.Unlock() - lrp, exists := m.routerPorts[name] - if !exists { - return nil, fmt.Errorf("logical router port %q not found", name) - } - result := *lrp - return &result, nil -} - -func (m *MockOVNClient) UpdateLogicalRouterPort(_ context.Context, lrp *nbdb.LogicalRouterPort) error { - m.mu.Lock() - defer m.mu.Unlock() - if _, exists := m.routerPorts[lrp.Name]; !exists { - return fmt.Errorf("logical router port %q not found", lrp.Name) - } - stored := *lrp - m.routerPorts[lrp.Name] = &stored - m.UpdateLogicalRouterPortCalls++ - return nil -} - -// DHCP Options - -func (m *MockOVNClient) CreateDHCPOptions(_ context.Context, opts *nbdb.DHCPOptions) (string, error) { - m.mu.Lock() - defer m.mu.Unlock() - if opts.UUID == "" { - opts.UUID = utils.GenerateResourceID("dhcp") - } - stored := *opts - m.dhcpOpts[opts.UUID] = &stored - return opts.UUID, nil -} - -func (m *MockOVNClient) DeleteDHCPOptions(_ context.Context, uuid string) error { - m.mu.Lock() - defer m.mu.Unlock() - if _, exists := m.dhcpOpts[uuid]; !exists { - return fmt.Errorf("DHCP options %q not found", uuid) - } - delete(m.dhcpOpts, uuid) - return nil -} - -func (m *MockOVNClient) FindDHCPOptionsByCIDR(_ context.Context, cidr string) (*nbdb.DHCPOptions, error) { - m.mu.Lock() - defer m.mu.Unlock() - for _, opts := range m.dhcpOpts { - if opts.CIDR == cidr { - result := *opts - return &result, nil - } - } - return nil, fmt.Errorf("DHCP options for CIDR %q not found", cidr) -} - -func (m *MockOVNClient) FindDHCPOptionsByExternalID(_ context.Context, key, value string) (*nbdb.DHCPOptions, error) { - m.mu.Lock() - defer m.mu.Unlock() - for _, opts := range m.dhcpOpts { - if opts.ExternalIDs[key] == value { - result := *opts - return &result, nil - } - } - return nil, fmt.Errorf("DHCP options with external_id %s=%s not found", key, value) -} - -func (m *MockOVNClient) ListDHCPOptions(_ context.Context) ([]nbdb.DHCPOptions, error) { - m.mu.Lock() - defer m.mu.Unlock() - result := make([]nbdb.DHCPOptions, 0, len(m.dhcpOpts)) - for _, opts := range m.dhcpOpts { - result = append(result, *opts) - } - return result, nil -} - -// NAT - -func (m *MockOVNClient) AddNAT(_ context.Context, routerName string, nat *nbdb.NAT) error { - m.mu.Lock() - defer m.mu.Unlock() - lr, exists := m.routers[routerName] - if !exists { - return fmt.Errorf("logical router %q not found", routerName) - } - if nat.UUID == "" { - nat.UUID = utils.GenerateResourceID("nat") - } - stored := *nat - m.nats[nat.UUID] = &stored - lr.NAT = append(lr.NAT, nat.UUID) - return nil -} - -func (m *MockOVNClient) DeleteNAT(_ context.Context, routerName string, natType, logicalIP string) error { - m.mu.Lock() - defer m.mu.Unlock() - lr, exists := m.routers[routerName] - if !exists { - return fmt.Errorf("logical router %q not found", routerName) - } - // Find the NAT entry - var foundUUID string - for uuid, n := range m.nats { - if n.Type == natType && n.LogicalIP == logicalIP { - foundUUID = uuid - break - } - } - if foundUUID == "" { - return fmt.Errorf("NAT %s %s: %w", natType, logicalIP, ErrNATNotFound) - } - // Remove from router's NAT list - for i, uuid := range lr.NAT { - if uuid == foundUUID { - lr.NAT = append(lr.NAT[:i], lr.NAT[i+1:]...) - break - } - } - delete(m.nats, foundUUID) - return nil -} - -func (m *MockOVNClient) DeleteNATByExternalIP(_ context.Context, routerName string, natType, externalIP string) error { - m.mu.Lock() - defer m.mu.Unlock() - lr, exists := m.routers[routerName] - if !exists { - return fmt.Errorf("logical router %q not found", routerName) - } - var foundUUIDs []string - for uuid, n := range m.nats { - if n.Type == natType && n.ExternalIP == externalIP { - foundUUIDs = append(foundUUIDs, uuid) - } - } - if len(foundUUIDs) == 0 { - return fmt.Errorf("NAT %s external_ip=%s: %w", natType, externalIP, ErrNATNotFound) - } - for _, foundUUID := range foundUUIDs { - for i, uuid := range lr.NAT { - if uuid == foundUUID { - lr.NAT = append(lr.NAT[:i], lr.NAT[i+1:]...) - break - } - } - delete(m.nats, foundUUID) - } - return nil -} - -func (m *MockOVNClient) DeleteAllNATsByExternalIP(_ context.Context, natType, externalIP string) (int, error) { - m.mu.Lock() - defer m.mu.Unlock() - - // Find all matching NAT rules. - var foundUUIDs []string - for uuid, n := range m.nats { - if n.Type == natType && n.ExternalIP == externalIP { - foundUUIDs = append(foundUUIDs, uuid) - } - } - if len(foundUUIDs) == 0 { - return 0, nil - } - - // Remove from all routers that reference them. - uuidSet := make(map[string]struct{}, len(foundUUIDs)) - for _, u := range foundUUIDs { - uuidSet[u] = struct{}{} - } - for _, lr := range m.routers { - filtered := lr.NAT[:0] - for _, uuid := range lr.NAT { - if _, stale := uuidSet[uuid]; !stale { - filtered = append(filtered, uuid) - } - } - lr.NAT = filtered - } - - // Delete the NAT rows. - for _, u := range foundUUIDs { - delete(m.nats, u) - } - return len(foundUUIDs), nil -} - -func (m *MockOVNClient) FindNATByExternalIP(_ context.Context, natType, externalIP string) (*nbdb.NAT, error) { - m.mu.Lock() - defer m.mu.Unlock() - for _, n := range m.nats { - if n.Type == natType && n.ExternalIP == externalIP { - return n, nil - } - } - return nil, nil -} - -// Static Routes - -func (m *MockOVNClient) AddStaticRoute(_ context.Context, routerName string, route *nbdb.LogicalRouterStaticRoute) error { - m.mu.Lock() - defer m.mu.Unlock() - lr, exists := m.routers[routerName] - if !exists { - return fmt.Errorf("logical router %q not found", routerName) - } - if route.UUID == "" { - route.UUID = utils.GenerateResourceID("route") - } - stored := *route - m.staticRoutes[route.UUID] = &stored - lr.StaticRoutes = append(lr.StaticRoutes, route.UUID) - return nil -} - -func (m *MockOVNClient) FindStaticRoute(_ context.Context, routerName, ipPrefix string) (*nbdb.LogicalRouterStaticRoute, error) { - m.mu.Lock() - defer m.mu.Unlock() - lr, exists := m.routers[routerName] - if !exists { - return nil, fmt.Errorf("logical router %q not found", routerName) - } - for _, uuid := range lr.StaticRoutes { - r, ok := m.staticRoutes[uuid] - if !ok { - continue - } - if r.IPPrefix == ipPrefix { - result := *r - return &result, nil - } - } - return nil, nil -} - -func (m *MockOVNClient) DeleteStaticRoute(_ context.Context, routerName string, ipPrefix string) error { - m.mu.Lock() - defer m.mu.Unlock() - lr, exists := m.routers[routerName] - if !exists { - return fmt.Errorf("logical router %q not found", routerName) - } - // Find the route - var foundUUID string - for uuid, r := range m.staticRoutes { - if r.IPPrefix == ipPrefix { - foundUUID = uuid - break - } - } - if foundUUID == "" { - return fmt.Errorf("static route %s not found", ipPrefix) - } - // Remove from router's StaticRoutes list - for i, uuid := range lr.StaticRoutes { - if uuid == foundUUID { - lr.StaticRoutes = append(lr.StaticRoutes[:i], lr.StaticRoutes[i+1:]...) - break - } - } - delete(m.staticRoutes, foundUUID) - return nil -} - -// Port Groups - -func (m *MockOVNClient) CreatePortGroup(_ context.Context, name string, ports []string) error { - m.mu.Lock() - defer m.mu.Unlock() - if _, exists := m.portGroups[name]; exists { - return fmt.Errorf("port group %q already exists", name) - } - pg := &nbdb.PortGroup{ - UUID: utils.GenerateResourceID("pg"), - Name: name, - Ports: ports, - } - m.portGroups[name] = pg - return nil -} - -func (m *MockOVNClient) DeletePortGroup(_ context.Context, name string) error { - m.mu.Lock() - defer m.mu.Unlock() - pg, exists := m.portGroups[name] - if !exists { - return fmt.Errorf("port group %q not found", name) - } - // Mirror libovsdb's reference-integrity rejection: a port group with - // referenced ACL rows cannot be deleted directly. Live OVN's - // `Where(pg).Delete()` only removes the PG row and would leak the ACLs; - // forcing every caller through ClearACLs first matches that contract. - if len(pg.ACLs) > 0 { - return fmt.Errorf("port group %q has %d ACLs still attached; clear ACLs before delete", name, len(pg.ACLs)) - } - delete(m.portGroups, name) - return nil -} - -// UpdatePortGroupMemberships applies all add/remove port-group joins for one -// LSP. SB Address_Set rows for `_ip4` / `_ip6` are auto-derived by -// ovn-northd in production and intentionally not modelled in the mock. -func (m *MockOVNClient) UpdatePortGroupMemberships(_ context.Context, lspName string, addPGs, removePGs []string) error { - m.mu.Lock() - defer m.mu.Unlock() - lsp, exists := m.ports[lspName] - if !exists { - return fmt.Errorf("logical switch port %q not found", lspName) - } - for _, pgName := range addPGs { - pg, exists := m.portGroups[pgName] - if !exists { - return fmt.Errorf("port group %q not found", pgName) - } - if !slices.Contains(pg.Ports, lsp.UUID) { - pg.Ports = append(pg.Ports, lsp.UUID) - } - } - for _, pgName := range removePGs { - pg, exists := m.portGroups[pgName] - if !exists { - return fmt.Errorf("port group %q not found", pgName) - } - for i, u := range pg.Ports { - if u == lsp.UUID { - pg.Ports = append(pg.Ports[:i], pg.Ports[i+1:]...) - break - } - } - } - return nil -} - -// ListPortGroupsForPort returns names of port groups whose Ports contains the -// given LSP's UUID. Mirrors the live client; the mock returns an empty slice -// (not an error) when the LSP exists but has no memberships, and an error -// only when the LSP itself is unknown. -func (m *MockOVNClient) ListPortGroupsForPort(_ context.Context, lspName string) ([]string, error) { - m.mu.Lock() - defer m.mu.Unlock() - lsp, exists := m.ports[lspName] - if !exists { - return nil, fmt.Errorf("logical switch port %q not found", lspName) - } - names := make([]string, 0) - for name, pg := range m.portGroups { - if slices.Contains(pg.Ports, lsp.UUID) { - names = append(names, name) - } - } - return names, nil -} - -// GetPortGroup returns the port group with the given name, or an error if it -// doesn't exist. Mirrors the live client. -func (m *MockOVNClient) GetPortGroup(_ context.Context, name string) (*nbdb.PortGroup, error) { - m.mu.Lock() - defer m.mu.Unlock() - pg, exists := m.portGroups[name] - if !exists { - return nil, fmt.Errorf("port group %q not found", name) - } - return pg, nil -} - -// ListPortGroups returns a snapshot of every port group currently in the mock -// store. The slice is freshly allocated; mutating it does not affect the mock. -func (m *MockOVNClient) ListPortGroups(_ context.Context) ([]nbdb.PortGroup, error) { - m.mu.Lock() - defer m.mu.Unlock() - out := make([]nbdb.PortGroup, 0, len(m.portGroups)) - for _, pg := range m.portGroups { - out = append(out, *pg) - } - return out, nil -} - -// ACLs - -func (m *MockOVNClient) AddACLs(_ context.Context, portGroupName string, specs []ACLSpec) error { - m.mu.Lock() - defer m.mu.Unlock() - m.addACLCalls++ - if m.AddACLErrAfter > 0 && m.addACLCalls == m.AddACLErrAfter { - return fmt.Errorf("injected AddACLs failure on call %d", m.addACLCalls) - } - pg, exists := m.portGroups[portGroupName] - if !exists { - return fmt.Errorf("port group %q not found", portGroupName) - } - for _, spec := range specs { - acl := &nbdb.ACL{ - UUID: utils.GenerateResourceID("acl"), - Direction: spec.Direction, - Priority: spec.Priority, - Match: spec.Match, - Action: spec.Action, - Log: spec.Log, - } - if spec.Name != "" { - name := spec.Name - acl.Name = &name - } - if spec.Severity != "" { - severity := spec.Severity - acl.Severity = &severity - } - m.acls[acl.UUID] = acl - pg.ACLs = append(pg.ACLs, acl.UUID) - } - return nil -} - -func (m *MockOVNClient) ClearACLs(_ context.Context, portGroupName string) error { - m.mu.Lock() - defer m.mu.Unlock() - pg, exists := m.portGroups[portGroupName] - if !exists { - return fmt.Errorf("port group %q not found", portGroupName) - } - for _, aclUUID := range pg.ACLs { - delete(m.acls, aclUUID) - } - pg.ACLs = nil - return nil -} - -// ListLogicalRouterPorts returns every LRP across the mock state. -func (m *MockOVNClient) ListLogicalRouterPorts(_ context.Context) ([]nbdb.LogicalRouterPort, error) { - m.mu.Lock() - defer m.mu.Unlock() - result := make([]nbdb.LogicalRouterPort, 0, len(m.routerPorts)) - for _, lrp := range m.routerPorts { - result = append(result, *lrp) - } - return result, nil -} - -// SetGatewayChassis is the idempotent read-then-decide path mirrored from -// LiveOVNClient. Tests rely on the mock applying the same "no-op when already -// correct" semantics so the call counters distinguish create vs. update vs. -// no-op (mulga-999). -func (m *MockOVNClient) SetGatewayChassis(_ context.Context, lrpName string, chassisName string, priority int) error { - m.mu.Lock() - defer m.mu.Unlock() - lrp, exists := m.routerPorts[lrpName] - if !exists { - return fmt.Errorf("logical router port %q not found", lrpName) - } - gcName := lrpName + "-" + chassisName - for _, gc := range m.gatewayChassis { - if gc.Name != gcName { - continue - } - if gc.Priority == priority { - return nil - } - gc.Priority = priority - m.UpdateGatewayChassisPriorityCalls++ - return nil - } - gc := &nbdb.GatewayChassis{ - UUID: utils.GenerateResourceID("gc"), - Name: gcName, - ChassisName: chassisName, - Priority: priority, - ExternalIDs: map[string]string{}, - Options: map[string]string{}, - } - m.gatewayChassis[gc.UUID] = gc - lrp.GatewayChassis = append(lrp.GatewayChassis, gc.UUID) - m.SetGatewayChassisCalls++ - return nil -} - -func (m *MockOVNClient) GetGatewayChassisByName(_ context.Context, name string) (*nbdb.GatewayChassis, error) { - m.mu.Lock() - defer m.mu.Unlock() - for _, gc := range m.gatewayChassis { - if gc.Name == name { - result := *gc - return &result, nil - } - } - return nil, nil -} - -func (m *MockOVNClient) ListGatewayChassis(_ context.Context) ([]nbdb.GatewayChassis, error) { - m.mu.Lock() - defer m.mu.Unlock() - result := make([]nbdb.GatewayChassis, 0, len(m.gatewayChassis)) - for _, gc := range m.gatewayChassis { - result = append(result, *gc) - } - return result, nil -} - -func (m *MockOVNClient) DeleteGatewayChassis(_ context.Context, lrpName string, gcUUID string) error { - m.mu.Lock() - defer m.mu.Unlock() - lrp, exists := m.routerPorts[lrpName] - if !exists { - return fmt.Errorf("logical router port %q not found", lrpName) - } - if _, ok := m.gatewayChassis[gcUUID]; !ok { - return fmt.Errorf("gateway_chassis %q not found", gcUUID) - } - for i, u := range lrp.GatewayChassis { - if u == gcUUID { - lrp.GatewayChassis = append(lrp.GatewayChassis[:i], lrp.GatewayChassis[i+1:]...) - break - } - } - delete(m.gatewayChassis, gcUUID) - m.DeleteGatewayChassisCalls++ - return nil -} - -// SeedGatewayChassis lets tests pre-populate a Gateway_Chassis row directly, -// bypassing the idempotent SetGatewayChassis path. Useful for setting up a -// "stale row" scenario for reconcileGatewayChassis tests (mulga-999). -func (m *MockOVNClient) SeedGatewayChassis(lrpName string, gc *nbdb.GatewayChassis) { - m.mu.Lock() - defer m.mu.Unlock() - if gc.UUID == "" { - gc.UUID = utils.GenerateResourceID("gc") - } - m.gatewayChassis[gc.UUID] = gc - if lrp, ok := m.routerPorts[lrpName]; ok { - lrp.GatewayChassis = append(lrp.GatewayChassis, gc.UUID) - } -} diff --git a/spinifex/services/vpcd/ovn_test.go b/spinifex/services/vpcd/ovn_test.go deleted file mode 100644 index 4132a3583..000000000 --- a/spinifex/services/vpcd/ovn_test.go +++ /dev/null @@ -1,168 +0,0 @@ -package vpcd - -import ( - "context" - "testing" - - "github.com/mulgadc/spinifex/spinifex/services/vpcd/nbdb" -) - -func TestNamedUUID(t *testing.T) { - tests := []struct { - name string - prefix string - input string - expected string - }{ - {"hyphen replacement", "sw_", "vpc-abc123", "sw_vpc_abc123"}, - {"slash and dot replacement", "rp_", "10.0.0.1/24", "rp_10_0_0_1_24"}, - {"no special characters", "lr_", "vpcabc", "lr_vpcabc"}, - {"empty name", "sw_", "", "sw_"}, - {"empty prefix", "", "vpc-1", "vpc_1"}, - {"both empty", "", "", ""}, - {"multiple consecutive hyphens", "x_", "a--b", "x_a__b"}, - {"mixed special chars", "p_", "a.b-c/d", "p_a_b_c_d"}, - // ACL match expressions embed these characters; OVSDB rejects unsanitised - // named-uuids with "Type mismatch for member 'uuid-name'", which would - // silently break every default-SG ACL transaction. - {"acl match at sign", "acl_", "outport == @pg-sg-XYZ && ip4", "acl_outport_____pg_sg_XYZ____ip4"}, - {"space replacement", "acl_", "a b c", "acl_a_b_c"}, - {"equals replacement", "acl_", "a==b", "acl_a__b"}, - {"ampersand replacement", "acl_", "a&&b", "acl_a__b"}, - {"colon replacement", "acl_", "ip6.src == fe80::1", "acl_ip6_src____fe80__1"}, - {"parens and quotes", "acl_", `ip4.src == "10.0.0.1"`, "acl_ip4_src_____10_0_0_1_"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := namedUUID(tt.prefix, tt.input) - if got != tt.expected { - t.Errorf("namedUUID(%q, %q) = %q, want %q", tt.prefix, tt.input, got, tt.expected) - } - }) - } -} - -func TestMockOVNClient_DeleteAllNATsByExternalIP(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - // Create two routers with NAT rules sharing the same external IP. - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-r1", - ExternalIDs: map[string]string{"spinifex:vpc_id": "r1"}, - }) - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-r2", - ExternalIDs: map[string]string{"spinifex:vpc_id": "r2"}, - }) - - _ = mock.AddNAT(ctx, "vpc-r1", &nbdb.NAT{ - Type: "dnat_and_snat", ExternalIP: "192.168.1.50", LogicalIP: "10.0.0.1", - }) - _ = mock.AddNAT(ctx, "vpc-r2", &nbdb.NAT{ - Type: "dnat_and_snat", ExternalIP: "192.168.1.50", LogicalIP: "10.200.0.1", - }) - // Unrelated NAT on r1 that should be untouched. - _ = mock.AddNAT(ctx, "vpc-r1", &nbdb.NAT{ - Type: "dnat_and_snat", ExternalIP: "192.168.1.99", LogicalIP: "10.0.0.2", - }) - - removed, err := mock.DeleteAllNATsByExternalIP(ctx, "dnat_and_snat", "192.168.1.50") - if err != nil { - t.Fatalf("DeleteAllNATsByExternalIP: %v", err) - } - if removed != 2 { - t.Errorf("expected 2 removed, got %d", removed) - } - - r1, _ := mock.GetLogicalRouter(ctx, "vpc-r1") - if len(r1.NAT) != 1 { - t.Errorf("r1 should retain 1 unrelated NAT, got %d", len(r1.NAT)) - } - - r2, _ := mock.GetLogicalRouter(ctx, "vpc-r2") - if len(r2.NAT) != 0 { - t.Errorf("r2 should have 0 NAT rules, got %d", len(r2.NAT)) - } - - // Deleting again should be a no-op. - removed, err = mock.DeleteAllNATsByExternalIP(ctx, "dnat_and_snat", "192.168.1.50") - if err != nil { - t.Fatalf("second DeleteAllNATsByExternalIP: %v", err) - } - if removed != 0 { - t.Errorf("expected 0 removed on second call, got %d", removed) - } -} - -// --- SetGatewayChassis idempotency (mulga-999) --- - -func TestSetGatewayChassis_Idempotent(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - if err := mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-A"}); err != nil { - t.Fatalf("CreateLogicalRouter: %v", err) - } - if err := mock.CreateLogicalRouterPort(ctx, "vpc-A", &nbdb.LogicalRouterPort{Name: "gw-A"}); err != nil { - t.Fatalf("CreateLogicalRouterPort: %v", err) - } - - if err := mock.SetGatewayChassis(ctx, "gw-A", "chassis-1", 20); err != nil { - t.Fatalf("SetGatewayChassis (1): %v", err) - } - if err := mock.SetGatewayChassis(ctx, "gw-A", "chassis-1", 20); err != nil { - t.Fatalf("SetGatewayChassis (2): %v", err) - } - - rows, err := mock.ListGatewayChassis(ctx) - if err != nil { - t.Fatalf("ListGatewayChassis: %v", err) - } - if len(rows) != 1 { - t.Fatalf("expected 1 row, got %d (%v)", len(rows), rows) - } - if mock.SetGatewayChassisCalls != 1 { - t.Errorf("expected 1 create call, got %d", mock.SetGatewayChassisCalls) - } - if mock.UpdateGatewayChassisPriorityCalls != 0 { - t.Errorf("expected 0 priority updates, got %d", mock.UpdateGatewayChassisPriorityCalls) - } -} - -func TestSetGatewayChassis_UpdatesPriority(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - if err := mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-A"}); err != nil { - t.Fatalf("CreateLogicalRouter: %v", err) - } - if err := mock.CreateLogicalRouterPort(ctx, "vpc-A", &nbdb.LogicalRouterPort{Name: "gw-A"}); err != nil { - t.Fatalf("CreateLogicalRouterPort: %v", err) - } - - if err := mock.SetGatewayChassis(ctx, "gw-A", "chassis-1", 20); err != nil { - t.Fatalf("SetGatewayChassis at 20: %v", err) - } - if err := mock.SetGatewayChassis(ctx, "gw-A", "chassis-1", 15); err != nil { - t.Fatalf("SetGatewayChassis at 15: %v", err) - } - - rows, err := mock.ListGatewayChassis(ctx) - if err != nil { - t.Fatalf("ListGatewayChassis: %v", err) - } - if len(rows) != 1 { - t.Fatalf("expected 1 row (mutated, not duplicated), got %d", len(rows)) - } - if rows[0].Priority != 15 { - t.Errorf("expected priority 15 after update, got %d", rows[0].Priority) - } - if mock.UpdateGatewayChassisPriorityCalls != 1 { - t.Errorf("expected 1 priority update, got %d", mock.UpdateGatewayChassisPriorityCalls) - } -} diff --git a/spinifex/services/vpcd/reconcile.go b/spinifex/services/vpcd/reconcile.go deleted file mode 100644 index b1618cde1..000000000 --- a/spinifex/services/vpcd/reconcile.go +++ /dev/null @@ -1,451 +0,0 @@ -package vpcd - -import ( - "context" - "encoding/json" - "errors" - "log/slog" - "time" - - handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip" - handlers_ec2_igw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/igw" - handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" - "github.com/mulgadc/spinifex/spinifex/services/vpcd/nbdb" - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" -) - -// reconcileLeaderBucket holds a single CAS-elected leader key. MaxAge bounds -// recovery time when an elected vpcd crashes before releasing the lock. -const ( - reconcileLeaderBucket = "spinifex-vpcd-reconcile" - reconcileLeaderKey = "leader" - reconcileLeaderTTL = 60 * time.Second -) - -// Bounded wait for JetStream to come up at vpcd startup. On a cold multi-node -// start NATS clustering takes a few seconds and js.CreateKeyValue returns -// "context deadline exceeded" until quorum forms. Retrying makes the election -// deterministic; without it every vpcd falls through and races the same OVN -// NB transactions (mulga-js-72). Vars (not consts) so tests can shrink them. -var ( - reconcileLeaderRetryFor = 60 * time.Second - reconcileLeaderRetryStep = 1 * time.Second -) - -// AcquireReconcileLeader returns release+true exactly once across all vpcds in -// a cluster, gating the startup Reconcile/ReconcileFromKV passes. Other vpcds -// get release=nil, elected=false and skip reconcile (runtime VPC events use the -// vpcd-workers queue group, so they remain handled cluster-wide). -// -// Bounded-retry contract: when JetStream KV is not yet reachable (cold -// multi-node start, NATS quorum still forming) we retry for -// reconcileLeaderRetryFor before giving up. On exhaustion we return -// elected=false rather than running unguarded — concurrent reconciles produce -// silent OVN NB damage (mulga-js-72). Operators can restart vpcd to retry -// reconcile once NATS recovers. -func AcquireReconcileLeader(nc *nats.Conn, holder string) (func(), bool) { - js, _ := nc.JetStream() - - var ( - kv nats.KeyValue - err error - ) - deadline := time.Now().Add(reconcileLeaderRetryFor) - for { - kv, err = js.CreateKeyValue(&nats.KeyValueConfig{ - Bucket: reconcileLeaderBucket, - History: 1, - TTL: reconcileLeaderTTL, - }) - if err == nil { - break - } - if time.Now().After(deadline) { - slog.Error("vpcd reconcile-leader: JetStream KV unreachable after retry, skipping reconcile", - "holder", holder, "waited", reconcileLeaderRetryFor, "err", err) - return nil, false - } - slog.Debug("vpcd reconcile-leader: JetStream KV not ready, retrying", - "holder", holder, "err", err) - time.Sleep(reconcileLeaderRetryStep) - } - - if _, err := kv.Create(reconcileLeaderKey, []byte(holder)); err != nil { - slog.Info("vpcd reconcile-leader: another vpcd is leader, skipping reconcile", "holder", holder, "err", err) - return nil, false - } - - slog.Info("vpcd reconcile-leader: elected", "holder", holder) - return func() { - if err := kv.Delete(reconcileLeaderKey); err != nil { - slog.Warn("vpcd reconcile-leader: failed to release lock (TTL will reap)", "holder", holder, "err", err) - } - }, true -} - -// ReconcileResult tracks what was created during reconciliation. -// -// IGWsReconciled has "called" semantics rather than "created" because -// reconcileIGW is self-healing and idempotent — every successful invocation -// increments, regardless of whether new entities were added. Gauging "what -// changed" requires inspecting OVN directly. -type ReconcileResult struct { - RoutersCreated int - SwitchesCreated int - IGWsReconciled int - PortsCreated int - NATsReconciled int -} - -// Reconcile ensures OVN topology matches the expected state from the bootstrap config. -// This runs on vpcd startup before subscribing to NATS topics. -// -// Pass 1 (bootstrap): Uses [bootstrap] from spinifex.toml to create the default VPC -// topology. This covers first-install where admin init ran before services started. -// -// All operations are idempotent — safe to call on every startup. -func Reconcile(ctx context.Context, topo *TopologyHandler, bootstrap *BootstrapVPC) ReconcileResult { - var result ReconcileResult - - if bootstrap == nil || bootstrap.VpcId == "" { - slog.Debug("vpcd reconcile: no bootstrap config, skipping") - return result - } - - slog.Info("vpcd reconcile: checking bootstrap VPC topology", - "vpc_id", bootstrap.VpcId, - "subnet_id", bootstrap.SubnetId, - ) - - // 1. Ensure VPC router exists - routerName := "vpc-" + bootstrap.VpcId - if _, err := topo.ovn.GetLogicalRouter(ctx, routerName); err != nil { - slog.Info("vpcd reconcile: creating VPC router", "router", routerName) - if err := topo.reconcileVPC(ctx, bootstrap.VpcId, bootstrap.Cidr); err != nil { - slog.Error("vpcd reconcile: failed to create VPC router", "err", err) - } else { - result.RoutersCreated++ - } - } else { - slog.Debug("vpcd reconcile: VPC router exists", "router", routerName) - } - - // 2. Ensure subnet switch + router port + DHCP exists - if bootstrap.SubnetId != "" { - switchName := "subnet-" + bootstrap.SubnetId - if _, err := topo.ovn.GetLogicalSwitch(ctx, switchName); err != nil { - slog.Info("vpcd reconcile: creating subnet topology", "switch", switchName) - if err := topo.reconcileSubnet(ctx, bootstrap.SubnetId, bootstrap.VpcId, bootstrap.SubnetCidr); err != nil { - slog.Error("vpcd reconcile: failed to create subnet topology", "err", err) - } else { - result.SwitchesCreated++ - } - } else { - slog.Debug("vpcd reconcile: subnet switch exists", "switch", switchName) - } - } - - // 3. Ensure IGW topology. reconcileIGW is ensure-then-skip per child, so - // call it unconditionally — gating on "ext switch missing" trapped - // partial topologies (switch + LRP present, no localnet LSP) in an - // unhealable state across vpcd restarts. - if err := topo.reconcileIGW(ctx, bootstrap.VpcId, bootstrap.IgwId); err != nil { - slog.Error("vpcd reconcile: failed to ensure IGW topology", "err", err) - } else { - result.IGWsReconciled++ - } - - slog.Info("vpcd reconcile: complete", - "routers_created", result.RoutersCreated, - "switches_created", result.SwitchesCreated, - "igws_reconciled", result.IGWsReconciled, - ) - - return result -} - -// ReconcileFromKV reads all VPCs, subnets, IGW attachments, and ENIs from NATS KV -// and ensures OVN topology matches. This is Pass 2 from the Phase 12 plan — it -// handles reboots, OVN DB loss, and any missed events. All operations are idempotent. -// -// chassisNames is the set of OVN chassis names discovered from the SBDB at -// startup; reconcileGatewayChassis uses it to delete stale Gateway_Chassis -// rows (e.g. left over from a chassis_name change across reboot) and rebind -// every gateway LRP against the live chassis (mulga-999). -func ReconcileFromKV(ctx context.Context, nc *nats.Conn, topo *TopologyHandler, chassisNames []string) ReconcileResult { - var result ReconcileResult - - // 0. Reconcile gateway_chassis bindings before any per-VPC work. State - // drift here (a gateway_chassis row pointing at a chassis that no longer - // exists) leaves the cr-gw* port unbound — centralized NAT silently - // breaks. Single SB-derived list, applied once. - if err := topo.reconcileGatewayChassis(ctx, chassisNames); err != nil { - slog.Warn("vpcd reconcile-kv: gateway_chassis reconcile failed", "err", err) - } - - js, err := nc.JetStream() - if err != nil { - slog.Warn("vpcd reconcile-kv: failed to get JetStream context", "err", err) - return result - } - - // 1. Reconcile VPCs: ensure each VPC has a logical router - vpcKV, err := js.KeyValue(handlers_ec2_vpc.KVBucketVPCs) - if err != nil { - slog.Debug("vpcd reconcile-kv: VPC KV bucket not available (first boot before daemon?)", "err", err) - return result - } - - // Build a map of VPC ID → CIDR for subnet/IGW reconciliation - type vpcInfo struct { - VpcId string - CidrBlock string - } - vpcMap := make(map[string]vpcInfo) - - vpcKeys, err := vpcKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { - slog.Warn("vpcd reconcile-kv: failed to list VPC keys", "err", err) - } - for _, key := range vpcKeys { - if key == utils.VersionKey { - continue - } - entry, err := vpcKV.Get(key) - if err != nil { - continue - } - var rec handlers_ec2_vpc.VPCRecord - if err := json.Unmarshal(entry.Value(), &rec); err != nil { - slog.Warn("vpcd reconcile-kv: failed to unmarshal VPC record", "key", key, "err", err) - continue - } - vpcMap[rec.VpcId] = vpcInfo{VpcId: rec.VpcId, CidrBlock: rec.CidrBlock} - - routerName := "vpc-" + rec.VpcId - if _, err := topo.ovn.GetLogicalRouter(ctx, routerName); err != nil { - slog.Info("vpcd reconcile-kv: creating VPC router", "router", routerName) - if err := topo.reconcileVPC(ctx, rec.VpcId, rec.CidrBlock); err != nil { - slog.Error("vpcd reconcile-kv: failed to create VPC router", "err", err) - } else { - result.RoutersCreated++ - } - } - } - - // 2. Reconcile subnets: ensure each subnet has a logical switch + router port + DHCP - subnetKV, err := js.KeyValue(handlers_ec2_vpc.KVBucketSubnets) - if err != nil { - slog.Debug("vpcd reconcile-kv: subnet KV bucket not available", "err", err) - } else { - subnetKeys, err := subnetKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { - slog.Warn("vpcd reconcile-kv: failed to list subnet keys", "err", err) - } - for _, key := range subnetKeys { - if key == utils.VersionKey { - continue - } - entry, err := subnetKV.Get(key) - if err != nil { - continue - } - var rec handlers_ec2_vpc.SubnetRecord - if err := json.Unmarshal(entry.Value(), &rec); err != nil { - slog.Warn("vpcd reconcile-kv: failed to unmarshal subnet record", "key", key, "err", err) - continue - } - - switchName := "subnet-" + rec.SubnetId - if _, err := topo.ovn.GetLogicalSwitch(ctx, switchName); err != nil { - slog.Info("vpcd reconcile-kv: creating subnet topology", "switch", switchName) - if err := topo.reconcileSubnet(ctx, rec.SubnetId, rec.VpcId, rec.CidrBlock); err != nil { - slog.Error("vpcd reconcile-kv: failed to create subnet topology", "err", err) - } else { - result.SwitchesCreated++ - } - } - } - } - - // 3. Reconcile IGW attachments: ensure attached IGWs have external switch + SNAT - igwKV, err := js.KeyValue(handlers_ec2_igw.KVBucketIGW) - if err != nil { - slog.Debug("vpcd reconcile-kv: IGW KV bucket not available", "err", err) - } else { - igwKeys, err := igwKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { - slog.Warn("vpcd reconcile-kv: failed to list IGW keys", "err", err) - } - for _, key := range igwKeys { - if key == utils.VersionKey { - continue - } - entry, err := igwKV.Get(key) - if err != nil { - continue - } - var rec handlers_ec2_igw.IGWRecord - if err := json.Unmarshal(entry.Value(), &rec); err != nil { - slog.Warn("vpcd reconcile-kv: failed to unmarshal IGW record", "key", key, "err", err) - continue - } - // Only reconcile attached IGWs - if rec.VpcId == "" || rec.State != "attached" { - continue - } - - if err := topo.reconcileIGW(ctx, rec.VpcId, rec.InternetGatewayId); err != nil { - slog.Error("vpcd reconcile-kv: failed to ensure IGW topology", "err", err, "igw_id", rec.InternetGatewayId) - } else { - result.IGWsReconciled++ - } - } - } - - // 4. Reconcile ENI ports: ensure each ENI has a logical switch port. - // eniMAC is populated here and consumed by Step 5 to build distributed NAT - // rules (direct bridge mode only; centralized mode doesn't need the MAC). - eniMAC := make(map[string]string) // eniID → MAC - eniKV, err := js.KeyValue(handlers_ec2_vpc.KVBucketENIs) - if err != nil { - slog.Debug("vpcd reconcile-kv: ENI KV bucket not available", "err", err) - } else { - eniKeys, err := eniKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { - slog.Warn("vpcd reconcile-kv: failed to list ENI keys", "err", err) - } - for _, key := range eniKeys { - if key == utils.VersionKey { - continue - } - entry, err := eniKV.Get(key) - if err != nil { - continue - } - var rec handlers_ec2_vpc.ENIRecord - if err := json.Unmarshal(entry.Value(), &rec); err != nil { - slog.Warn("vpcd reconcile-kv: failed to unmarshal ENI record", "key", key, "err", err) - continue - } - - eniMAC[rec.NetworkInterfaceId] = rec.MacAddress - - portName := "port-" + rec.NetworkInterfaceId - if _, err := topo.ovn.GetLogicalSwitchPort(ctx, portName); err != nil { - switchName := "subnet-" + rec.SubnetId - addrStr := rec.MacAddress + " " + rec.PrivateIpAddress - lsp := &nbdb.LogicalSwitchPort{ - Name: portName, - Addresses: []string{addrStr}, - PortSecurity: []string{addrStr}, - ExternalIDs: map[string]string{ - "spinifex:eni_id": rec.NetworkInterfaceId, - "spinifex:subnet_id": rec.SubnetId, - "spinifex:vpc_id": rec.VpcId, - }, - } - - // Attach DHCP options if available - dhcpOpts, dhcpErr := topo.ovn.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", rec.SubnetId) - if dhcpErr == nil { - lsp.DHCPv4Options = &dhcpOpts.UUID - } - - slog.Info("vpcd reconcile-kv: creating ENI port", "port", portName, "switch", switchName) - if err := topo.ovn.CreateLogicalSwitchPort(ctx, switchName, lsp); err != nil { - slog.Error("vpcd reconcile-kv: failed to create ENI port", "port", portName, "err", err) - } else { - result.PortsCreated++ - } - } - } - } - - // 5. Reconcile EIP NAT rules: ensure each associated EIP has a dnat_and_snat - // rule in OVN NB DB. NAT rules are only created by AssociateAddress events - // and are never otherwise reconciled — an OVN NB DB wipe or any state drift - // silently breaks inbound EIP traffic. This step is the safety net. - eipKV, err := js.KeyValue(handlers_ec2_eip.KVBucketEIPs) - if err != nil { - slog.Debug("vpcd reconcile-kv: EIP KV bucket not available", "err", err) - } else { - eipKeys, err := eipKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { - slog.Warn("vpcd reconcile-kv: failed to list EIP keys", "err", err) - } - for _, key := range eipKeys { - if key == utils.VersionKey { - continue - } - entry, err := eipKV.Get(key) - if err != nil { - continue - } - var rec handlers_ec2_eip.EIPRecord - if err := json.Unmarshal(entry.Value(), &rec); err != nil { - slog.Warn("vpcd reconcile-kv: failed to unmarshal EIP record", "key", key, "err", err) - continue - } - if rec.State != "associated" || rec.VpcId == "" || rec.PublicIp == "" || rec.PrivateIp == "" { - continue - } - - existing, err := topo.ovn.FindNATByExternalIP(ctx, "dnat_and_snat", rec.PublicIp) - if err != nil { - slog.Warn("vpcd reconcile-kv: failed to query NAT rule", "external_ip", rec.PublicIp, "err", err) - continue - } - if existing != nil && existing.LogicalIP == rec.PrivateIp { - slog.Debug("vpcd reconcile-kv: NAT rule correct, skipping", "external_ip", rec.PublicIp) - continue - } - - routerName := "vpc-" + rec.VpcId - natRule := &nbdb.NAT{ - Type: "dnat_and_snat", - ExternalIP: rec.PublicIp, - LogicalIP: rec.PrivateIp, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": rec.VpcId, - "spinifex:public_ip": rec.PublicIp, - }, - } - if !topo.useCentralizedNAT() && rec.ENIId != "" { - portName := "port-" + rec.ENIId - natRule.LogicalPort = &portName - if mac, ok := eniMAC[rec.ENIId]; ok && mac != "" { - natRule.ExternalMAC = &mac - } - } - - if existing != nil { - if removed, err := topo.ovn.DeleteAllNATsByExternalIP(ctx, "dnat_and_snat", rec.PublicIp); err != nil { - slog.Warn("vpcd reconcile-kv: failed to remove stale NAT rule", "external_ip", rec.PublicIp, "err", err) - continue - } else if removed > 0 { - slog.Info("vpcd reconcile-kv: removed stale NAT rule before re-add", "external_ip", rec.PublicIp, "removed", removed) - } - } - - if err := topo.ovn.AddNAT(ctx, routerName, natRule); err != nil { - slog.Error("vpcd reconcile-kv: failed to add NAT rule", "external_ip", rec.PublicIp, "logical_ip", rec.PrivateIp, "err", err) - continue - } - slog.Info("vpcd reconcile-kv: reconciled NAT rule", - "external_ip", rec.PublicIp, "logical_ip", rec.PrivateIp, "router", routerName) - result.NATsReconciled++ - } - } - - slog.Info("vpcd reconcile-kv: complete", - "routers_created", result.RoutersCreated, - "switches_created", result.SwitchesCreated, - "igws_reconciled", result.IGWsReconciled, - "ports_created", result.PortsCreated, - "nats_reconciled", result.NATsReconciled, - ) - - return result -} diff --git a/spinifex/services/vpcd/reconcile_sgs.go b/spinifex/services/vpcd/reconcile_sgs.go deleted file mode 100644 index 3566faabf..000000000 --- a/spinifex/services/vpcd/reconcile_sgs.go +++ /dev/null @@ -1,256 +0,0 @@ -package vpcd - -import ( - "context" - "encoding/json" - "errors" - "log/slog" - "os" - "strings" - "time" - - handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" -) - -// reconcileSGsLoopInterval is how often the reconciler walks KV + OVN to -// converge drift. Chosen so a vpcd that wins AcquireReconcileLeader (60s TTL) -// typically holds the bucket key across at least one cycle. -const reconcileSGsLoopInterval = 30 * time.Second - -// SGReconcileResult tracks what each periodic pass converged. -type SGReconcileResult struct { - PortGroupsRecreated int - PortMembershipsSynced int - OrphanPortGroupsRemoved int -} - -// ReconcileSGsLoop runs the SG/ENI convergence scans every 30s, gated on -// AcquireReconcileLeader so only one vpcd in the cluster runs them at a time. -// Returns when ctx is cancelled. -func ReconcileSGsLoop(ctx context.Context, nc *nats.Conn, topo *TopologyHandler) { - holder, _ := os.Hostname() - ticker := time.NewTicker(reconcileSGsLoopInterval) - defer ticker.Stop() - - for { - release, elected := AcquireReconcileLeader(nc, holder) - if elected { - result := ReconcileSGsOnce(ctx, nc, topo) - if result.PortGroupsRecreated+result.PortMembershipsSynced+result.OrphanPortGroupsRemoved > 0 { - slog.Info("vpcd reconcile-sgs: converged drift", - "port_groups_recreated", result.PortGroupsRecreated, - "port_memberships_synced", result.PortMembershipsSynced, - "orphan_port_groups_removed", result.OrphanPortGroupsRemoved, - ) - } - release() - } - - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - } -} - -// ReconcileSGsOnce runs a single pass. Exported so tests can drive it directly. -func ReconcileSGsOnce(ctx context.Context, nc *nats.Conn, topo *TopologyHandler) SGReconcileResult { - var result SGReconcileResult - if topo == nil || topo.ovn == nil { - slog.Warn("vpcd reconcile-sgs: skipped — topology handler not connected") - return result - } - - js, err := nc.JetStream() - if err != nil { - slog.Warn("vpcd reconcile-sgs: failed to get JetStream context", "err", err) - return result - } - - sgKV, err := js.KeyValue(handlers_ec2_vpc.KVBucketSecurityGroups) - if err != nil { - slog.Debug("vpcd reconcile-sgs: SG bucket not available", "err", err) - sgKV = nil - } - eniKV, err := js.KeyValue(handlers_ec2_vpc.KVBucketENIs) - if err != nil { - slog.Debug("vpcd reconcile-sgs: ENI bucket not available", "err", err) - eniKV = nil - } - - var sgs []handlers_ec2_vpc.SecurityGroupRecord - if sgKV != nil { - sgs = listSGRecords(sgKV) - } - - // SG records without an OVN port group → recreate. - if sgKV != nil { - result.PortGroupsRecreated = scanMissingPortGroups(ctx, topo, sgs) - } - - // OVN port groups with no matching SG record → tear down. Re-snapshot KV - // right before the orphan check: scanMissingPortGroups may have taken - // seconds on a large cluster, and any CreateSecurityGroup that landed in - // the meantime would otherwise look like an orphan against the stale - // snapshot and get silently deleted. - if sgKV != nil { - sgs = listSGRecords(sgKV) - } - result.OrphanPortGroupsRemoved = scanOrphanPortGroups(ctx, topo, sgs) - - // ENIs with SecurityGroupIds → reconcile membership. - if eniKV != nil { - result.PortMembershipsSynced = scanENIPortMembership(ctx, topo, eniKV) - } - - return result -} - -// listSGRecords reads every SG record from the bucket. Logs and skips -// malformed entries; the orphan-SG case can't occur on a healthy cluster. -func listSGRecords(sgKV nats.KeyValue) []handlers_ec2_vpc.SecurityGroupRecord { - keys, err := sgKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { - slog.Warn("vpcd reconcile-sgs: failed to list SG keys", "err", err) - return nil - } - var out []handlers_ec2_vpc.SecurityGroupRecord - for _, key := range keys { - if key == utils.VersionKey { - continue - } - entry, err := sgKV.Get(key) - if err != nil { - slog.Warn("vpcd reconcile-sgs: SG read failed", "key", key, "err", err) - continue - } - var rec handlers_ec2_vpc.SecurityGroupRecord - if err := json.Unmarshal(entry.Value(), &rec); err != nil { - slog.Warn("vpcd reconcile-sgs: SG unmarshal failed", "key", key, "err", err) - continue - } - out = append(out, rec) - } - return out -} - -// scanMissingPortGroups recreates the OVN port group + ACLs for any SG record -// whose port group has gone missing in OVN NB. Returns the count recreated. -func scanMissingPortGroups(ctx context.Context, topo *TopologyHandler, sgs []handlers_ec2_vpc.SecurityGroupRecord) int { - recreated := 0 - for _, sg := range sgs { - pgName := portGroupName(sg.GroupId) - if _, err := topo.ovn.GetPortGroup(ctx, pgName); err == nil { - continue - } - if err := topo.provisionSG(ctx, sg.GroupId, sgRulesToACLRules(sg.IngressRules), sgRulesToACLRules(sg.EgressRules)); err != nil { - slog.Warn("vpcd reconcile-sgs: failed to recreate port group for SG", "sg", sg.GroupId, "err", err) - continue - } - slog.Info("vpcd reconcile-sgs: recreated missing port group", "sg", sg.GroupId, "pg", pgName) - recreated++ - } - return recreated -} - -// scanENIPortMembership runs reconcilePortSGs for every ENI with a non-empty -// SecurityGroupIds list. Idempotent. Counts only ENIs whose membership -// actually changed — a converged ENI scanned with no drift is a no-op and -// must not appear in the converged-drift log line. -func scanENIPortMembership(ctx context.Context, topo *TopologyHandler, eniKV nats.KeyValue) int { - keys, err := eniKV.Keys() - if err != nil && !errors.Is(err, nats.ErrNoKeysFound) { - slog.Warn("vpcd reconcile-sgs: failed to list ENI keys", "err", err) - return 0 - } - synced := 0 - for _, key := range keys { - if key == utils.VersionKey { - continue - } - entry, err := eniKV.Get(key) - if err != nil { - slog.Warn("vpcd reconcile-sgs: ENI read failed", "key", key, "err", err) - continue - } - var rec handlers_ec2_vpc.ENIRecord - if err := json.Unmarshal(entry.Value(), &rec); err != nil { - slog.Warn("vpcd reconcile-sgs: ENI unmarshal failed", "key", key, "err", err) - continue - } - if len(rec.SecurityGroupIds) == 0 { - continue - } - portName := "port-" + rec.NetworkInterfaceId - changed, err := topo.reconcilePortSGs(ctx, portName, rec.SecurityGroupIds) - if err != nil { - // Most often: the LSP itself doesn't exist yet. Warn-and-continue. - slog.Warn("vpcd reconcile-sgs: port SG reconcile failed", "eni", rec.NetworkInterfaceId, "err", err) - continue - } - if changed { - synced++ - } - } - return synced -} - -// scanOrphanPortGroups removes any spinifex-managed port group (`sg_*`) in OVN -// that has no matching SG record in KV. Mirrors handleDeleteSG's teardown -// (ClearACLs → DeletePortGroup). The matching `_ip4` / `_ip6` -// Address_Set rows in SB are auto-derived from port group membership by -// ovn-northd, so removing the port group also removes its address sets. -// Non-`sg_*` port groups (e.g. third-party usage) are left alone. Returns -// the count removed. -func scanOrphanPortGroups(ctx context.Context, topo *TopologyHandler, sgs []handlers_ec2_vpc.SecurityGroupRecord) int { - pgs, err := topo.ovn.ListPortGroups(ctx) - if err != nil { - slog.Warn("vpcd reconcile-sgs: list port groups failed", "err", err) - return 0 - } - - expected := make(map[string]struct{}, len(sgs)) - for _, sg := range sgs { - expected[portGroupName(sg.GroupId)] = struct{}{} - } - - removed := 0 - for _, pg := range pgs { - if !strings.HasPrefix(pg.Name, "sg_") { - continue - } - if _, ok := expected[pg.Name]; ok { - continue - } - if err := topo.ovn.ClearACLs(ctx, pg.Name); err != nil { - slog.Warn("vpcd reconcile-sgs: orphan ClearACLs failed", "pg", pg.Name, "err", err) - continue - } - if err := topo.ovn.DeletePortGroup(ctx, pg.Name); err != nil { - slog.Warn("vpcd reconcile-sgs: orphan DeletePortGroup failed", "pg", pg.Name, "err", err) - continue - } - slog.Info("vpcd reconcile-sgs: removed orphan port group", "pg", pg.Name) - removed++ - } - return removed -} - -// sgRulesToACLRules converts handler-side SGRules to vpcd-side SGRuleForACL. -// The duplication keeps vpcd from importing the handler package. -func sgRulesToACLRules(rules []handlers_ec2_vpc.SGRule) []SGRuleForACL { - out := make([]SGRuleForACL, len(rules)) - for i, r := range rules { - out[i] = SGRuleForACL{ - IpProtocol: r.IpProtocol, - FromPort: r.FromPort, - ToPort: r.ToPort, - CidrIp: r.CidrIp, - SourceSG: r.SourceSG, - } - } - return out -} diff --git a/spinifex/services/vpcd/reconcile_sgs_test.go b/spinifex/services/vpcd/reconcile_sgs_test.go deleted file mode 100644 index b80fd5340..000000000 --- a/spinifex/services/vpcd/reconcile_sgs_test.go +++ /dev/null @@ -1,368 +0,0 @@ -package vpcd - -import ( - "context" - "encoding/json" - "testing" - "time" - - handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" - "github.com/mulgadc/spinifex/spinifex/services/vpcd/nbdb" - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -const sgReconcileAccountID = "000000000001" - -func seedSGBucket(t *testing.T, nc *nats.Conn, sgs []handlers_ec2_vpc.SecurityGroupRecord) nats.KeyValue { - t.Helper() - js, err := nc.JetStream() - require.NoError(t, err) - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketSecurityGroups, History: 1}) - require.NoError(t, err) - for _, sg := range sgs { - data, _ := json.Marshal(sg) - _, err := kv.Put(utils.AccountKey(sgReconcileAccountID, sg.GroupId), data) - require.NoError(t, err) - } - return kv -} - -func seedENIBucket(t *testing.T, nc *nats.Conn, enis []handlers_ec2_vpc.ENIRecord) nats.KeyValue { - t.Helper() - js, err := nc.JetStream() - require.NoError(t, err) - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketENIs, History: 1}) - require.NoError(t, err) - for _, e := range enis { - data, _ := json.Marshal(e) - _, err := kv.Put(utils.AccountKey(sgReconcileAccountID, e.NetworkInterfaceId), data) - require.NoError(t, err) - } - return kv -} - -// TestReconcileSGsOnce_RecreatesMissingPortGroup: an SG record exists in KV -// but the corresponding OVN port group is missing (e.g. OVN NB was wiped). -func TestReconcileSGsOnce_RecreatesMissingPortGroup(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - require.NoError(t, ovn.Connect(context.Background())) - topo := NewTopologyHandler(ovn) - - seedSGBucket(t, nc, []handlers_ec2_vpc.SecurityGroupRecord{{ - GroupId: "sg-missing0000000", - GroupName: "web", - VpcId: "vpc-x", - IngressRules: []handlers_ec2_vpc.SGRule{ - {IpProtocol: "tcp", FromPort: 80, ToPort: 80, CidrIp: "0.0.0.0/0"}, - }, - EgressRules: []handlers_ec2_vpc.SGRule{ - {IpProtocol: "-1", FromPort: 0, ToPort: 0, CidrIp: "0.0.0.0/0"}, - }, - CreatedAt: time.Now(), - }}) - seedENIBucket(t, nc, nil) - - result := ReconcileSGsOnce(context.Background(), nc, topo) - - assert.Equal(t, 1, result.PortGroupsRecreated) - pg, err := ovn.GetPortGroup(context.Background(), portGroupName("sg-missing0000000")) - require.NoError(t, err) - assert.Len(t, pg.ACLs, 6, "deny-ingress + deny-egress + dhcp-egress + dhcp-ingress + 1 allow ingress + 1 allow egress") -} - -// TestReconcileSGsOnce_RecreatesPortGroupIdempotent: a second pass on a fresh -// SG must not double-recreate when the port group already exists. -func TestReconcileSGsOnce_RecreatesPortGroupIdempotent(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - require.NoError(t, ovn.Connect(context.Background())) - topo := NewTopologyHandler(ovn) - - seedSGBucket(t, nc, []handlers_ec2_vpc.SecurityGroupRecord{{ - GroupId: "sg-idempotent0000", VpcId: "vpc-x", CreatedAt: time.Now(), - }}) - seedENIBucket(t, nc, nil) - - r1 := ReconcileSGsOnce(context.Background(), nc, topo) - assert.Equal(t, 1, r1.PortGroupsRecreated) - r2 := ReconcileSGsOnce(context.Background(), nc, topo) - assert.Equal(t, 0, r2.PortGroupsRecreated, "second pass must be a no-op") -} - -// TestReconcileSGsOnce_SyncsENIMembership: an ENI's KV record lists SGs but -// the LSP is in the wrong port groups (or none). -func TestReconcileSGsOnce_SyncsENIMembership(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - require.NoError(t, ovn.Connect(context.Background())) - topo := NewTopologyHandler(ovn) - ctx := context.Background() - - require.NoError(t, ovn.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{Name: "subnet-x"})) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - sgEvt := SGEvent{GroupId: "sg-eni0000000000000", VpcId: "vpc-x"} - sgData, _ := json.Marshal(sgEvt) - resp, err := nc.Request(TopicCreateSG, sgData, 2*time.Second) - require.NoError(t, err) - assertSuccess(t, resp, "create SG") - - // Create the LSP without joining any port group — that's the drift. - require.NoError(t, ovn.CreateLogicalSwitchPortInGroups(ctx, "subnet-x", - &nbdb.LogicalSwitchPort{Name: "port-eni-sync1", Addresses: []string{"02:00:00:aa:bb:01 10.0.0.5"}}, - nil)) - - seedSGBucket(t, nc, []handlers_ec2_vpc.SecurityGroupRecord{{ - GroupId: "sg-eni0000000000000", VpcId: "vpc-x", CreatedAt: time.Now(), - }}) - seedENIBucket(t, nc, []handlers_ec2_vpc.ENIRecord{{ - NetworkInterfaceId: "eni-sync1", - PrivateIpAddress: "10.0.0.5", - MacAddress: "02:00:00:aa:bb:01", - SecurityGroupIds: []string{"sg-eni0000000000000"}, - CreatedAt: time.Now(), - }}) - - result := ReconcileSGsOnce(ctx, nc, topo) - - assert.Equal(t, 1, result.PortMembershipsSynced) - groups, err := ovn.ListPortGroupsForPort(ctx, "port-eni-sync1") - require.NoError(t, err) - assert.Contains(t, groups, portGroupName("sg-eni0000000000000"), - "LSP must end up in the SG's port group after reconcile") -} - -// TestReconcileSGsOnce_AllScansClean: a fully-converged cluster sees zero work. -func TestReconcileSGsOnce_AllScansClean(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - require.NoError(t, ovn.Connect(context.Background())) - topo := NewTopologyHandler(ovn) - - seedSGBucket(t, nc, nil) - seedENIBucket(t, nc, nil) - - result := ReconcileSGsOnce(context.Background(), nc, topo) - - assert.Equal(t, SGReconcileResult{}, result, "no drift means no work") -} - -// TestReconcileSGsOnce_NoBucketsHandledGracefully: first-boot path where no -// KV buckets exist yet. -func TestReconcileSGsOnce_NoBucketsHandledGracefully(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - require.NoError(t, ovn.Connect(context.Background())) - topo := NewTopologyHandler(ovn) - - result := ReconcileSGsOnce(context.Background(), nc, topo) - assert.Equal(t, SGReconcileResult{}, result) -} - -// TestReconcileSGsOnce_NoDriftZeroSync: a fully converged ENI (LSP already in -// the right port group, IP already in the address set) must not be counted as -// synced. Otherwise the steady-state cluster logs phantom drift every cycle -// (Plan I.8 — "Constant churn means the reconciler is misclassifying steady -// state as drift"). -func TestReconcileSGsOnce_NoDriftZeroSync(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - require.NoError(t, ovn.Connect(context.Background())) - topo := NewTopologyHandler(ovn) - ctx := context.Background() - - require.NoError(t, ovn.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{Name: "subnet-c"})) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - sgEvt := SGEvent{GroupId: "sg-converged000000", VpcId: "vpc-x"} - sgData, _ := json.Marshal(sgEvt) - resp, err := nc.Request(TopicCreateSG, sgData, 2*time.Second) - require.NoError(t, err) - assertSuccess(t, resp, "create SG") - - // LSP already joined the port group — matches what handleCreateLSP / - // handleUpdatePortSGs leave behind on the happy path. The per-port-group - // `_ip4` Address_Set is auto-derived by ovn-northd in production. - pgName := portGroupName("sg-converged000000") - require.NoError(t, ovn.CreateLogicalSwitchPortInGroups(ctx, "subnet-c", - &nbdb.LogicalSwitchPort{Name: "port-eni-converged", Addresses: []string{"02:00:00:aa:bb:02 10.0.0.6"}}, - []string{pgName})) - - seedSGBucket(t, nc, []handlers_ec2_vpc.SecurityGroupRecord{{ - GroupId: "sg-converged000000", VpcId: "vpc-x", CreatedAt: time.Now(), - }}) - seedENIBucket(t, nc, []handlers_ec2_vpc.ENIRecord{{ - NetworkInterfaceId: "eni-converged", - PrivateIpAddress: "10.0.0.6", - MacAddress: "02:00:00:aa:bb:02", - SecurityGroupIds: []string{"sg-converged000000"}, - CreatedAt: time.Now(), - }}) - - result := ReconcileSGsOnce(ctx, nc, topo) - - assert.Equal(t, 0, result.PortMembershipsSynced, "fully converged ENI must not count as synced") - assert.Equal(t, 0, result.PortGroupsRecreated, "existing PG must not be recreated") - assert.Equal(t, 0, result.OrphanPortGroupsRemoved, "PG with matching SG record must not be torn down") -} - -// TestReconcileSGsOnce_DeletesOrphanPortGroup: an `sg_*` port group with no -// matching SG record must be torn down (PG + ACLs) within one pass. Section -// G2 of the manual test plan. The matching `_ip4` Address_Set is -// auto-derived in SB by ovn-northd and goes away with the port group. -func TestReconcileSGsOnce_DeletesOrphanPortGroup(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - require.NoError(t, ovn.Connect(context.Background())) - topo := NewTopologyHandler(ovn) - ctx := context.Background() - - const orphanPG = "sg_orphan0000000000" - require.NoError(t, ovn.CreatePortGroup(ctx, orphanPG, nil)) - - // Pre-existing ACL on the orphan PG to confirm the scan clears it before - // deleting (matches handleDeleteSG's order). - require.NoError(t, ovn.AddACLs(ctx, orphanPG, []ACLSpec{{ - Direction: "to-lport", Priority: 900, - Match: "outport == @" + orphanPG + " && ip4", Action: "drop", - }})) - - seedSGBucket(t, nc, nil) - seedENIBucket(t, nc, nil) - - result := ReconcileSGsOnce(ctx, nc, topo) - - assert.Equal(t, 1, result.OrphanPortGroupsRemoved) - _, err := ovn.GetPortGroup(ctx, orphanPG) - assert.Error(t, err, "orphan port group must be gone") - pgs, err := ovn.ListPortGroups(ctx) - require.NoError(t, err) - for _, pg := range pgs { - assert.NotEqual(t, orphanPG, pg.Name) - } -} - -// TestReconcileSGsOnce_LeavesNonSpinifexPortGroupsAlone: only `sg_*` port -// groups are spinifex-managed. A PG named differently (e.g. one a third -// party put in OVN) must not be touched. -func TestReconcileSGsOnce_LeavesNonSpinifexPortGroupsAlone(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - require.NoError(t, ovn.Connect(context.Background())) - topo := NewTopologyHandler(ovn) - ctx := context.Background() - - require.NoError(t, ovn.CreatePortGroup(ctx, "third-party-pg", nil)) - - seedSGBucket(t, nc, nil) - seedENIBucket(t, nc, nil) - - result := ReconcileSGsOnce(ctx, nc, topo) - assert.Equal(t, 0, result.OrphanPortGroupsRemoved) - _, err := ovn.GetPortGroup(ctx, "third-party-pg") - assert.NoError(t, err, "non-`sg_*` port group must be left untouched") -} - -// TestReconcileSGsOnce_ENIWithoutLSPLogs: an ENI's KV record references SGs -// but no OVN LSP exists yet. The scan should not count this ENI as synced. -func TestReconcileSGsOnce_ENIWithoutLSPLogs(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - require.NoError(t, ovn.Connect(context.Background())) - topo := NewTopologyHandler(ovn) - - seedSGBucket(t, nc, nil) - seedENIBucket(t, nc, []handlers_ec2_vpc.ENIRecord{{ - NetworkInterfaceId: "eni-no-lsp", - PrivateIpAddress: "10.0.0.1", - SecurityGroupIds: []string{"sg-something00000"}, - CreatedAt: time.Now(), - }}) - - result := ReconcileSGsOnce(context.Background(), nc, topo) - assert.Equal(t, 0, result.PortMembershipsSynced, "ENI with no LSP must not count as synced") -} - -// TestReconcileSGsOnce_NilTopologyHandler: defensive guard — a partial vpcd -// startup that fails before TopologyHandler.ovn is wired must not panic when -// the leader-elected reconciler tick fires. -func TestReconcileSGsOnce_NilTopologyHandler(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - - assert.Equal(t, SGReconcileResult{}, ReconcileSGsOnce(context.Background(), nc, nil), - "nil topo must return zero result, not panic") - assert.Equal(t, SGReconcileResult{}, ReconcileSGsOnce(context.Background(), nc, NewTopologyHandler(nil)), - "topo with nil ovn must return zero result, not panic") -} - -// TestReconcileSGsOnce_SkipsMalformedSGEntry: a corrupt SG record (non-JSON -// bytes) must be logged-and-skipped, not crash the reconciler. Otherwise a -// single bad write to the SG bucket would freeze convergence cluster-wide. -func TestReconcileSGsOnce_SkipsMalformedSGEntry(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - require.NoError(t, ovn.Connect(context.Background())) - topo := NewTopologyHandler(ovn) - - // Seed with one valid + one malformed entry. The malformed entry must not - // abort the scan — the valid SG must still get its port group created. - js, err := nc.JetStream() - require.NoError(t, err) - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketSecurityGroups, History: 1}) - require.NoError(t, err) - _, err = kv.Put(utils.AccountKey(sgReconcileAccountID, "sg-malformed00000"), []byte("not valid json")) - require.NoError(t, err) - good, _ := json.Marshal(handlers_ec2_vpc.SecurityGroupRecord{ - GroupId: "sg-good0000000000", VpcId: "vpc-x", CreatedAt: time.Now(), - }) - _, err = kv.Put(utils.AccountKey(sgReconcileAccountID, "sg-good0000000000"), good) - require.NoError(t, err) - - result := ReconcileSGsOnce(context.Background(), nc, topo) - assert.Equal(t, 1, result.PortGroupsRecreated, "valid SG must still be processed") - _, err = ovn.GetPortGroup(context.Background(), portGroupName("sg-good0000000000")) - assert.NoError(t, err) - _, err = ovn.GetPortGroup(context.Background(), portGroupName("sg-malformed00000")) - assert.Error(t, err, "malformed entry must not produce a port group") -} - -// TestReconcileSGsLoop_ExitsOnContextCancel: the periodic loop must stop when -// its context is cancelled. Otherwise vpcd shutdown leaks a goroutine that -// keeps holding the reconcile-leader lock until TTL expiry — blocking other -// vpcds from converging during the shutdown window. -func TestReconcileSGsLoop_ExitsOnContextCancel(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - require.NoError(t, ovn.Connect(context.Background())) - topo := NewTopologyHandler(ovn) - - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - ReconcileSGsLoop(ctx, nc, topo) - close(done) - }() - - cancel() - select { - case <-done: - case <-time.After(2 * time.Second): - t.Fatal("ReconcileSGsLoop did not return within 2s of ctx cancel") - } -} diff --git a/spinifex/services/vpcd/reconcile_test.go b/spinifex/services/vpcd/reconcile_test.go deleted file mode 100644 index e72d18a2e..000000000 --- a/spinifex/services/vpcd/reconcile_test.go +++ /dev/null @@ -1,892 +0,0 @@ -package vpcd - -import ( - "context" - "encoding/json" - "sync" - "sync/atomic" - "testing" - "time" - - handlers_ec2_eip "github.com/mulgadc/spinifex/spinifex/handlers/ec2/eip" - handlers_ec2_igw "github.com/mulgadc/spinifex/spinifex/handlers/ec2/igw" - handlers_ec2_vpc "github.com/mulgadc/spinifex/spinifex/handlers/ec2/vpc" - "github.com/mulgadc/spinifex/spinifex/services/vpcd/nbdb" - "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestAcquireReconcileLeader_OnlyOneWinner(t *testing.T) { - _, nc, _ := testutil.StartTestJetStream(t) - - releaseA, electedA := AcquireReconcileLeader(nc, "node-a") - require.True(t, electedA, "first caller must win") - require.NotNil(t, releaseA) - - releaseB, electedB := AcquireReconcileLeader(nc, "node-b") - assert.False(t, electedB, "second caller must lose while A holds the lock") - assert.Nil(t, releaseB) - - releaseA() - - // After release the next caller can claim it. - releaseC, electedC := AcquireReconcileLeader(nc, "node-c") - require.True(t, electedC, "next caller wins after release") - releaseC() -} - -// withReconcileLeaderRetry shrinks the bounded-retry window for tests so they -// run sub-second instead of waiting the production 60s deadline. -func withReconcileLeaderRetry(t *testing.T, retryFor, retryStep time.Duration) { - t.Helper() - prevFor, prevStep := reconcileLeaderRetryFor, reconcileLeaderRetryStep - reconcileLeaderRetryFor = retryFor - reconcileLeaderRetryStep = retryStep - t.Cleanup(func() { - reconcileLeaderRetryFor = prevFor - reconcileLeaderRetryStep = prevStep - }) -} - -func TestAcquireReconcileLeader_WaitsForJetStream(t *testing.T) { - // Cold-start scenario: NATS is up but JetStream is not yet enabled. - // AcquireReconcileLeader must retry until JetStream comes online and - // then win the election, instead of failing immediately. - ns, nc := testutil.StartTestNATS(t) - withReconcileLeaderRetry(t, 3*time.Second, 50*time.Millisecond) - - enableAt := time.Now().Add(300 * time.Millisecond) - go func() { - time.Sleep(time.Until(enableAt)) - _ = ns.EnableJetStream(nil) - }() - - start := time.Now() - release, elected := AcquireReconcileLeader(nc, "node-a") - elapsed := time.Since(start) - - require.True(t, elected, "must win election after JetStream comes up") - require.NotNil(t, release) - defer release() - assert.GreaterOrEqual(t, elapsed, 300*time.Millisecond, - "must not return before JetStream is enabled") - assert.Less(t, elapsed, 3*time.Second, "must return within retry deadline") -} - -func TestAcquireReconcileLeader_TimeoutReturnsNotElected(t *testing.T) { - // JetStream never enabled on this server — bounded retry must exhaust - // and return elected=false rather than fall through and run unguarded - // (mulga-js-72: concurrent reconciles produce silent OVN NB damage). - _, nc := testutil.StartTestNATS(t) - withReconcileLeaderRetry(t, 200*time.Millisecond, 20*time.Millisecond) - - start := time.Now() - release, elected := AcquireReconcileLeader(nc, "node-a") - elapsed := time.Since(start) - - assert.False(t, elected, "must skip reconcile when JetStream KV unreachable") - assert.Nil(t, release, "no release closure when not elected") - assert.GreaterOrEqual(t, elapsed, 200*time.Millisecond, "must wait full deadline") -} - -func TestAcquireReconcileLeader_ConcurrentExactlyOneElected(t *testing.T) { - _, nc, _ := testutil.StartTestJetStream(t) - - const goroutines = 5 - var ( - wg sync.WaitGroup - electedCt atomic.Int32 - releases = make(chan func(), goroutines) - ) - wg.Add(goroutines) - for i := range goroutines { - go func(id int) { - defer wg.Done() - release, elected := AcquireReconcileLeader(nc, "node-"+string(rune('a'+id))) - if elected { - electedCt.Add(1) - releases <- release - } else { - assert.Nil(t, release, "non-leader must return nil release") - } - }(i) - } - wg.Wait() - close(releases) - - assert.Equal(t, int32(1), electedCt.Load(), "exactly one caller must win") - for r := range releases { - r() - } -} - -func TestReconcile_NoBootstrap(t *testing.T) { - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - topo := NewTopologyHandler(ovn) - - result := Reconcile(context.Background(), topo, nil) - assert.Equal(t, 0, result.RoutersCreated) - assert.Equal(t, 0, result.SwitchesCreated) - assert.Equal(t, 0, result.IGWsReconciled) -} - -func TestReconcile_EmptyBootstrap(t *testing.T) { - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - topo := NewTopologyHandler(ovn) - - result := Reconcile(context.Background(), topo, &BootstrapVPC{}) - assert.Equal(t, 0, result.RoutersCreated) -} - -func TestReconcile_CreatesBootstrapTopology(t *testing.T) { - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - topo := NewTopologyHandler(ovn, - WithExternalNetwork("pool", []ExternalPoolConfig{{ - Name: "wan", - RangeStart: "192.168.1.200", - RangeEnd: "192.168.1.250", - Gateway: "192.168.1.1", - GatewayIP: "192.168.1.200", - PrefixLen: 23, - }}), - WithChassisNames([]string{"chassis-node1"}), - ) - - bootstrap := &BootstrapVPC{ - AccountID: "000000000001", - VpcId: "vpc-test123", - SubnetId: "subnet-test456", - Cidr: "172.31.0.0/16", - SubnetCidr: "172.31.0.0/20", - } - - result := Reconcile(context.Background(), topo, bootstrap) - assert.Equal(t, 1, result.RoutersCreated) - assert.Equal(t, 1, result.SwitchesCreated) - assert.Equal(t, 1, result.IGWsReconciled) - - // Verify OVN objects exist - ctx := context.Background() - - _, err := ovn.GetLogicalRouter(ctx, "vpc-vpc-test123") - require.NoError(t, err) - - _, err = ovn.GetLogicalSwitch(ctx, "subnet-subnet-test456") - require.NoError(t, err) - - _, err = ovn.GetLogicalSwitch(ctx, "ext-vpc-test123") - require.NoError(t, err) - - // Router port for subnet - _, err = ovn.GetLogicalRouterPort(ctx, "rtr-subnet-test456") - require.NoError(t, err) - - // Gateway router port - _, err = ovn.GetLogicalRouterPort(ctx, "gw-vpc-test123") - require.NoError(t, err) - - // DHCP options should exist - _, err = ovn.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", "subnet-test456") - require.NoError(t, err) -} - -func TestReconcile_Idempotent(t *testing.T) { - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - topo := NewTopologyHandler(ovn, - WithExternalNetwork("pool", []ExternalPoolConfig{{ - Name: "wan", - Gateway: "192.168.1.1", - GatewayIP: "192.168.1.200", - PrefixLen: 23, - }}), - WithChassisNames([]string{"chassis-node1"}), - ) - - bootstrap := &BootstrapVPC{ - AccountID: "000000000001", - VpcId: "vpc-idem", - SubnetId: "subnet-idem", - Cidr: "172.31.0.0/16", - SubnetCidr: "172.31.0.0/20", - } - - // First run creates everything - r1 := Reconcile(context.Background(), topo, bootstrap) - assert.Equal(t, 1, r1.RoutersCreated) - assert.Equal(t, 1, r1.SwitchesCreated) - assert.Equal(t, 1, r1.IGWsReconciled) - - // Second run should skip router/switch creation (already exist). - // IGWsReconciled has "called" semantics — every successful pass counts, - // even when it's a no-op self-heal walk. - r2 := Reconcile(context.Background(), topo, bootstrap) - assert.Equal(t, 0, r2.RoutersCreated) - assert.Equal(t, 0, r2.SwitchesCreated) - assert.Equal(t, 1, r2.IGWsReconciled) - - // Pass must not have duplicated the default route. - lr, err := ovn.GetLogicalRouter(context.Background(), "vpc-vpc-idem") - require.NoError(t, err) - assert.Len(t, lr.StaticRoutes, 1, "default route must not duplicate on retry") -} - -func TestReconcile_PartialState(t *testing.T) { - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - topo := NewTopologyHandler(ovn, - WithExternalNetwork("pool", []ExternalPoolConfig{{ - Name: "wan", - Gateway: "192.168.1.1", - GatewayIP: "192.168.1.200", - PrefixLen: 23, - }}), - WithChassisNames([]string{"chassis-node1"}), - ) - - bootstrap := &BootstrapVPC{ - AccountID: "000000000001", - VpcId: "vpc-partial", - SubnetId: "subnet-partial", - Cidr: "172.31.0.0/16", - SubnetCidr: "172.31.0.0/20", - } - - // Pre-create just the router (simulating partial OVN state) - ctx := context.Background() - _ = topo.reconcileVPC(ctx, "vpc-partial", "172.31.0.0/16") - // IGW ID not needed for pre-creating just the router - - // Reconcile should skip router but create subnet + IGW - result := Reconcile(ctx, topo, bootstrap) - assert.Equal(t, 0, result.RoutersCreated) - assert.Equal(t, 1, result.SwitchesCreated) - assert.Equal(t, 1, result.IGWsReconciled) -} - -// --- ReconcileFromKV tests --- - -// seedKVBuckets creates NATS KV buckets and populates them with test data. -func seedKVBuckets(t *testing.T, nc *nats.Conn, vpcs []handlers_ec2_vpc.VPCRecord, subnets []handlers_ec2_vpc.SubnetRecord, igws []handlers_ec2_igw.IGWRecord, enis []handlers_ec2_vpc.ENIRecord) { - t.Helper() - js, err := nc.JetStream() - require.NoError(t, err) - - vpcKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketVPCs, History: 1}) - require.NoError(t, err) - for _, v := range vpcs { - data, _ := json.Marshal(v) - _, err := vpcKV.Put(utils.AccountKey("000000000001", v.VpcId), data) - require.NoError(t, err) - } - - subnetKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketSubnets, History: 1}) - require.NoError(t, err) - for _, s := range subnets { - data, _ := json.Marshal(s) - _, err := subnetKV.Put(utils.AccountKey("000000000001", s.SubnetId), data) - require.NoError(t, err) - } - - igwKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_igw.KVBucketIGW, History: 1}) - require.NoError(t, err) - for _, i := range igws { - data, _ := json.Marshal(i) - _, err := igwKV.Put(utils.AccountKey("000000000001", i.InternetGatewayId), data) - require.NoError(t, err) - } - - eniKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketENIs, History: 1}) - require.NoError(t, err) - for _, e := range enis { - data, _ := json.Marshal(e) - _, err := eniKV.Put(utils.AccountKey("000000000001", e.NetworkInterfaceId), data) - require.NoError(t, err) - } -} - -func TestReconcileFromKV_CreatesFullTopology(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(ovn, - WithExternalNetwork("pool", []ExternalPoolConfig{{ - Name: "wan", - RangeStart: "10.0.0.200", - RangeEnd: "10.0.0.250", - Gateway: "10.0.0.1", - GatewayIP: "10.0.0.200", - PrefixLen: 24, - }}), - WithChassisNames([]string{"chassis-node1"}), - ) - - seedKVBuckets(t, nc, - []handlers_ec2_vpc.VPCRecord{{ - VpcId: "vpc-kv1", CidrBlock: "10.100.0.0/16", State: "available", - CreatedAt: time.Now(), - }}, - []handlers_ec2_vpc.SubnetRecord{{ - SubnetId: "subnet-kv1", VpcId: "vpc-kv1", CidrBlock: "10.100.1.0/24", - State: "available", CreatedAt: time.Now(), - }}, - []handlers_ec2_igw.IGWRecord{{ - InternetGatewayId: "igw-kv1", VpcId: "vpc-kv1", State: "attached", - CreatedAt: time.Now(), - }}, - []handlers_ec2_vpc.ENIRecord{{ - NetworkInterfaceId: "eni-kv1", SubnetId: "subnet-kv1", VpcId: "vpc-kv1", - PrivateIpAddress: "10.100.1.10", MacAddress: "02:00:00:aa:bb:cc", - Status: "in-use", CreatedAt: time.Now(), - }}, - ) - - result := ReconcileFromKV(ctx, nc, topo, nil) - assert.Equal(t, 1, result.RoutersCreated) - assert.Equal(t, 1, result.SwitchesCreated) - assert.Equal(t, 1, result.IGWsReconciled) - assert.Equal(t, 1, result.PortsCreated) - - // Verify OVN objects - _, err := ovn.GetLogicalRouter(ctx, "vpc-vpc-kv1") - require.NoError(t, err) - - _, err = ovn.GetLogicalSwitch(ctx, "subnet-subnet-kv1") - require.NoError(t, err) - - _, err = ovn.GetLogicalSwitch(ctx, "ext-vpc-kv1") - require.NoError(t, err) - - lsp, err := ovn.GetLogicalSwitchPort(ctx, "port-eni-kv1") - require.NoError(t, err) - assert.Contains(t, lsp.Addresses[0], "10.100.1.10") -} - -func TestReconcileFromKV_Idempotent(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(ovn, - WithExternalNetwork("pool", []ExternalPoolConfig{{ - Name: "wan", Gateway: "10.0.0.1", GatewayIP: "10.0.0.200", PrefixLen: 24, - }}), - WithChassisNames([]string{"chassis-node1"}), - ) - - seedKVBuckets(t, nc, - []handlers_ec2_vpc.VPCRecord{{ - VpcId: "vpc-idem2", CidrBlock: "10.200.0.0/16", State: "available", - CreatedAt: time.Now(), - }}, - []handlers_ec2_vpc.SubnetRecord{{ - SubnetId: "subnet-idem2", VpcId: "vpc-idem2", CidrBlock: "10.200.1.0/24", - State: "available", CreatedAt: time.Now(), - }}, - nil, nil, - ) - - r1 := ReconcileFromKV(ctx, nc, topo, nil) - assert.Equal(t, 1, r1.RoutersCreated) - assert.Equal(t, 1, r1.SwitchesCreated) - - // Second run: everything exists - r2 := ReconcileFromKV(ctx, nc, topo, nil) - assert.Equal(t, 0, r2.RoutersCreated) - assert.Equal(t, 0, r2.SwitchesCreated) -} - -func TestReconcileFromKV_SkipsDetachedIGW(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(ovn, - WithExternalNetwork("pool", []ExternalPoolConfig{{ - Name: "wan", Gateway: "10.0.0.1", GatewayIP: "10.0.0.200", PrefixLen: 24, - }}), - ) - - seedKVBuckets(t, nc, - []handlers_ec2_vpc.VPCRecord{{ - VpcId: "vpc-det", CidrBlock: "10.50.0.0/16", State: "available", - CreatedAt: time.Now(), - }}, - nil, - []handlers_ec2_igw.IGWRecord{{ - InternetGatewayId: "igw-det", VpcId: "", State: "available", - CreatedAt: time.Now(), - }}, - nil, - ) - - result := ReconcileFromKV(ctx, nc, topo, nil) - assert.Equal(t, 1, result.RoutersCreated) // VPC router created - assert.Equal(t, 0, result.IGWsReconciled) // Detached IGW skipped - - _, err := ovn.GetLogicalSwitch(ctx, "ext-vpc-det") - assert.Error(t, err) // External switch should NOT exist -} - -func TestReconcileFromKV_NoBuckets(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - - topo := NewTopologyHandler(ovn) - - // No KV buckets seeded — should handle gracefully - result := ReconcileFromKV(context.Background(), nc, topo, nil) - assert.Equal(t, 0, result.RoutersCreated) - assert.Equal(t, 0, result.SwitchesCreated) - assert.Equal(t, 0, result.IGWsReconciled) - assert.Equal(t, 0, result.PortsCreated) -} - -func TestReconcileFromKV_VersionKeysAndBadJSON(t *testing.T) { - // Tests that _version keys are skipped and malformed JSON records are handled gracefully. - // This covers the "continue" branches for version key filtering and unmarshal errors. - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(ovn, - WithExternalNetwork("pool", []ExternalPoolConfig{{ - Name: "wan", - RangeStart: "10.0.0.200", - RangeEnd: "10.0.0.250", - Gateway: "10.0.0.1", - GatewayIP: "10.0.0.200", - PrefixLen: 24, - }}), - WithChassisNames([]string{"chassis-node1"}), - ) - - js, err := nc.JetStream() - require.NoError(t, err) - - // Create VPC bucket with _version key and one bad JSON record - vpcKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketVPCs, History: 1}) - require.NoError(t, err) - _, err = vpcKV.PutString(utils.VersionKey, "1") - require.NoError(t, err) - _, err = vpcKV.Put("bad-vpc-record", []byte("not-json")) - require.NoError(t, err) - // Also add a valid VPC record - vpcData, _ := json.Marshal(handlers_ec2_vpc.VPCRecord{ - VpcId: "vpc-ver1", CidrBlock: "10.100.0.0/16", State: "available", - CreatedAt: time.Now(), - }) - _, err = vpcKV.Put(utils.AccountKey("000000000001", "vpc-ver1"), vpcData) - require.NoError(t, err) - - // Create subnet bucket with _version key and one bad JSON record - subnetKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketSubnets, History: 1}) - require.NoError(t, err) - _, err = subnetKV.PutString(utils.VersionKey, "1") - require.NoError(t, err) - _, err = subnetKV.Put("bad-subnet", []byte("{invalid")) - require.NoError(t, err) - subnetData, _ := json.Marshal(handlers_ec2_vpc.SubnetRecord{ - SubnetId: "subnet-ver1", VpcId: "vpc-ver1", CidrBlock: "10.100.1.0/24", - State: "available", CreatedAt: time.Now(), - }) - _, err = subnetKV.Put(utils.AccountKey("000000000001", "subnet-ver1"), subnetData) - require.NoError(t, err) - - // Create IGW bucket with _version key and one bad JSON record - igwKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_igw.KVBucketIGW, History: 1}) - require.NoError(t, err) - _, err = igwKV.PutString(utils.VersionKey, "1") - require.NoError(t, err) - _, err = igwKV.Put("bad-igw", []byte("???")) - require.NoError(t, err) - igwData, _ := json.Marshal(handlers_ec2_igw.IGWRecord{ - InternetGatewayId: "igw-ver1", VpcId: "vpc-ver1", State: "attached", - CreatedAt: time.Now(), - }) - _, err = igwKV.Put(utils.AccountKey("000000000001", "igw-ver1"), igwData) - require.NoError(t, err) - - // Create ENI bucket with _version key and one bad JSON record - eniKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketENIs, History: 1}) - require.NoError(t, err) - _, err = eniKV.PutString(utils.VersionKey, "1") - require.NoError(t, err) - _, err = eniKV.Put("bad-eni", []byte("nope")) - require.NoError(t, err) - eniData, _ := json.Marshal(handlers_ec2_vpc.ENIRecord{ - NetworkInterfaceId: "eni-ver1", SubnetId: "subnet-ver1", VpcId: "vpc-ver1", - PrivateIpAddress: "10.100.1.10", MacAddress: "02:00:00:aa:bb:01", - Status: "in-use", CreatedAt: time.Now(), - }) - _, err = eniKV.Put(utils.AccountKey("000000000001", "eni-ver1"), eniData) - require.NoError(t, err) - - result := ReconcileFromKV(ctx, nc, topo, nil) - - // Valid records should still be reconciled despite bad records - assert.Equal(t, 1, result.RoutersCreated) - assert.Equal(t, 1, result.SwitchesCreated) - assert.Equal(t, 1, result.IGWsReconciled) - assert.Equal(t, 1, result.PortsCreated) - - // Verify OVN objects created from valid records - _, err = ovn.GetLogicalRouter(ctx, "vpc-vpc-ver1") - require.NoError(t, err) - _, err = ovn.GetLogicalSwitch(ctx, "subnet-subnet-ver1") - require.NoError(t, err) - _, err = ovn.GetLogicalSwitch(ctx, "ext-vpc-ver1") - require.NoError(t, err) - _, err = ovn.GetLogicalSwitchPort(ctx, "port-eni-ver1") - require.NoError(t, err) -} - -func TestReconcileFromKV_EmptyBuckets(t *testing.T) { - // Tests the nats.ErrNoKeysFound branch when KV buckets exist but are empty. - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - - topo := NewTopologyHandler(ovn) - - js, err := nc.JetStream() - require.NoError(t, err) - - // Create all KV buckets but leave them empty (no keys at all) - _, err = js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketVPCs, History: 1}) - require.NoError(t, err) - _, err = js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketSubnets, History: 1}) - require.NoError(t, err) - _, err = js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_igw.KVBucketIGW, History: 1}) - require.NoError(t, err) - _, err = js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_vpc.KVBucketENIs, History: 1}) - require.NoError(t, err) - - result := ReconcileFromKV(context.Background(), nc, topo, nil) - assert.Equal(t, 0, result.RoutersCreated) - assert.Equal(t, 0, result.SwitchesCreated) - assert.Equal(t, 0, result.IGWsReconciled) - assert.Equal(t, 0, result.PortsCreated) -} - -// --- reconcileGatewayChassis tests (mulga-999) --- - -// seedGatewayLRP creates a router + LRP tagged spinifex:role=gateway so -// reconcileGatewayChassis's rebind step has something to bind. -func seedGatewayLRP(t *testing.T, ovn *MockOVNClient, routerName, lrpName string) { - t.Helper() - ctx := context.Background() - require.NoError(t, ovn.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: routerName})) - require.NoError(t, ovn.CreateLogicalRouterPort(ctx, routerName, &nbdb.LogicalRouterPort{ - Name: lrpName, - ExternalIDs: map[string]string{"spinifex:role": "gateway"}, - })) -} - -func TestReconcileGatewayChassis_RemovesStaleRows(t *testing.T) { - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - topo := NewTopologyHandler(ovn) - ctx := context.Background() - - seedGatewayLRP(t, ovn, "vpc-A", "gw-A") - // One stale (chassis-old not in valid set), one live (UUID-A is). - ovn.SeedGatewayChassis("gw-A", &nbdb.GatewayChassis{ - Name: "gw-A-chassis-old", - ChassisName: "chassis-old", - Priority: 20, - }) - ovn.SeedGatewayChassis("gw-A", &nbdb.GatewayChassis{ - Name: "gw-A-UUID-A", - ChassisName: "UUID-A", - Priority: 20, - }) - - require.NoError(t, topo.reconcileGatewayChassis(ctx, []string{"UUID-A"})) - - rows, err := ovn.ListGatewayChassis(ctx) - require.NoError(t, err) - require.Len(t, rows, 1, "stale row should be deleted, live row should remain") - assert.Equal(t, "UUID-A", rows[0].ChassisName) - assert.Equal(t, 1, ovn.DeleteGatewayChassisCalls) -} - -func TestReconcileGatewayChassis_RebindsAllGatewayLRPs(t *testing.T) { - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - topo := NewTopologyHandler(ovn) - ctx := context.Background() - - seedGatewayLRP(t, ovn, "vpc-A", "gw-A") - seedGatewayLRP(t, ovn, "vpc-B", "gw-B") - // Untagged LRP must NOT be rebinded. - require.NoError(t, ovn.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-C"})) - require.NoError(t, ovn.CreateLogicalRouterPort(ctx, "vpc-C", &nbdb.LogicalRouterPort{ - Name: "rtr-subnet-C", - ExternalIDs: map[string]string{"spinifex:role": "internal"}, - })) - - require.NoError(t, topo.reconcileGatewayChassis(ctx, []string{"chassis-A", "chassis-B"})) - - // 2 gateway LRPs × 2 chassis = 4 SetGatewayChassis create calls. The - // internal LRP must contribute zero. - assert.Equal(t, 4, ovn.SetGatewayChassisCalls) - rows, err := ovn.ListGatewayChassis(ctx) - require.NoError(t, err) - assert.Len(t, rows, 4) - - // Verify priorities — first chassis gets 20, second 15. - prioByGC := make(map[string]int, len(rows)) - for _, gc := range rows { - prioByGC[gc.Name] = gc.Priority - } - assert.Equal(t, 20, prioByGC["gw-A-chassis-A"]) - assert.Equal(t, 15, prioByGC["gw-A-chassis-B"]) - assert.Equal(t, 20, prioByGC["gw-B-chassis-A"]) - assert.Equal(t, 15, prioByGC["gw-B-chassis-B"]) -} - -// TestReconcileIGW_RewritesStaleGatewayPortNetworks guards the mulga-siv-26 -// D8 self-heal path. Pre-seed an IGW topology with a stale pool-IP on the -// gateway LRP (the buggy state shipped to env20); the startup retrofit must -// rewrite every such LRP in place to link-local via UpdateLogicalRouterPort. -// Internal-role LRPs and LRPs already correct must not be touched. -func TestReconcileIGW_RewritesStaleGatewayPortNetworks(t *testing.T) { - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - ctx := context.Background() - topo := NewTopologyHandler(ovn) - - // Stale gateway LRP on VPC A — the buggy pool-IP CIDR. - require.NoError(t, ovn.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-A"})) - require.NoError(t, ovn.CreateLogicalRouterPort(ctx, "vpc-A", &nbdb.LogicalRouterPort{ - Name: "gw-A", - MAC: "02:00:00:74:e8:d2", - Networks: []string{"192.168.0.160/24"}, - ExternalIDs: map[string]string{"spinifex:role": "gateway"}, - })) - // Already-correct gateway LRP on VPC B — must not be touched. - require.NoError(t, ovn.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-B"})) - require.NoError(t, ovn.CreateLogicalRouterPort(ctx, "vpc-B", &nbdb.LogicalRouterPort{ - Name: "gw-B", - MAC: "02:00:00:bb:6d:14", - Networks: []string{"169.254.0.1/30"}, - ExternalIDs: map[string]string{"spinifex:role": "gateway"}, - })) - // Internal LRP on VPC C — must not be touched even though Networks differ. - require.NoError(t, ovn.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-C"})) - require.NoError(t, ovn.CreateLogicalRouterPort(ctx, "vpc-C", &nbdb.LogicalRouterPort{ - Name: "rtr-subnet-C", - Networks: []string{"10.0.1.1/24"}, - ExternalIDs: map[string]string{"spinifex:role": "internal"}, - })) - - topo.RetrofitAllGatewayPortNetworks(ctx) - - gwA, err := ovn.GetLogicalRouterPort(ctx, "gw-A") - require.NoError(t, err) - assert.Equal(t, []string{"169.254.0.1/30"}, gwA.Networks, - "stale gateway Networks must be rewritten in place") - - gwB, err := ovn.GetLogicalRouterPort(ctx, "gw-B") - require.NoError(t, err) - assert.Equal(t, []string{"169.254.0.1/30"}, gwB.Networks) - - internal, err := ovn.GetLogicalRouterPort(ctx, "rtr-subnet-C") - require.NoError(t, err) - assert.Equal(t, []string{"10.0.1.1/24"}, internal.Networks, - "internal LRP must be untouched") - - assert.Equal(t, 1, ovn.UpdateLogicalRouterPortCalls, - "only the stale gateway LRP should trigger Update") - - // Idempotent second pass — all gateway LRPs now correct, no Update. - ovn.UpdateLogicalRouterPortCalls = 0 - topo.RetrofitAllGatewayPortNetworks(ctx) - assert.Equal(t, 0, ovn.UpdateLogicalRouterPortCalls, - "no Update expected when every gateway LRP already link-local") -} - -func TestReconcileGatewayChassis_NoOpWhenAlreadyCorrect(t *testing.T) { - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - topo := NewTopologyHandler(ovn) - ctx := context.Background() - - seedGatewayLRP(t, ovn, "vpc-A", "gw-A") - require.NoError(t, topo.reconcileGatewayChassis(ctx, []string{"chassis-A"})) - // Reset counters from the first (creating) pass. - ovn.SetGatewayChassisCalls = 0 - ovn.DeleteGatewayChassisCalls = 0 - ovn.UpdateGatewayChassisPriorityCalls = 0 - - // Second pass: state is already correct; idempotent path must take no - // destructive or mutative action. - require.NoError(t, topo.reconcileGatewayChassis(ctx, []string{"chassis-A"})) - - assert.Equal(t, 0, ovn.SetGatewayChassisCalls, "no new creates expected") - assert.Equal(t, 0, ovn.DeleteGatewayChassisCalls, "no deletes expected") - assert.Equal(t, 0, ovn.UpdateGatewayChassisPriorityCalls, "no priority updates expected") -} - -// seedEIPKVBucket creates the EIP KV bucket and populates it with test records. -func seedEIPKVBucket(t *testing.T, nc *nats.Conn, eips []handlers_ec2_eip.EIPRecord) { - t.Helper() - js, err := nc.JetStream() - require.NoError(t, err) - eipKV, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: handlers_ec2_eip.KVBucketEIPs, History: 1}) - require.NoError(t, err) - for _, e := range eips { - data, _ := json.Marshal(e) - _, err := eipKV.Put(utils.AccountKey("000000000001", e.AllocationId), data) - require.NoError(t, err) - } -} - -func TestReconcileFromKV_ReconcilesEIPNATRules(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(ovn, - WithExternalNetwork("pool", []ExternalPoolConfig{{ - Name: "wan", Gateway: "10.0.0.1", GatewayIP: "10.0.0.200", PrefixLen: 24, - }}), - WithChassisNames([]string{"chassis-node1"}), - ) - - // Seed VPC + IGW so the router exists for AddNAT. - seedKVBuckets(t, nc, - []handlers_ec2_vpc.VPCRecord{{ - VpcId: "vpc-nat1", CidrBlock: "10.10.0.0/16", State: "available", CreatedAt: time.Now(), - }}, - nil, - []handlers_ec2_igw.IGWRecord{{ - InternetGatewayId: "igw-nat1", VpcId: "vpc-nat1", State: "attached", CreatedAt: time.Now(), - }}, - nil, - ) - - // One associated EIP (missing NAT rule), one unassociated EIP (must be skipped). - seedEIPKVBucket(t, nc, []handlers_ec2_eip.EIPRecord{ - { - AllocationId: "eipalloc-1", PublicIp: "10.0.0.210", PoolName: "wan", - State: "associated", VpcId: "vpc-nat1", PrivateIp: "10.10.1.5", ENIId: "eni-abc123", - CreatedAt: time.Now(), - }, - { - AllocationId: "eipalloc-2", PublicIp: "10.0.0.211", PoolName: "wan", - State: "allocated", - CreatedAt: time.Now(), - }, - }) - - result := ReconcileFromKV(ctx, nc, topo, nil) - assert.Equal(t, 1, result.NATsReconciled, "one missing NAT rule must be created") - - nat, err := ovn.FindNATByExternalIP(ctx, "dnat_and_snat", "10.0.0.210") - require.NoError(t, err) - require.NotNil(t, nat, "dnat_and_snat rule must exist after reconcile") - assert.Equal(t, "10.10.1.5", nat.LogicalIP) - assert.Equal(t, "dnat_and_snat", nat.Type) - - // Unassociated EIP must produce no NAT rule. - nat2, err := ovn.FindNATByExternalIP(ctx, "dnat_and_snat", "10.0.0.211") - require.NoError(t, err) - assert.Nil(t, nat2, "unassociated EIP must not produce a NAT rule") -} - -func TestReconcileFromKV_EIPNATIdempotent(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(ovn, - WithExternalNetwork("pool", []ExternalPoolConfig{{ - Name: "wan", Gateway: "10.0.0.1", GatewayIP: "10.0.0.200", PrefixLen: 24, - }}), - WithChassisNames([]string{"chassis-node1"}), - ) - - seedKVBuckets(t, nc, - []handlers_ec2_vpc.VPCRecord{{ - VpcId: "vpc-nat2", CidrBlock: "10.20.0.0/16", State: "available", CreatedAt: time.Now(), - }}, - nil, - []handlers_ec2_igw.IGWRecord{{ - InternetGatewayId: "igw-nat2", VpcId: "vpc-nat2", State: "attached", CreatedAt: time.Now(), - }}, - nil, - ) - seedEIPKVBucket(t, nc, []handlers_ec2_eip.EIPRecord{{ - AllocationId: "eipalloc-3", PublicIp: "10.0.0.220", PoolName: "wan", - State: "associated", VpcId: "vpc-nat2", PrivateIp: "10.20.1.7", ENIId: "eni-def456", - CreatedAt: time.Now(), - }}) - - // First reconcile: creates the NAT rule. - r1 := ReconcileFromKV(ctx, nc, topo, nil) - assert.Equal(t, 1, r1.NATsReconciled) - - // Second reconcile: rule already correct, must skip. - r2 := ReconcileFromKV(ctx, nc, topo, nil) - assert.Equal(t, 0, r2.NATsReconciled, "correct NAT rule must not be re-created") -} - -func TestReconcileFromKV_EIPNATStaleRuleReplaced(t *testing.T) { - _, nc := startTestJetStreamNATS(t) - ovn := NewMockOVNClient() - _ = ovn.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(ovn, - WithExternalNetwork("pool", []ExternalPoolConfig{{ - Name: "wan", Gateway: "10.0.0.1", GatewayIP: "10.0.0.200", PrefixLen: 24, - }}), - WithChassisNames([]string{"chassis-node1"}), - ) - - seedKVBuckets(t, nc, - []handlers_ec2_vpc.VPCRecord{{ - VpcId: "vpc-nat3", CidrBlock: "10.30.0.0/16", State: "available", CreatedAt: time.Now(), - }}, - nil, - []handlers_ec2_igw.IGWRecord{{ - InternetGatewayId: "igw-nat3", VpcId: "vpc-nat3", State: "attached", CreatedAt: time.Now(), - }}, - nil, - ) - seedEIPKVBucket(t, nc, []handlers_ec2_eip.EIPRecord{{ - AllocationId: "eipalloc-4", PublicIp: "10.0.0.230", PoolName: "wan", - State: "associated", VpcId: "vpc-nat3", PrivateIp: "10.30.1.9", ENIId: "eni-ghi789", - CreatedAt: time.Now(), - }}) - - // Create the router so we can pre-seed a stale NAT rule against it. - require.NoError(t, ovn.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-vpc-nat3"})) - require.NoError(t, ovn.AddNAT(ctx, "vpc-vpc-nat3", &nbdb.NAT{ - Type: "dnat_and_snat", ExternalIP: "10.0.0.230", LogicalIP: "10.30.1.99", - })) - - result := ReconcileFromKV(ctx, nc, topo, nil) - assert.Equal(t, 1, result.NATsReconciled, "stale NAT rule must be replaced") - - nat, err := ovn.FindNATByExternalIP(ctx, "dnat_and_snat", "10.0.0.230") - require.NoError(t, err) - require.NotNil(t, nat) - assert.Equal(t, "10.30.1.9", nat.LogicalIP, "logical IP must be updated to match KV") -} diff --git a/spinifex/services/vpcd/security.go b/spinifex/services/vpcd/security.go deleted file mode 100644 index 6424f1381..000000000 --- a/spinifex/services/vpcd/security.go +++ /dev/null @@ -1,284 +0,0 @@ -package vpcd - -import ( - "context" - "encoding/json" - "fmt" - "log/slog" - - "github.com/nats-io/nats.go" -) - -// NATS topics for security group lifecycle events. -const ( - TopicCreateSG = "vpc.create-sg" - TopicDeleteSG = "vpc.delete-sg" - TopicUpdateSG = "vpc.update-sg" -) - -// denyACLSeverity is the syslog severity OVN uses when a packet matches a -// default-deny ACL. "info" is loud enough to be captured by a syslog forwarder -// yet quiet enough to avoid paging on every dropped scan packet. Operators can -// promote it at their log collector if they want higher-priority alerts. -const denyACLSeverity = "info" - -// SGEvent carries security group state from the handler to vpcd. -type SGEvent struct { - GroupId string `json:"group_id"` - VpcId string `json:"vpc_id"` - IngressRules []SGRuleForACL `json:"ingress_rules,omitempty"` - EgressRules []SGRuleForACL `json:"egress_rules,omitempty"` -} - -// handleCreateSG creates an OVN Port Group and initial ACLs for a new security group. -func (h *TopologyHandler) handleCreateSG(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt SGEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.create-sg event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - if err := h.provisionSG(ctx, evt.GroupId, evt.IngressRules, evt.EgressRules); err != nil { - slog.Error("vpcd: failed to provision security group", "group_id", evt.GroupId, "err", err) - respond(msg, err) - return - } - - slog.Info("vpcd: created security group port group", - "pg", portGroupName(evt.GroupId), - "group_id", evt.GroupId, - "vpc_id", evt.VpcId, - "ingress_rules", len(evt.IngressRules), - "egress_rules", len(evt.EgressRules), - ) - respond(msg, nil) -} - -// provisionSG creates the OVN port group, default-deny ACLs, and allow ACLs -// for the given SG. Used by handleCreateSG (CreateSecurityGroup path) and the -// reconciler's scan-2 (SG record without OVN port group). -// -// SG-to-SG match expressions reference `$_ip4` / `$_ip6` — these -// resolve against the SB Address_Set rows that ovn-northd auto-derives from -// each port group's port addresses (ovn-nb(5): "For each port group, there -// are two address sets generated to the Address_Set table of the -// OVN_Southbound database, ... with name being the name of the Port_Group -// followed by a suffix _ip4 / _ip6"). vpcd must NOT create explicit NB -// Address_Set rows with those names; that produces a duplicate sync to SB -// and wedges northd in an `OVNSB commit failed` retry loop. -// -// All-or-nothing: on any failure after the port group exists, the port group -// and any partial ACLs are torn down so the reconciler's scan-2 can recreate -// the SG from scratch on its next pass. A partial state would otherwise look -// healthy to scanMissingPortGroups (port group exists) while silently -// dropping legitimate traffic via the default-deny ACL. -func (h *TopologyHandler) provisionSG(ctx context.Context, groupId string, ingress, egress []SGRuleForACL) error { - pgName := portGroupName(groupId) - - if err := h.ovn.CreatePortGroup(ctx, pgName, nil); err != nil { - return fmt.Errorf("create port group %s: %w", pgName, err) - } - - done := false - defer func() { - if done { - return - } - if err := h.ovn.ClearACLs(ctx, pgName); err != nil { - slog.Warn("vpcd: provisionSG cleanup ClearACLs failed", "pg", pgName, "err", err) - } - if err := h.ovn.DeletePortGroup(ctx, pgName); err != nil { - slog.Warn("vpcd: provisionSG cleanup DeletePortGroup failed", "pg", pgName, "err", err) - } - }() - - // Default deny ACLs (priority 900, logged for CMMC SC.L1-3.13.1), - // infrastructure DHCP allows (priority 1050), and tenant allow rules - // (priority 1000) go in one OVSDB transaction so a 60-rule SG creation - // is one round-trip, not 64. - specs := append(infrastructureACLs(pgName), ruleACLSpecs(pgName, ingress, egress)...) - if err := h.ovn.AddACLs(ctx, pgName, specs); err != nil { - return fmt.Errorf("add ACLs on %s: %w", pgName, err) - } - done = true - return nil -} - -// handleDeleteSG deletes the OVN Port Group and all associated ACLs. -func (h *TopologyHandler) handleDeleteSG(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt SGEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.delete-sg event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - pgName := portGroupName(evt.GroupId) - - // Clear all ACLs before deleting the port group. Fail-fast — leaving stale - // ACLs causes DeletePortGroup to be rejected by libovsdb (dangling ref). - if err := h.ovn.ClearACLs(ctx, pgName); err != nil { - slog.Error("vpcd: failed to clear ACLs", "pg", pgName, "err", err) - respond(msg, err) - return - } - - if err := h.ovn.DeletePortGroup(ctx, pgName); err != nil { - slog.Error("vpcd: failed to delete port group", "pg", pgName, "err", err) - respond(msg, err) - return - } - - slog.Info("vpcd: deleted security group port group", - "pg", pgName, - "group_id", evt.GroupId, - "vpc_id", evt.VpcId, - ) - respond(msg, nil) -} - -// handleUpdateSG replaces all ACLs for a security group with the current rule set. -func (h *TopologyHandler) handleUpdateSG(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt SGEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.update-sg event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - pgName := portGroupName(evt.GroupId) - - // Clear existing ACLs - if err := h.ovn.ClearACLs(ctx, pgName); err != nil { - slog.Error("vpcd: failed to clear ACLs for update", "pg", pgName, "err", err) - respond(msg, err) - return - } - - // Re-add default deny + DHCP-infra allow ACLs + current rules in one transaction. - specs := append(infrastructureACLs(pgName), ruleACLSpecs(pgName, evt.IngressRules, evt.EgressRules)...) - if err := h.ovn.AddACLs(ctx, pgName, specs); err != nil { - slog.Error("vpcd: failed to add ACLs for update", "pg", pgName, "err", err) - respond(msg, err) - return - } - - slog.Info("vpcd: updated security group ACLs", - "pg", pgName, - "group_id", evt.GroupId, - "vpc_id", evt.VpcId, - "ingress_rules", len(evt.IngressRules), - "egress_rules", len(evt.EgressRules), - ) - respond(msg, nil) -} - -// ruleACLSpecs builds the priority-1000 allow ACL specs for the given rules. -// Allow rules are not logged — accept logging on a private network is high -// volume and low signal; only denies carry Log=true. -func ruleACLSpecs(pgName string, ingress, egress []SGRuleForACL) []ACLSpec { - specs := make([]ACLSpec, 0, len(ingress)+len(egress)) - for _, rule := range ingress { - specs = append(specs, ACLSpec{Direction: "to-lport", Priority: 1000, Match: BuildIngressACLMatch(pgName, rule), Action: "allow-related"}) - } - for _, rule := range egress { - specs = append(specs, ACLSpec{Direction: "from-lport", Priority: 1000, Match: BuildEgressACLMatch(pgName, rule), Action: "allow-related"}) - } - return specs -} - -// denyIngressACL builds the default-deny ingress ACL for a port group with -// logging enabled (CMMC SC.L1-3.13.1 boundary monitoring). -func denyIngressACL(pgName string) ACLSpec { - return ACLSpec{ - Direction: "to-lport", - Priority: 900, - Match: fmt.Sprintf("outport == @%s && ip4", pgName), - Action: "drop", - Name: pgName + "-deny-ingress", - Log: true, - Severity: denyACLSeverity, - } -} - -// denyEgressACL is the egress counterpart to denyIngressACL. -func denyEgressACL(pgName string) ACLSpec { - return ACLSpec{ - Direction: "from-lport", - Priority: 900, - Match: fmt.Sprintf("inport == @%s && ip4", pgName), - Action: "drop", - Name: pgName + "-deny-egress", - Log: true, - Severity: denyACLSeverity, - } -} - -// infrastructureACLs returns the platform-level ACLs that every port group -// carries regardless of tenant rules: the priority-900 default-deny pair and -// the priority-1050 DHCPv4 client/server allow pair. OVN's built-in DHCP -// responder runs inside the LS pipeline *after* the port-group ACL stage, so -// without these allows a narrow-egress tenant SG (e.g. egress restricted to -// the VPC CIDR) silently drops the guest's DHCPDISCOVER (dst=255.255.255.255) -// against the default-deny and the VM never gets an IPv4 lease. AWS users -// can't write a rule for the limited broadcast address, so the platform must -// allow DHCP itself — matching AWS's implicit behaviour. -// -// Priority 1050 sits above the default-deny (900) and above tenant rules -// (1000) so DHCP works regardless of tenant configuration; tenants in our -// model don't write ACLs at >1000, so there is no collision. -func infrastructureACLs(pgName string) []ACLSpec { - return []ACLSpec{ - denyIngressACL(pgName), - denyEgressACL(pgName), - dhcpEgressACL(pgName), - dhcpIngressACL(pgName), - } -} - -// dhcpEgressACL allows the guest's DHCP client traffic (DHCPDISCOVER, -// DHCPREQUEST: udp.src=68, udp.dst=67) out of the port. Action is plain -// "allow", not "allow-related" — DHCP is single-shot request/reply and -// allow-related on broadcast UDP interacts oddly with OVN's CT zones. -func dhcpEgressACL(pgName string) ACLSpec { - return ACLSpec{ - Direction: "from-lport", - Priority: 1050, - Match: fmt.Sprintf("inport == @%s && udp && udp.src == 68 && udp.dst == 67", pgName), - Action: "allow", - Name: pgName + "-allow-dhcp-egress", - } -} - -// dhcpIngressACL allows the OVN-generated DHCP server reply (DHCPOFFER, -// DHCPACK: udp.src=67, udp.dst=68) back to the guest. OVN's DHCP responder -// emits the reply from inside the LS pipeline, so without this matching -// to-lport allow the reply hits the priority-900 ingress default-deny. -func dhcpIngressACL(pgName string) ACLSpec { - return ACLSpec{ - Direction: "to-lport", - Priority: 1050, - Match: fmt.Sprintf("outport == @%s && udp && udp.src == 67 && udp.dst == 68", pgName), - Action: "allow", - Name: pgName + "-allow-dhcp-ingress", - } -} diff --git a/spinifex/services/vpcd/security_test.go b/spinifex/services/vpcd/security_test.go deleted file mode 100644 index 274303363..000000000 --- a/spinifex/services/vpcd/security_test.go +++ /dev/null @@ -1,598 +0,0 @@ -package vpcd - -import ( - "context" - "encoding/json" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestSecurity_CreatePortGroup(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - evt := SGEvent{ - GroupId: "sg-abc123", - VpcId: "vpc-test1", - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicCreateSG, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "create SG port group") - - // Verify port group was created in the mock - mock.mu.Lock() - pg, exists := mock.portGroups["sg_abc123"] - mock.mu.Unlock() - assert.True(t, exists, "port group sg_abc123 should exist") - assert.NotNil(t, pg) - - // Verify the infrastructure ACL set was emitted: 2 default-deny at - // priority 900 (logged for CMMC SC.L1-3.13.1) plus 2 DHCP-infra allows - // at priority 1050 (unlogged — high-volume, low-signal). - type aclSnapshot struct { - action string - log bool - severity string - name string - priority int - direction string - match string - } - var snapshots []aclSnapshot - mock.mu.Lock() - aclCount := len(pg.ACLs) - for _, aclUUID := range pg.ACLs { - a := mock.acls[aclUUID] - if a == nil { - continue - } - snap := aclSnapshot{ - action: a.Action, - log: a.Log, - priority: a.Priority, - direction: a.Direction, - match: a.Match, - } - if a.Severity != nil { - snap.severity = *a.Severity - } - if a.Name != nil { - snap.name = *a.Name - } - snapshots = append(snapshots, snap) - } - mock.mu.Unlock() - - assert.Equal(t, 4, aclCount, "should have 4 infra ACLs (2 deny + 2 DHCP allow)") - denyCount, dhcpAllowCount := 0, 0 - for _, s := range snapshots { - switch s.action { - case "drop": - denyCount++ - assert.True(t, s.log, "default deny ACL must have log=true for boundary monitoring") - assert.Equal(t, "info", s.severity, "default deny ACL must use info severity") - assert.Contains(t, s.name, "sg_abc123-deny-", "default deny ACL must have a name for syslog correlation") - assert.Equal(t, 900, s.priority, "default deny ACL must be priority 900") - case "allow": - dhcpAllowCount++ - assert.False(t, s.log, "DHCP infra allow must not be logged (high-volume, low-signal)") - assert.Equal(t, 1050, s.priority, "DHCP infra allow must be priority 1050 (above tenant rules and default-deny)") - assert.Contains(t, s.name, "sg_abc123-allow-dhcp-", "DHCP allow must have a name for operator correlation") - } - } - assert.Equal(t, 2, denyCount, "should have 2 drop ACLs") - assert.Equal(t, 2, dhcpAllowCount, "should have 2 DHCP allow ACLs (egress client→server, ingress server→client)") -} - -// TestSecurity_DHCPInfraACLs locks the exact shape of the DHCPv4 infra-allow -// ACLs that vpcd emits on every port group. This is the fix for the -// "narrow-egress SG silently breaks DHCP" bug: OVN's DHCP responder runs -// inside the LS pipeline *after* the port-group ACL stage, so without these -// the guest's DHCPDISCOVER (dst=255.255.255.255) hits the priority-900 -// default-deny under any non-0.0.0.0/0 egress rule and the VM never gets a -// lease. -func TestSecurity_DHCPInfraACLs(t *testing.T) { - const pg = "sg_dhcp_shape" - - egress := dhcpEgressACL(pg) - assert.Equal(t, "from-lport", egress.Direction) - assert.Equal(t, 1050, egress.Priority, "must outrank tenant rules (1000) and default-deny (900)") - assert.Equal(t, "allow", egress.Action, "plain allow — DHCP is single-shot, no conntrack") - assert.Equal(t, "inport == @sg_dhcp_shape && udp && udp.src == 68 && udp.dst == 67", egress.Match) - assert.Equal(t, "sg_dhcp_shape-allow-dhcp-egress", egress.Name) - assert.False(t, egress.Log) - - ingress := dhcpIngressACL(pg) - assert.Equal(t, "to-lport", ingress.Direction) - assert.Equal(t, 1050, ingress.Priority) - assert.Equal(t, "allow", ingress.Action) - assert.Equal(t, "outport == @sg_dhcp_shape && udp && udp.src == 67 && udp.dst == 68", ingress.Match) - assert.Equal(t, "sg_dhcp_shape-allow-dhcp-ingress", ingress.Name) - assert.False(t, ingress.Log) -} - -// TestSecurity_InfrastructureACLs_Order locks the deny-before-allow ordering -// that provisionSG/handleUpdateSG rely on when concatenating infrastructure -// ACLs with tenant rule ACLs. -func TestSecurity_InfrastructureACLs_Order(t *testing.T) { - specs := infrastructureACLs("sg_order") - require.Len(t, specs, 4) - assert.Equal(t, "sg_order-deny-ingress", specs[0].Name) - assert.Equal(t, "sg_order-deny-egress", specs[1].Name) - assert.Equal(t, "sg_order-allow-dhcp-egress", specs[2].Name) - assert.Equal(t, "sg_order-allow-dhcp-ingress", specs[3].Name) -} - -func TestSecurity_DeletePortGroup(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Create first - createEvt := SGEvent{GroupId: "sg-del1", VpcId: "vpc-test2"} - data, _ := json.Marshal(createEvt) - resp, err := nc.Request(TopicCreateSG, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "create SG for delete test") - - // Verify it exists - mock.mu.Lock() - _, exists := mock.portGroups["sg_del1"] - mock.mu.Unlock() - assert.True(t, exists) - - // Delete - delEvt := SGEvent{GroupId: "sg-del1", VpcId: "vpc-test2"} - data, _ = json.Marshal(delEvt) - resp, err = nc.Request(TopicDeleteSG, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "delete SG port group") - - // Verify removed - mock.mu.Lock() - _, exists = mock.portGroups["sg_del1"] - mock.mu.Unlock() - assert.False(t, exists, "port group sg_del1 should be deleted") -} - -func TestSecurity_UpdateSGAddRules(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Create SG first (no rules) - createEvt := SGEvent{GroupId: "sg-upd1", VpcId: "vpc-test3"} - data, _ := json.Marshal(createEvt) - resp, err := nc.Request(TopicCreateSG, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "create SG for update test") - - // Update with ingress rules - updateEvt := SGEvent{ - GroupId: "sg-upd1", - VpcId: "vpc-test3", - IngressRules: []SGRuleForACL{ - {IpProtocol: "tcp", FromPort: 22, ToPort: 22, CidrIp: "10.0.0.0/8"}, - {IpProtocol: "tcp", FromPort: 443, ToPort: 443, CidrIp: "0.0.0.0/0"}, - }, - } - data, _ = json.Marshal(updateEvt) - resp, err = nc.Request(TopicUpdateSG, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "update SG with ingress rules") - - // Verify ACLs were created: 2 default deny + 2 ingress allow = 4 - type aclSnapshot struct { - action string - match string - log bool - } - var snapshots []aclSnapshot - mock.mu.Lock() - pg := mock.portGroups["sg_upd1"] - aclCount := len(pg.ACLs) - for _, aclUUID := range pg.ACLs { - a := mock.acls[aclUUID] - if a == nil { - continue - } - snapshots = append(snapshots, aclSnapshot{action: a.Action, match: a.Match, log: a.Log}) - } - mock.mu.Unlock() - - assert.Equal(t, 6, aclCount, "should have 6 ACLs (2 deny + 2 DHCP infra allow + 2 ingress allow)") - - // Check that at least one match contains tcp.dst == 22. Also verify - // logging policy: denies logged, allows not logged (CMMC SC.L1-3.13.1). - foundSSH := false - foundHTTPS := false - for _, s := range snapshots { - switch s.action { - case "drop": - assert.True(t, s.log, "deny ACL must be logged") - case "allow-related": - assert.False(t, s.log, "allow ACL must not be logged (high volume, low signal)") - } - if containsAll(s.match, "tcp.dst == 22", "ip4.src == 10.0.0.0/8") { - foundSSH = true - } - if containsAll(s.match, "tcp.dst == 443") { - foundHTTPS = true - } - } - assert.True(t, foundSSH, "should have SSH ACL with source CIDR") - assert.True(t, foundHTTPS, "should have HTTPS ACL") -} - -// TestSecurity_CreateSG_FailsFastOnAddACLError locks the Phase 1.1 invariant -// that handleCreateSG returns an AddACLs error to the caller and tears down -// the half-built SG. The mock fails the batched AddACLs call, and the test -// asserts the handler responds with failure AND that the port group is -// gone — so the next reconciler scan recreates from scratch instead of -// observing a half-built PG with no ACLs. -func TestSecurity_CreateSG_FailsFastOnAddACLError(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - mock.AddACLErrAfter = 1 - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - evt := SGEvent{GroupId: "sg-failacl", VpcId: "vpc-failacl"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicCreateSG, data, 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "create SG must fail when AddACL fails") - - mock.mu.Lock() - _, pgExists := mock.portGroups["sg_failacl"] - mock.mu.Unlock() - assert.False(t, pgExists, "port group must be torn down on failed provisionSG") -} - -// TestSecurity_UpdateSG_FailsFastOnAddACLError locks the same fail-fast policy -// in handleUpdateSG: clear ACLs, re-add deny ACLs, then add allow ACLs — any -// failure must propagate to the caller instead of being slog.Warn-ed away. -func TestSecurity_UpdateSG_FailsFastOnAddACLError(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - createEvt := SGEvent{GroupId: "sg-updfail", VpcId: "vpc-updfail"} - data, _ := json.Marshal(createEvt) - resp, err := nc.Request(TopicCreateSG, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "create SG before update") - - mock.mu.Lock() - mock.addACLCalls = 0 - mock.AddACLErrAfter = 1 - mock.mu.Unlock() - - updateEvt := SGEvent{ - GroupId: "sg-updfail", - VpcId: "vpc-updfail", - IngressRules: []SGRuleForACL{ - {IpProtocol: "tcp", FromPort: 22, ToPort: 22, CidrIp: "10.0.0.0/8"}, - {IpProtocol: "tcp", FromPort: 443, ToPort: 443, CidrIp: "0.0.0.0/0"}, - }, - } - data, _ = json.Marshal(updateEvt) - resp, err = nc.Request(TopicUpdateSG, data, 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "update SG must fail when AddACL fails mid-batch") -} - -// TestSecurity_CreateSG_WithRules_FailsFastOnAddACLError locks the fail-fast -// policy when handleCreateSG carries allow rules: the batched AddACLs call -// includes both deny ACLs and rule ACLs, and a failure must still tear down -// the PG/AS so the reconciler doesn't mistake a half-built PG as healthy. -func TestSecurity_CreateSG_WithRules_FailsFastOnAddACLError(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - mock.AddACLErrAfter = 1 - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - evt := SGEvent{ - GroupId: "sg-rulefail", - VpcId: "vpc-rulefail", - IngressRules: []SGRuleForACL{ - {IpProtocol: "tcp", FromPort: 22, ToPort: 22, CidrIp: "10.0.0.0/8"}, - {IpProtocol: "tcp", FromPort: 443, ToPort: 443, CidrIp: "0.0.0.0/0"}, - }, - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicCreateSG, data, 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "create SG with rules must fail when AddACLs fails") - - mock.mu.Lock() - _, pgExists := mock.portGroups["sg_rulefail"] - mock.mu.Unlock() - assert.False(t, pgExists, "port group must be torn down when AddACLs fails") -} - -// TestSecurity_HandleCreateSG_OVNNotConnected, ...HandleDeleteSG_..., and -// ...HandleUpdateSG_... lock the defensive guard at the top of each handler: -// when vpcd's OVN client failed to connect (or has been Closed), the SG event -// must surface a clear error to the caller instead of nil-deref'ing. -func TestSecurity_HandleCreateSG_OVNNotConnected(t *testing.T) { - _, nc := startTestNATS(t) - topo := NewTopologyHandler(nil) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - data, _ := json.Marshal(SGEvent{GroupId: "sg-noovn", VpcId: "vpc-noovn"}) - resp, err := nc.Request(TopicCreateSG, data, 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "create SG must fail when ovn client is nil") -} - -func TestSecurity_HandleDeleteSG_OVNNotConnected(t *testing.T) { - _, nc := startTestNATS(t) - topo := NewTopologyHandler(nil) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - data, _ := json.Marshal(SGEvent{GroupId: "sg-noovn", VpcId: "vpc-noovn"}) - resp, err := nc.Request(TopicDeleteSG, data, 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "delete SG must fail when ovn client is nil") -} - -func TestSecurity_HandleUpdateSG_OVNNotConnected(t *testing.T) { - _, nc := startTestNATS(t) - topo := NewTopologyHandler(nil) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - data, _ := json.Marshal(SGEvent{GroupId: "sg-noovn", VpcId: "vpc-noovn"}) - resp, err := nc.Request(TopicUpdateSG, data, 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "update SG must fail when ovn client is nil") -} - -// TestSecurity_HandleCreateSG_BadJSON / ..._HandleDeleteSG_BadJSON / -// ..._HandleUpdateSG_BadJSON: malformed event payloads must surface as -// failures, not be silently dropped — the handler-side caller relies on the -// ack to know the request was processed. -func TestSecurity_HandleCreateSG_BadJSON(t *testing.T) { - _, nc := startTestNATS(t) - topo := NewTopologyHandler(NewMockOVNClient()) - _ = topo.ovn.Connect(context.Background()) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - resp, err := nc.Request(TopicCreateSG, []byte("not json"), 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "create SG with malformed payload must fail") -} - -func TestSecurity_HandleDeleteSG_BadJSON(t *testing.T) { - _, nc := startTestNATS(t) - topo := NewTopologyHandler(NewMockOVNClient()) - _ = topo.ovn.Connect(context.Background()) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - resp, err := nc.Request(TopicDeleteSG, []byte("not json"), 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "delete SG with malformed payload must fail") -} - -func TestSecurity_HandleUpdateSG_BadJSON(t *testing.T) { - _, nc := startTestNATS(t) - topo := NewTopologyHandler(NewMockOVNClient()) - _ = topo.ovn.Connect(context.Background()) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - resp, err := nc.Request(TopicUpdateSG, []byte("not json"), 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "update SG with malformed payload must fail") -} - -// TestSecurity_HandleDeleteSG_FailsFastOnClearACLsError: handleDeleteSG must -// surface the ClearACLs error rather than swallow it. We trigger this by -// requesting a delete for an SG that was never created — the mock's ClearACLs -// returns "port group not found", which must propagate as a failed response. -func TestSecurity_HandleDeleteSG_FailsFastOnClearACLsError(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - data, _ := json.Marshal(SGEvent{GroupId: "sg-never-created", VpcId: "vpc-x"}) - resp, err := nc.Request(TopicDeleteSG, data, 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "delete must fail when port group does not exist") -} - -// TestSecurity_HandleUpdateSG_FailsFastOnClearACLsError: handleUpdateSG must -// also surface the ClearACLs error — silently swallowing it would leave the -// caller thinking the rule update succeeded when in fact the old ACLs are -// still in place. -func TestSecurity_HandleUpdateSG_FailsFastOnClearACLsError(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - data, _ := json.Marshal(SGEvent{ - GroupId: "sg-never-created", - VpcId: "vpc-x", - IngressRules: []SGRuleForACL{{IpProtocol: "tcp", FromPort: 22, ToPort: 22, CidrIp: "0.0.0.0/0"}}, - }) - resp, err := nc.Request(TopicUpdateSG, data, 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "update must fail when port group does not exist") -} - -// TestSecurity_HandleDeleteSG_Idempotent_AddressSetMissing: if the address set -// has already been cleaned up out-of-band but the port group still exists, the -// delete must NOT propagate the AS-not-found error — the goal state is "both -// gone", and we've already achieved it for the AS. Locks the matching -// fail-fast invariant in TestSecurity_DeleteSG_FailsFastOnAddressSetError -// (already covered) by exercising the inverse direction: a fresh delete after -// a successful create must succeed end-to-end without errors leaking. -// -// (kept here to avoid duplicating the create→delete fixture; the existing -// happy-path test exercises the success branch but does not assert that -// neither half-state lingers.) -func TestSecurity_HandleDeleteSG_LeavesNoResidualState(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - data, _ := json.Marshal(SGEvent{GroupId: "sg-residual", VpcId: "vpc-x"}) - resp, err := nc.Request(TopicCreateSG, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "create SG") - - resp, err = nc.Request(TopicDeleteSG, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "delete SG") - - mock.mu.Lock() - _, pgExists := mock.portGroups["sg_residual"] - leftoverACLs := 0 - for _, a := range mock.acls { - if a != nil && (a.Match == "outport == @sg_residual && ip4" || a.Match == "inport == @sg_residual && ip4") { - leftoverACLs++ - } - } - mock.mu.Unlock() - assert.False(t, pgExists, "port group must be gone") - assert.Zero(t, leftoverACLs, "no ACL rows must reference the deleted PG") -} - -// containsAll checks if s contains all substrings. -func containsAll(s string, subs ...string) bool { - for _, sub := range subs { - if !contains(s, sub) { - return false - } - } - return true -} - -func contains(s, sub string) bool { - return len(s) >= len(sub) && searchString(s, sub) -} - -func searchString(s, sub string) bool { - for i := 0; i <= len(s)-len(sub); i++ { - if s[i:i+len(sub)] == sub { - return true - } - } - return false -} diff --git a/spinifex/services/vpcd/topology.go b/spinifex/services/vpcd/topology.go deleted file mode 100644 index 60d7bb488..000000000 --- a/spinifex/services/vpcd/topology.go +++ /dev/null @@ -1,2101 +0,0 @@ -package vpcd - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "log/slog" - "net" - "strconv" - "strings" - "time" - - "github.com/mulgadc/spinifex/spinifex/services/vpcd/dhcp" - "github.com/mulgadc/spinifex/spinifex/services/vpcd/nbdb" - "github.com/mulgadc/spinifex/spinifex/types" - "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" -) - -// waitForFlowsHV shells out to `ovn-nbctl --wait=hv sync`, which bumps -// NB_Global.nb_cfg and blocks until every connected chassis has acknowledged -// the new sequence number — i.e. ovn-northd has compiled NB -> SB and -// ovn-controller has installed the resulting flows. Used after IGW attach -// so newly-launched VMs aren't unreachable while their gateway chassis is -// still catching up (mulga-siv-105). -// -// Bounded by ovn-nbctl --timeout=30 (seconds). On overrun we log a WARN -// and return nil — the caller continues. In practice flows converge within -// seconds; a 30s overrun means something OVN-side is wedged enough that -// failing VPC create wouldn't improve things. -// -// Declared as a var so tests can stub it. -var waitForFlowsHV = func() error { - start := time.Now() - cmd := sudoCommand("ovn-nbctl", - "--no-leader-only", - "--timeout=30", - "--wait=hv", - "sync", - ) - if out, err := cmd.CombinedOutput(); err != nil { - slog.Warn("vpcd: OVN flows-ready barrier overran; continuing without confirmation", - "elapsed", time.Since(start), - "err", err, - "output", strings.TrimSpace(string(out)), - ) - return nil - } - slog.Debug("vpcd: OVN flows-ready barrier complete", "elapsed", time.Since(start)) - return nil -} - -// NATS topics for VPC lifecycle events published by the daemon. -const ( - TopicVPCCreate = "vpc.create" - TopicVPCDelete = "vpc.delete" - TopicSubnetCreate = "vpc.create-subnet" - TopicSubnetDelete = "vpc.delete-subnet" - TopicCreatePort = "vpc.create-port" - TopicDeletePort = "vpc.delete-port" - TopicUpdatePortSGs = "vpc.update-port-sgs" - TopicPortStatus = "vpc.port-status" - TopicIGWAttach = "vpc.igw-attach" - TopicIGWDetach = "vpc.igw-detach" - TopicAddNAT = "vpc.add-nat" - TopicDeleteNAT = "vpc.delete-nat" - TopicAddNATGateway = "vpc.add-nat-gateway" - TopicDeleteNATGateway = "vpc.delete-nat-gateway" -) - -// gatewayPortNetwork is the link-local CIDR every IGW gateway-LRP carries -// in distributed-NAT (direct) mode. Per-VM dnat_and_snat handles ARP per -// chassis so the LRP IP itself never goes on the wire. The default route's -// OutputPort is set explicitly so the WAN nexthop need not be on this subnet. -// -// Centralized-NAT (veth/macvlan) bypasses this constant: the gateway LRP is -// the on-wire egress point and must hold a WAN-subnet IP from the pool's -// gw_lrp_range, otherwise the upstream router silently drops ARP requests -// from an off-subnet sender (RFC 826) and the default route never resolves -// (mulga-siv-36). -const gatewayPortNetwork = "169.254.0.1/30" - -// gatewayIPExtID is the LRP external_ids key holding the gw IP allocated -// from the pool's gw_lrp_range. Persisted so reconcile/retrofit can recover -// the assignment without re-allocating, and so siblings see it as "used". -const gatewayIPExtID = "spinifex:gateway_ip" - -// VPCEvent is published on vpc.create after a VPC is persisted. -type VPCEvent struct { - VpcId string `json:"vpc_id"` - CidrBlock string `json:"cidr_block"` - VNI int64 `json:"vni"` -} - -// SubnetEvent is published on vpc.create-subnet / vpc.delete-subnet. -type SubnetEvent struct { - SubnetId string `json:"subnet_id"` - VpcId string `json:"vpc_id"` - CidrBlock string `json:"cidr_block"` -} - -// PortEvent is published on vpc.create-port / vpc.delete-port. -// -// SecurityGroupIds carries the SG membership the port should join at create -// time so vpcd can wire OVN port-group membership atomically with the LSP -// create. Empty on delete-port (handleDeletePort discovers current -// memberships from the libovsdb cache). -type PortEvent struct { - NetworkInterfaceId string `json:"network_interface_id"` - SubnetId string `json:"subnet_id"` - VpcId string `json:"vpc_id"` - PrivateIpAddress string `json:"private_ip_address"` - MacAddress string `json:"mac_address"` - SecurityGroupIds []string `json:"security_group_ids,omitempty"` -} - -// UpdatePortSGsEvent is published on vpc.update-port-sgs after -// ModifyNetworkInterfaceAttribute changes an ENI's SG membership. The payload -// is declarative — vpcd reads its libovsdb cache to discover current -// memberships and computes the diff against SecurityGroupIds. -type UpdatePortSGsEvent struct { - NetworkInterfaceId string `json:"network_interface_id"` - PrivateIpAddress string `json:"private_ip_address"` - SecurityGroupIds []string `json:"security_group_ids"` -} - -// NATEvent is published on vpc.add-nat / vpc.delete-nat for 1:1 public IP NAT. -type NATEvent struct { - VpcId string `json:"vpc_id"` - ExternalIP string `json:"external_ip"` - LogicalIP string `json:"logical_ip"` - PortName string `json:"port_name"` // logical port for distributed NAT - MAC string `json:"mac"` // external MAC for distributed NAT -} - -// NATGatewayEvent is published on vpc.add-nat-gateway / vpc.delete-nat-gateway. -type NATGatewayEvent struct { - VpcId string `json:"vpc_id"` - NatGatewayId string `json:"nat_gateway_id"` - PublicIp string `json:"public_ip"` - SubnetCidr string `json:"subnet_cidr"` // private subnet CIDR for SNAT rule -} - -// Bridge mode constants for external connectivity. -const ( - // BridgeModeMacvlan uses a macvlan sub-interface on the WAN NIC. SSH-safe - // for single-NIC hosts but requires centralized NAT and MAC alignment. - BridgeModeMacvlan = "macvlan" - // BridgeModeDirect adds the WAN NIC directly to br-external as an OVS port. - // Enables distributed NAT and avoids macvlan workarounds. Only safe when - // the WAN NIC is NOT the SSH/management NIC. - BridgeModeDirect = "direct" - // BridgeModeVeth uses a veth pair to link a Linux bridge (br-wan) to an - // OVS bridge (br-ext). Requires centralized NAT like macvlan because the - // Linux bridge intermediary breaks distributed NAT hairpin routing. - BridgeModeVeth = "veth" - // OvnExternalBridge is the OVS bridge that ovn-bridge-mappings targets - // for the "external" localnet. Owned by setup-ovn.sh's ovn-bridge-mappings - // setup and independent of dhcp_bind_bridge (which is Linux-side in veth - // mode). Named as a constant so the vpcd sanity check refers to the - // contract, not a hardcoded string (D18). - OvnExternalBridge = "br-ext" -) - -// TopologyHandler translates VPC lifecycle NATS events into OVN NB DB operations. -type TopologyHandler struct { - ovn OVNClient - externalMode string - externalPools []ExternalPoolConfig - chassisNames []string // OVN chassis names for gateway HA scheduling - bridgeMode string // "direct" or "macvlan" — controls NAT mode and localnet options - // nc is used to talk to vpcd's DHCPManager via vpc.dhcp.acquire / - // vpc.dhcp.release for gateway LRP IPs in centralized NAT on - // source="dhcp" pools (mulga-siv-38). nil when no DHCP pool is wired or - // the test stack supplies a static-only mock. - nc *nats.Conn -} - -// NewTopologyHandler creates a new TopologyHandler with optional external network config. -func NewTopologyHandler(ovn OVNClient, opts ...TopologyOption) *TopologyHandler { - h := &TopologyHandler{ovn: ovn} - for _, opt := range opts { - opt(h) - } - return h -} - -// TopologyOption configures a TopologyHandler. -type TopologyOption func(*TopologyHandler) - -// WithExternalNetwork configures external connectivity for public subnets. -func WithExternalNetwork(mode string, pools []ExternalPoolConfig) TopologyOption { - return func(h *TopologyHandler) { - h.externalMode = mode - h.externalPools = pools - } -} - -// WithChassisNames sets the OVN chassis names for gateway HA scheduling. -func WithChassisNames(names []string) TopologyOption { - return func(h *TopologyHandler) { - h.chassisNames = names - } -} - -// WithBridgeMode sets the external bridge mode ("direct", "macvlan", or "veth"). -// Direct bridge enables distributed NAT; macvlan and veth use centralized NAT. -// Defaults to macvlan if not set (backward-compatible). -func WithBridgeMode(mode string) TopologyOption { - return func(h *TopologyHandler) { - h.bridgeMode = mode - } -} - -// WithNATSConn wires the NATS connection used to talk to vpcd's DHCPManager -// for gateway-LRP DHCP-acquire (mulga-siv-38). Tests with mock OVN clients -// can omit this — the static / auto-derive paths still work. -func WithNATSConn(nc *nats.Conn) TopologyOption { - return func(h *TopologyHandler) { - h.nc = nc - } -} - -// useCentralizedNAT returns true if the bridge mode requires centralized NAT. -// Macvlan and veth modes both need centralized NAT — macvlan because of MAC -// filtering, veth because the Linux bridge intermediary breaks distributed NAT -// hairpin routing. Only direct bridge mode supports distributed NAT. -func (h *TopologyHandler) useCentralizedNAT() bool { - return h.bridgeMode != BridgeModeDirect -} - -// ensureLocalnetOptions aligns options on an existing localnet port with the -// current bridge mode. CreateLogicalSwitchPort only seeds options at first -// creation; once created, stale options persist forever (e.g. vpcd came up in -// the wrong mode once — pre-Fix 1 — or an operator cleared options manually). -// This read-before-write is a no-op when already correct, so it is safe to -// call on every reconcile. -// -// network_name=external is required regardless of bridge mode — it binds the -// port to the ovn-bridge-mappings entry for the external OVS bridge. -// Centralized mode (veth/macvlan): options:nat-addresses=router must be set. -// Distributed mode (direct): options:nat-addresses must be absent. -func (h *TopologyHandler) ensureLocalnetOptions(ctx context.Context, extPortName string) error { - lsp, err := h.ovn.GetLogicalSwitchPort(ctx, extPortName) - if err != nil { - return fmt.Errorf("get localnet port %s: %w", extPortName, err) - } - wantNat := h.useCentralizedNAT() - currentNat, hasNat := lsp.Options["nat-addresses"] - currentNetName, hasNetName := lsp.Options["network_name"] - natOK := (wantNat && hasNat && currentNat == "router") || (!wantNat && !hasNat) - netNameOK := hasNetName && currentNetName == "external" - if natOK && netNameOK { - return nil - } - if lsp.Options == nil { - lsp.Options = map[string]string{} - } - lsp.Options["network_name"] = "external" - if wantNat { - lsp.Options["nat-addresses"] = "router" - } else { - delete(lsp.Options, "nat-addresses") - } - slog.Info("vpcd: retrofitting localnet options", - "port", extPortName, "bridge_mode", h.bridgeMode, - "nat-addresses", lsp.Options["nat-addresses"], - "network_name", lsp.Options["network_name"]) - if err := h.ovn.UpdateLogicalSwitchPort(ctx, lsp); err != nil { - return fmt.Errorf("update localnet port %s options: %w", extPortName, err) - } - return nil -} - -// expectedGatewayPortNetwork returns the Networks CIDR a gateway LRP for -// vpcId should carry. In direct mode, or in centralized mode without a pool, -// it is the link-local fallback. In centralized mode with a DHCP pool the -// gateway IP is acquired via vpc.dhcp.acquire (mulga-siv-38). With a static -// pool the IP comes from gw_lrp_range or the auto-derived top-of-subnet -// block (mulga-siv-36). Returns the gw IP (empty when link-local) so -// callers can persist it to external_ids. -func (h *TopologyHandler) expectedGatewayPortNetwork(ctx context.Context, vpcId string) (network, gwIP string, err error) { - if !h.useCentralizedNAT() { - return gatewayPortNetwork, "", nil - } - pool := h.findExternalPool("", "") - if pool == nil { - return gatewayPortNetwork, "", nil - } - if pool.IsDHCP() { - ip, prefix, dhcpErr := h.allocateGatewayLRPIPViaDHCP(ctx, vpcId, pool) - if dhcpErr != nil { - return "", "", dhcpErr - } - return fmt.Sprintf("%s/%d", ip, prefix), ip, nil - } - ip, prefix, ok, allocErr := h.allocateGatewayLRPIP(ctx, vpcId, pool) - if allocErr != nil { - return "", "", allocErr - } - if !ok { - return gatewayPortNetwork, "", nil - } - return fmt.Sprintf("%s/%d", ip, prefix), ip, nil -} - -// ensureGatewayPortNetworks rewrites the gateway LRP's Networks column in -// place when it drifts from the mode's expected value. CreateLogicalRouterPort -// is a no-op when the row exists, so reconcile of an upgraded cluster -// otherwise keeps the stale pool-IP networks (mulga-siv-26 D8) or, in -// centralized NAT, the off-subnet link-local that breaks ARP upstream -// (mulga-siv-36). Idempotent — no UPDATE is issued when already correct. -func (h *TopologyHandler) ensureGatewayPortNetworks(ctx context.Context, gwPortName string) error { - lrp, err := h.ovn.GetLogicalRouterPort(ctx, gwPortName) - if err != nil { - return fmt.Errorf("get gateway router port %s: %w", gwPortName, err) - } - vpcId := lrp.ExternalIDs["spinifex:vpc_id"] - if vpcId == "" { - // No VPC tag — this is from the legacy "gw-vpc-..." naming or a - // hand-edited row; fall back to link-local to keep behavior safe. - if len(lrp.Networks) == 1 && lrp.Networks[0] == gatewayPortNetwork { - return nil - } - slog.Info("vpcd: rewriting stale gateway port networks (no vpc_id tag)", - "port", gwPortName, "old", lrp.Networks, "new", gatewayPortNetwork) - lrp.Networks = []string{gatewayPortNetwork} - return h.ovn.UpdateLogicalRouterPort(ctx, lrp) - } - wantNetwork, wantGwIP, err := h.expectedGatewayPortNetwork(ctx, vpcId) - if err != nil { - return fmt.Errorf("compute expected network for %s: %w", gwPortName, err) - } - currentGwIP := lrp.ExternalIDs[gatewayIPExtID] - if len(lrp.Networks) == 1 && lrp.Networks[0] == wantNetwork && currentGwIP == wantGwIP { - return nil - } - slog.Info("vpcd: rewriting stale gateway port networks", - "port", gwPortName, "old", lrp.Networks, "new", wantNetwork, - "old_gw_ip", currentGwIP, "new_gw_ip", wantGwIP) - lrp.Networks = []string{wantNetwork} - if lrp.ExternalIDs == nil { - lrp.ExternalIDs = map[string]string{} - } - if wantGwIP == "" { - delete(lrp.ExternalIDs, gatewayIPExtID) - } else { - lrp.ExternalIDs[gatewayIPExtID] = wantGwIP - } - if err := h.ovn.UpdateLogicalRouterPort(ctx, lrp); err != nil { - return fmt.Errorf("update gateway router port %s: %w", gwPortName, err) - } - return nil -} - -// RetrofitAllGatewayPortNetworks walks every LRP tagged -// spinifex:role=gateway and ensures Networks matches the current mode. Needed -// because Reconcile/ReconcileFromKV early-return when the external switch -// already exists, so reconcileIGW never runs against pre-existing topologies -// shipped by older builds (mulga-siv-26 D8 / mulga-siv-36). Idempotent — the -// underlying ensureGatewayPortNetworks no-ops when already correct. -func (h *TopologyHandler) RetrofitAllGatewayPortNetworks(ctx context.Context) { - lrps, err := h.ovn.ListLogicalRouterPorts(ctx) - if err != nil { - slog.Warn("vpcd: gateway-port retrofit skipped — list LRPs failed", "err", err) - return - } - for _, lrp := range lrps { - if lrp.ExternalIDs["spinifex:role"] != "gateway" { - continue - } - if err := h.ensureGatewayPortNetworks(ctx, lrp.Name); err != nil { - slog.Error("vpcd: gateway-port retrofit failed", "port", lrp.Name, "err", err) - } - } -} - -// RetrofitAllExternalLocalnetOptions walks every OVN logical switch tagged -// spinifex:role=external and calls ensureLocalnetOptions on its localnet -// port. OVN is the source of truth — NATS KV records for IGWs may be absent -// or stale (external switch was created via a live event whose KV record -// later expired, or an operator cleared options manually). Walking OVN -// directly catches all of these on every vpcd startup. Idempotent. -func (h *TopologyHandler) RetrofitAllExternalLocalnetOptions(ctx context.Context) { - switches, err := h.ovn.ListLogicalSwitches(ctx) - if err != nil { - slog.Warn("vpcd: retrofit skipped — list logical switches failed", "err", err) - return - } - for i := range switches { - ls := &switches[i] - if ls.ExternalIDs["spinifex:role"] != "external" { - continue - } - vpcID := ls.ExternalIDs["spinifex:vpc_id"] - if vpcID == "" { - continue - } - extPortName := "ext-port-" + vpcID - if err := h.ensureLocalnetOptions(ctx, extPortName); err != nil { - slog.Error("vpcd: retrofit localnet options failed", "port", extPortName, "err", err) - } - } -} - -// dnsServer returns the DNS server string for OVN DHCP options. -// Uses dns_servers from the external pool config (auto-detected by admin init). -// Falls back to 8.8.8.8 and 1.1.1.1 if none configured. -// OVN DHCP expects the format "{ip1, ip2}" for multiple servers. -func (h *TopologyHandler) dnsServer() string { - pool := h.findExternalPool("", "") - if pool != nil && len(pool.DNSServers) > 0 { - return "{" + strings.Join(pool.DNSServers, ", ") + "}" - } - // Fallback: public DNS - return "{8.8.8.8, 1.1.1.1}" -} - -// findExternalPool returns the first pool matching the given region/AZ, -// using the fallback order: AZ-scoped → region-scoped → unscoped. -func (h *TopologyHandler) findExternalPool(region, az string) *ExternalPoolConfig { - // 1. AZ-scoped match - for i := range h.externalPools { - p := &h.externalPools[i] - if p.AZ != "" && p.AZ == az && p.Region == region { - return p - } - } - // 2. Region-scoped (no AZ) - for i := range h.externalPools { - p := &h.externalPools[i] - if p.AZ == "" && p.Region != "" && p.Region == region { - return p - } - } - // 3. Unscoped (no region, no AZ — global pool) - for i := range h.externalPools { - p := &h.externalPools[i] - if p.Region == "" && p.AZ == "" { - return p - } - } - return nil -} - -// Subscribe registers NATS subscriptions for VPC lifecycle topics. -// Global topology events (VPC/subnet/IGW) use a queue group so exactly one vpcd -// instance handles each event — all instances connect to the same OVN NB DB, so -// any one can process these. Per-node port events use regular subscriptions so -// all instances see them (future: route to the specific node). -func (h *TopologyHandler) Subscribe(nc *nats.Conn) ([]*nats.Subscription, error) { - type sub struct { - topic string - handler nats.MsgHandler - queue bool // true = use queue group (one handler), false = fan-out (all handlers) - } - - subs := []sub{ - {TopicVPCCreate, h.handleVPCCreate, true}, - {TopicVPCDelete, h.handleVPCDelete, true}, - {TopicSubnetCreate, h.handleSubnetCreate, true}, - {TopicSubnetDelete, h.handleSubnetDelete, true}, - {TopicCreatePort, h.handleCreatePort, true}, - {TopicDeletePort, h.handleDeletePort, true}, - {TopicUpdatePortSGs, h.handleUpdatePortSGs, true}, - {TopicIGWAttach, h.handleIGWAttach, true}, - {TopicIGWDetach, h.handleIGWDetach, true}, - {TopicAddNAT, h.handleAddNAT, true}, - {TopicDeleteNAT, h.handleDeleteNAT, true}, - {TopicAddNATGateway, h.handleAddNATGateway, true}, - {TopicDeleteNATGateway, h.handleDeleteNATGateway, true}, - {TopicCreateSG, h.handleCreateSG, true}, - {TopicDeleteSG, h.handleDeleteSG, true}, - {TopicUpdateSG, h.handleUpdateSG, true}, - } - - var result []*nats.Subscription - for _, s := range subs { - var natsSub *nats.Subscription - var err error - if s.queue { - natsSub, err = nc.QueueSubscribe(s.topic, "vpcd-workers", s.handler) - } else { - natsSub, err = nc.Subscribe(s.topic, s.handler) - } - if err != nil { - for _, r := range result { - _ = r.Unsubscribe() - } - return nil, fmt.Errorf("subscribe %s: %w", s.topic, err) - } - result = append(result, natsSub) - slog.Info("Subscribed to VPC topic", "topic", s.topic) - } - - return result, nil -} - -// --- VPC (LogicalRouter) --- - -func (h *TopologyHandler) handleVPCCreate(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt VPCEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.create event", "err", err) - respond(msg, err) - return - } - - routerName := "vpc-" + evt.VpcId - ctx := context.Background() - - // Idempotent: skip if router already exists (another vpcd instance may have created it) - if _, err := h.ovn.GetLogicalRouter(ctx, routerName); err == nil { - slog.Debug("vpcd: logical router already exists, skipping", "router", routerName) - respond(msg, nil) - return - } - - lr := &nbdb.LogicalRouter{ - Name: routerName, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": evt.VpcId, - "spinifex:vni": strconv.FormatInt(evt.VNI, 10), - // Cleanup paths (handleIGWDetach, NAT sweeps) read this to scope - // their SNAT removal to the VPC's own CIDR. When unset, those - // paths fall back to an overbroad 10.0.0.0/8 and emit noisy - // "VPC CIDR missing from router metadata" WARNs. reconcileVPC - // stores the same key on startup; keep parity here. - "spinifex:cidr": evt.CidrBlock, - }, - } - - if err := h.ovn.CreateLogicalRouter(ctx, lr); err != nil { - slog.Error("vpcd: failed to create logical router", "router", routerName, "err", err) - respond(msg, err) - return - } - - slog.Info("vpcd: created logical router for VPC", "router", routerName, "vpc_id", evt.VpcId, "cidr", evt.CidrBlock) - respond(msg, nil) -} - -func (h *TopologyHandler) handleVPCDelete(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt VPCEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.delete event", "err", err) - respond(msg, err) - return - } - - routerName := "vpc-" + evt.VpcId - ctx := context.Background() - - // List all switches to find ones belonging to this VPC - switches, err := h.ovn.ListLogicalSwitches(ctx) - if err != nil { - slog.Warn("vpcd: failed to list switches during VPC delete", "err", err) - } else { - for _, ls := range switches { - if ls.ExternalIDs["spinifex:vpc_id"] == evt.VpcId { - // Delete switch ports first - for range ls.Ports { - // Ports are UUIDs; best-effort cleanup - } - if err := h.ovn.DeleteLogicalSwitch(ctx, ls.Name); err != nil { - slog.Warn("vpcd: failed to delete switch during VPC cascade", "switch", ls.Name, "err", err) - } - } - } - } - - // Delete DHCP options for this VPC - dhcpOpts, err := h.ovn.ListDHCPOptions(ctx) - if err != nil { - slog.Warn("vpcd: failed to list DHCP options during VPC delete", "err", err) - } else { - for _, opts := range dhcpOpts { - if opts.ExternalIDs["spinifex:vpc_id"] == evt.VpcId { - if err := h.ovn.DeleteDHCPOptions(ctx, opts.UUID); err != nil { - slog.Warn("vpcd: failed to delete DHCP options during VPC cascade", "uuid", opts.UUID, "err", err) - } - } - } - } - - if err := h.ovn.DeleteLogicalRouter(ctx, routerName); err != nil { - slog.Error("vpcd: failed to delete logical router", "router", routerName, "err", err) - respond(msg, err) - return - } - - slog.Info("vpcd: deleted logical router for VPC", "router", routerName, "vpc_id", evt.VpcId) - respond(msg, nil) -} - -// --- Subnet (LogicalSwitch + RouterPort + DHCP) --- - -func (h *TopologyHandler) handleSubnetCreate(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt SubnetEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.create-subnet event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - switchName := "subnet-" + evt.SubnetId - routerName := "vpc-" + evt.VpcId - routerPortName := "rtr-" + evt.SubnetId - switchRouterPortName := "rtr-port-" + evt.SubnetId - - // Parse subnet CIDR to compute gateway IP - gwIP, mask, err := subnetGateway(evt.CidrBlock) - if err != nil { - slog.Error("vpcd: invalid subnet CIDR", "cidr", evt.CidrBlock, "err", err) - respond(msg, err) - return - } - gwCIDR := fmt.Sprintf("%s/%d", gwIP, mask) - - // Generate a deterministic MAC for the router port - routerMAC := generateMAC(evt.SubnetId) - - // Idempotent: skip if switch already exists (another vpcd instance may have created it) - if _, err := h.ovn.GetLogicalSwitch(ctx, switchName); err == nil { - slog.Debug("vpcd: subnet topology already exists, skipping", "switch", switchName) - respond(msg, nil) - return - } - - // 1. Create LogicalSwitch - ls := &nbdb.LogicalSwitch{ - Name: switchName, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": evt.SubnetId, - "spinifex:vpc_id": evt.VpcId, - }, - } - if err := h.ovn.CreateLogicalSwitch(ctx, ls); err != nil { - slog.Error("vpcd: failed to create logical switch", "switch", switchName, "err", err) - respond(msg, err) - return - } - - // 2. Create LogicalRouterPort on the VPC router - lrp := &nbdb.LogicalRouterPort{ - Name: routerPortName, - MAC: routerMAC, - Networks: []string{gwCIDR}, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": evt.SubnetId, - "spinifex:vpc_id": evt.VpcId, - }, - } - if err := h.ovn.CreateLogicalRouterPort(ctx, routerName, lrp); err != nil { - slog.Error("vpcd: failed to create router port", "port", routerPortName, "err", err) - // Best-effort cleanup - _ = h.ovn.DeleteLogicalSwitch(ctx, switchName) - respond(msg, err) - return - } - - // 3. Create LogicalSwitchPort (type=router) connecting switch to router - lsp := &nbdb.LogicalSwitchPort{ - Name: switchRouterPortName, - Type: "router", - Addresses: []string{"router"}, - Options: map[string]string{ - "router-port": routerPortName, - }, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": evt.SubnetId, - "spinifex:vpc_id": evt.VpcId, - }, - } - if err := h.ovn.CreateLogicalSwitchPort(ctx, switchName, lsp); err != nil { - slog.Error("vpcd: failed to create switch router port", "port", switchRouterPortName, "err", err) - _ = h.ovn.DeleteLogicalRouterPort(ctx, routerName, routerPortName) - _ = h.ovn.DeleteLogicalSwitch(ctx, switchName) - respond(msg, err) - return - } - - // 4. Create DHCP_Options for the subnet - dhcpOpts := &nbdb.DHCPOptions{ - CIDR: evt.CidrBlock, - Options: map[string]string{ - "server_id": gwIP, - "server_mac": routerMAC, - "lease_time": "3600", - "router": gwIP, - "dns_server": h.dnsServer(), - "mtu": "1442", // Geneve overhead - }, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": evt.SubnetId, - "spinifex:vpc_id": evt.VpcId, - }, - } - if _, err := h.ovn.CreateDHCPOptions(ctx, dhcpOpts); err != nil { - slog.Error("vpcd: failed to create DHCP options", "cidr", evt.CidrBlock, "err", err) - // Non-fatal: switch and router port are still useful - } - - slog.Info("vpcd: created subnet topology", - "switch", switchName, - "router_port", routerPortName, - "gateway", gwCIDR, - "subnet_id", evt.SubnetId, - ) - respond(msg, nil) -} - -func (h *TopologyHandler) handleSubnetDelete(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt SubnetEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.delete-subnet event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - switchName := "subnet-" + evt.SubnetId - routerName := "vpc-" + evt.VpcId - routerPortName := "rtr-" + evt.SubnetId - switchRouterPortName := "rtr-port-" + evt.SubnetId - - // 1. Delete switch router port - if err := h.ovn.DeleteLogicalSwitchPort(ctx, switchName, switchRouterPortName); err != nil { - slog.Warn("vpcd: failed to delete switch router port", "port", switchRouterPortName, "err", err) - } - - // 2. Delete router port - if err := h.ovn.DeleteLogicalRouterPort(ctx, routerName, routerPortName); err != nil { - slog.Warn("vpcd: failed to delete router port", "port", routerPortName, "err", err) - } - - // 3. Delete DHCP options for this subnet - dhcpOpts, err := h.ovn.FindDHCPOptionsByCIDR(ctx, evt.CidrBlock) - if err == nil { - if err := h.ovn.DeleteDHCPOptions(ctx, dhcpOpts.UUID); err != nil { - slog.Warn("vpcd: failed to delete DHCP options", "cidr", evt.CidrBlock, "err", err) - } - } - - // 4. Delete the logical switch - if err := h.ovn.DeleteLogicalSwitch(ctx, switchName); err != nil { - slog.Error("vpcd: failed to delete logical switch", "switch", switchName, "err", err) - respond(msg, err) - return - } - - slog.Info("vpcd: deleted subnet topology", "switch", switchName, "subnet_id", evt.SubnetId) - respond(msg, nil) -} - -// --- Port (LogicalSwitchPort for VM/ENI) --- - -func (h *TopologyHandler) handleCreatePort(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt PortEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.create-port event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - portName := "port-" + evt.NetworkInterfaceId - switchName := "subnet-" + evt.SubnetId - - // Idempotent: if the port already exists from a previous attempt that - // crashed before joining its SG port groups, converge the memberships now - // instead of waiting up to a full reconciler interval — that gap would - // leave a port with zero ACLs (OVN default = unrestricted), defeating the - // atomic-create guarantee on the recovery path. - if _, err := h.ovn.GetLogicalSwitchPort(ctx, portName); err == nil { - if _, err := h.reconcilePortSGs(ctx, portName, evt.SecurityGroupIds); err != nil { - slog.Error("vpcd: failed to reconcile SGs for existing port", "port", portName, "err", err) - respond(msg, err) - return - } - slog.Debug("vpcd: logical switch port already exists, reconciled SG memberships", "port", portName) - respond(msg, nil) - return - } - - addrStr := fmt.Sprintf("%s %s", evt.MacAddress, evt.PrivateIpAddress) - - lsp := &nbdb.LogicalSwitchPort{ - Name: portName, - Addresses: []string{addrStr}, - PortSecurity: []string{addrStr}, - ExternalIDs: map[string]string{ - "spinifex:eni_id": evt.NetworkInterfaceId, - "spinifex:subnet_id": evt.SubnetId, - "spinifex:vpc_id": evt.VpcId, - }, - } - - // Look up DHCP options for the subnet and attach to the port - dhcpOpts, err := h.ovn.FindDHCPOptionsByExternalID(ctx, "spinifex:subnet_id", evt.SubnetId) - if err != nil { - slog.Warn("vpcd: DHCP options not found for subnet, port will not have DHCP", "subnet", evt.SubnetId, "err", err) - } else { - lsp.DHCPv4Options = &dhcpOpts.UUID - } - - // LSP create + switch-port mutate + SG port-group joins + per-SG address-set - // inserts MUST be one OVSDB transaction. A two-step shape leaves a window - // where the port exists outside any port group (OVN default = - // unrestricted) — opposite of the enforcement guarantee. The matching - // `_ip4` Address_Set in SB is auto-derived by ovn-northd from the - // port group's port addresses once the LSP joins. - pgNames := make([]string, 0, len(evt.SecurityGroupIds)) - for _, sgId := range evt.SecurityGroupIds { - pgNames = append(pgNames, portGroupName(sgId)) - } - if err := h.ovn.CreateLogicalSwitchPortInGroups(ctx, switchName, lsp, pgNames); err != nil { - slog.Error("vpcd: failed to create logical switch port", "port", portName, "switch", switchName, "err", err) - respond(msg, err) - return - } - - // Logged at INFO so the ` ` string is captured in journals — the SB - // Address_Set ovn-northd derives for SG-to-SG ACL matches keys off this exact - // field, and a malformed addresses entry silently breaks cross-SG traffic. - slog.Info("vpcd: created logical switch port for ENI", - "port", portName, - "switch", switchName, - "eni_id", evt.NetworkInterfaceId, - "ip", evt.PrivateIpAddress, - "mac", evt.MacAddress, - "addr_str", addrStr, - "sgs", evt.SecurityGroupIds, - "port_groups", pgNames, - ) - respond(msg, nil) -} - -func (h *TopologyHandler) handleDeletePort(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt PortEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.delete-port event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - portName := "port-" + evt.NetworkInterfaceId - switchName := "subnet-" + evt.SubnetId - - // Remove the LSP from every port group before deleting it. OVSDB rejects - // deletes of a row still referenced by another row's set column (the port - // group's ports set), so swallowing this error guarantees the next - // DeleteLogicalSwitchPort fails with a misleading reference-integrity - // error and hides the real first-step cause. - if _, err := h.reconcilePortSGs(ctx, portName, nil); err != nil { - slog.Error("vpcd: failed to clear port group memberships before delete", "port", portName, "err", err) - respond(msg, err) - return - } - - if err := h.ovn.DeleteLogicalSwitchPort(ctx, switchName, portName); err != nil { - slog.Error("vpcd: failed to delete logical switch port", "port", portName, "switch", switchName, "err", err) - respond(msg, err) - return - } - - slog.Info("vpcd: deleted logical switch port for ENI", - "port", portName, - "switch", switchName, - "eni_id", evt.NetworkInterfaceId, - ) - respond(msg, nil) -} - -// handleUpdatePortSGs reconciles the port group membership for an ENI's LSP -// against the desired SG list in the event. The payload is declarative — vpcd -// computes the add/remove diff from the current libovsdb cache state. Same -// helper is used by the reconciler. -func (h *TopologyHandler) handleUpdatePortSGs(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt UpdatePortSGsEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.update-port-sgs event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - portName := "port-" + evt.NetworkInterfaceId - - if _, err := h.reconcilePortSGs(ctx, portName, evt.SecurityGroupIds); err != nil { - slog.Error("vpcd: failed to reconcile port SGs", - "port", portName, "eni_id", evt.NetworkInterfaceId, "sgs", evt.SecurityGroupIds, "err", err) - respond(msg, err) - return - } - - slog.Info("vpcd: updated port group memberships", - "port", portName, - "eni_id", evt.NetworkInterfaceId, - "sgs", evt.SecurityGroupIds, - ) - respond(msg, nil) -} - -// reconcilePortSGs converges the OVN port-group membership of lspName against -// desiredSGs. Reads current membership from the libovsdb cache, computes -// add/remove sets, and applies each side incrementally. Idempotent — replaying -// the same desired state is a no-op. Self-healing — a previous partial failure -// converges on the next call. desiredSGs may be nil (delete-port path) to -// remove the port from every group. -// -// SG-to-SG rule matches like "ip4.src == $_ip4" resolve against the SB -// Address_Set rows that ovn-northd auto-derives from each port group's port -// addresses, so port-group join/leave alone is enough — no explicit -// address-set membership maintenance is required here. -// -// Returns changed=true iff at least one port-group join or leave was applied; -// callers use this to distinguish actual drift from a no-op pass. -func (h *TopologyHandler) reconcilePortSGs(ctx context.Context, lspName string, desiredSGs []string) (bool, error) { - desired := make(map[string]struct{}, len(desiredSGs)) - for _, sgId := range desiredSGs { - desired[portGroupName(sgId)] = struct{}{} - } - - currentNames, err := h.ovn.ListPortGroupsForPort(ctx, lspName) - if err != nil { - return false, fmt.Errorf("list current port groups for %s: %w", lspName, err) - } - current := make(map[string]struct{}, len(currentNames)) - for _, name := range currentNames { - current[name] = struct{}{} - } - - addPGs := make([]string, 0) - for name := range desired { - if _, ok := current[name]; !ok { - addPGs = append(addPGs, name) - } - } - removePGs := make([]string, 0) - for name := range current { - if _, ok := desired[name]; !ok { - removePGs = append(removePGs, name) - } - } - - if len(addPGs) == 0 && len(removePGs) == 0 { - return false, nil - } - - // Single OVSDB transaction so a 5-SG → different-5-SG modify never - // exposes an intermediate state with fewer port groups (which would - // let the OVN default = unrestricted apply for the gap). - if err := h.ovn.UpdatePortGroupMemberships(ctx, lspName, addPGs, removePGs); err != nil { - return false, err - } - // Logged at INFO so SG-to-SG ACL failures can be correlated with the exact - // port-group join/leave events that drive ovn-northd's SB Address_Set - // derivation - slog.Info("vpcd: reconciled port group memberships", - "port", lspName, - "added", addPGs, - "removed", removePGs, - "desired", desiredSGs, - ) - return true, nil -} - -// --- Internet Gateway (external connectivity + NAT) --- - -func (h *TopologyHandler) handleIGWAttach(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt types.IGWEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.igw-attach event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - routerName := "vpc-" + evt.VpcId - extSwitchName := "ext-" + evt.VpcId - extPortName := "ext-port-" + evt.VpcId - gwPortName := "gw-" + evt.VpcId - switchGWPortName := "gw-port-" + evt.VpcId - - // Idempotent: skip if external switch already exists - if _, err := h.ovn.GetLogicalSwitch(ctx, extSwitchName); err == nil { - slog.Debug("vpcd: IGW topology already exists, skipping", "switch", extSwitchName) - respond(msg, nil) - return - } - - // 1. Create external logical switch (localnet for physical uplink) - extSwitch := &nbdb.LogicalSwitch{ - Name: extSwitchName, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": evt.VpcId, - "spinifex:igw_id": evt.InternetGatewayId, - "spinifex:role": "external", - }, - } - if err := h.ovn.CreateLogicalSwitch(ctx, extSwitch); err != nil { - slog.Error("vpcd: failed to create external switch", "switch", extSwitchName, "err", err) - respond(msg, err) - return - } - - // 2. Create localnet port on external switch (maps to physical network) - localnetOpts := map[string]string{ - "network_name": "external", - } - // nat-addresses=router: OVN sends gratuitous ARPs for all NAT external IPs - // using the router port MAC. Required for centralized NAT modes (macvlan, - // veth) so that ARP replies for NAT IPs reach hosts correctly. Not needed - // for direct bridge since OVS sees all traffic on the wire without MAC - // filtering and distributed NAT handles ARP per-chassis. - if h.useCentralizedNAT() { - localnetOpts["nat-addresses"] = "router" - } - localnetPort := &nbdb.LogicalSwitchPort{ - Name: extPortName, - Type: "localnet", - Addresses: []string{"unknown"}, - Options: localnetOpts, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": evt.VpcId, - "spinifex:igw_id": evt.InternetGatewayId, - }, - } - if err := h.ovn.CreateLogicalSwitchPort(ctx, extSwitchName, localnetPort); err != nil { - slog.Error("vpcd: failed to create localnet port", "port", extPortName, "err", err) - _ = h.ovn.DeleteLogicalSwitch(ctx, extSwitchName) - respond(msg, err) - return - } - - // Resolve external pool for this VPC's WAN nexthop and gw LRP IP. - // - // Direct mode: link-local LRP is fine — distributed NAT puts a per-VM - // dnat_and_snat row with external_mac/logical_port on every chassis, so - // the LRP IP itself never goes on the wire. - // - // Centralized mode (veth/macvlan): the LRP is the on-wire egress, so it - // must hold a WAN-subnet IP. Allocate one from pool.GwLrpRange (or - // auto-derive from the WAN subnet) so each VPC gets a distinct sender - // IP (mulga-siv-36). Without this the upstream router silently drops - // ARP for the WAN nexthop (RFC 826) and the default route never - // resolves. - // TODO: use VPC's region/AZ once we track it; for now use first matching pool. - pool := h.findExternalPool("", "") - gatewayNetwork := gatewayPortNetwork - wanGateway := "169.254.0.2" - gwLrpIP := "" - - switch { - case pool != nil: - wanGateway = pool.Gateway - if h.useCentralizedNAT() { - network, ip, allocErr := h.expectedGatewayPortNetwork(ctx, evt.VpcId) - if allocErr != nil { - slog.Error("vpcd: failed to allocate gw LRP IP for centralized NAT", - "vpc_id", evt.VpcId, "pool", pool.Name, "err", allocErr) - _ = h.ovn.DeleteLogicalSwitch(ctx, extSwitchName) - respond(msg, allocErr) - return - } - gwLrpIP = ip - gatewayNetwork = network - } - slog.Info("vpcd: using external pool for IGW", - "pool", pool.Name, - "source", pool.Source, - "lrp_network", gatewayNetwork, - "wan_gateway", wanGateway, - ) - case h.externalMode == "pool": - slog.Warn("vpcd: external mode is set but no matching pool found, using link-local fallback") - } - - // 3. Create gateway router port on the VPC router connecting to external switch - gwMAC := generateMAC("gw-" + evt.VpcId) - lrpExtIDs := map[string]string{ - "spinifex:vpc_id": evt.VpcId, - "spinifex:igw_id": evt.InternetGatewayId, - "spinifex:role": "gateway", - } - if gwLrpIP != "" { - lrpExtIDs[gatewayIPExtID] = gwLrpIP - } - lrp := &nbdb.LogicalRouterPort{ - Name: gwPortName, - MAC: gwMAC, - Networks: []string{gatewayNetwork}, - ExternalIDs: lrpExtIDs, - } - if err := h.ovn.CreateLogicalRouterPort(ctx, routerName, lrp); err != nil { - slog.Error("vpcd: failed to create gateway router port", "port", gwPortName, "err", err) - _ = h.ovn.DeleteLogicalSwitch(ctx, extSwitchName) - respond(msg, err) - return - } - - // 4. Create switch port connecting external switch to router - switchGWPort := &nbdb.LogicalSwitchPort{ - Name: switchGWPortName, - Type: "router", - Addresses: []string{"router"}, - Options: map[string]string{ - "router-port": gwPortName, - }, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": evt.VpcId, - "spinifex:igw_id": evt.InternetGatewayId, - }, - } - if err := h.ovn.CreateLogicalSwitchPort(ctx, extSwitchName, switchGWPort); err != nil { - slog.Error("vpcd: failed to create switch gateway port", "port", switchGWPortName, "err", err) - _ = h.ovn.DeleteLogicalRouterPort(ctx, routerName, gwPortName) - _ = h.ovn.DeleteLogicalSwitch(ctx, extSwitchName) - respond(msg, err) - return - } - - // 5. No blanket SNAT rule — AWS behavior requires that only instances with - // public IPs (via MapPublicIpOnLaunch or EIPs) can route through the IGW. - // Per-VM dnat_and_snat rules created by handleAddNAT provide both inbound - // DNAT and outbound SNAT for public instances. Private subnet instances - // (no public IP, no NAT rule) cannot reach the internet — their packets - // leave the router with a private source IP that the upstream router drops. - // A future NAT Gateway feature will add scoped SNAT for private subnets. - - // 6. Add default route pointing to the WAN gateway - // OutputPort must be set explicitly because the gateway router port uses - // link-local 169.254.0.1/30, whose network does not contain the WAN - // nexthop (e.g. 192.168.1.1). Without it OVN northd silently drops the - // route from the southbound DB. - defaultRoute := &nbdb.LogicalRouterStaticRoute{ - IPPrefix: "0.0.0.0/0", - Nexthop: wanGateway, - OutputPort: &gwPortName, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": evt.VpcId, - "spinifex:igw_id": evt.InternetGatewayId, - }, - } - if err := h.ovn.AddStaticRoute(ctx, routerName, defaultRoute); err != nil { - slog.Warn("vpcd: failed to add default route", "router", routerName, "err", err) - } - - // 7. Schedule gateway chassis for HA — tells OVN which hosts can handle external traffic - if len(h.chassisNames) > 0 { - for i, chassis := range h.chassisNames { - priority := max( - // First chassis gets highest priority - 20-(i*5), 1) - if err := h.ovn.SetGatewayChassis(ctx, gwPortName, chassis, priority); err != nil { - slog.Warn("vpcd: failed to set gateway chassis", "port", gwPortName, "chassis", chassis, "priority", priority, "err", err) - } else { - slog.Info("vpcd: set gateway chassis", "port", gwPortName, "chassis", chassis, "priority", priority) - } - } - } else { - slog.Warn("vpcd: no chassis names configured — gateway port has no chassis binding, external traffic will not flow") - } - - slog.Info("vpcd: attached internet gateway to VPC", - "igw_id", evt.InternetGatewayId, - "vpc_id", evt.VpcId, - "ext_switch", extSwitchName, - "gw_port", gwPortName, - "lrp_network", gatewayNetwork, - "wan_gateway", wanGateway, - "chassis_count", len(h.chassisNames), - ) - - // Block until ovn-northd compiles + the gateway chassis has installed - // flows for this VPC. Without this, a subsequent RunInstances -> - // vpc.add-nat can complete while the datapath is still dark, and the - // new VM is unreachable on its public IP for the duration of the - // flow-install latency (typically seconds; observed up to minutes when - // the chassis was loaded during CI). mulga-siv-105. - _ = waitForFlowsHV() - respond(msg, nil) -} - -func (h *TopologyHandler) handleIGWDetach(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt types.IGWEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.igw-detach event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - routerName := "vpc-" + evt.VpcId - extSwitchName := "ext-" + evt.VpcId - extPortName := "ext-port-" + evt.VpcId - gwPortName := "gw-" + evt.VpcId - switchGWPortName := "gw-port-" + evt.VpcId - - // 1. Delete default route - if err := h.ovn.DeleteStaticRoute(ctx, routerName, "0.0.0.0/0"); err != nil { - slog.Warn("vpcd: failed to delete default route", "router", routerName, "err", err) - } - - // 2. Delete SNAT rule(s) for this IGW - // Find VPC CIDR for the SNAT rule - router, err := h.ovn.GetLogicalRouter(ctx, routerName) - if err != nil { - slog.Warn("vpcd: failed to get router for NAT cleanup", "router", routerName, "err", err) - } else { - vpcCIDR := router.ExternalIDs["spinifex:cidr"] - if vpcCIDR == "" { - vpcCIDR = "10.0.0.0/8" - slog.Warn("vpcd: VPC CIDR missing from router metadata, using overbroad fallback for NAT cleanup", - "router", routerName, "fallbackCIDR", vpcCIDR) - } - if err := h.ovn.DeleteNAT(ctx, routerName, "snat", vpcCIDR); err != nil { - slog.Warn("vpcd: failed to delete SNAT rule", "router", routerName, "err", err) - } - } - - // 3. Delete switch gateway port - if err := h.ovn.DeleteLogicalSwitchPort(ctx, extSwitchName, switchGWPortName); err != nil { - slog.Warn("vpcd: failed to delete switch gateway port", "port", switchGWPortName, "err", err) - } - - // 4. Delete gateway router port - if err := h.ovn.DeleteLogicalRouterPort(ctx, routerName, gwPortName); err != nil { - slog.Warn("vpcd: failed to delete gateway router port", "port", gwPortName, "err", err) - } - - // 5. Delete localnet port - if err := h.ovn.DeleteLogicalSwitchPort(ctx, extSwitchName, extPortName); err != nil { - slog.Warn("vpcd: failed to delete localnet port", "port", extPortName, "err", err) - } - - // 6. Delete external switch - if err := h.ovn.DeleteLogicalSwitch(ctx, extSwitchName); err != nil { - slog.Error("vpcd: failed to delete external switch", "switch", extSwitchName, "err", err) - respond(msg, err) - return - } - - // 7. Release any DHCP-acquired gateway LRP lease (mulga-siv-38). Best- - // effort — upstream server expires the lease on its own if this fails. - h.releaseGatewayLRPLease(evt.VpcId) - - slog.Info("vpcd: detached internet gateway from VPC", - "igw_id", evt.InternetGatewayId, - "vpc_id", evt.VpcId, - ) - respond(msg, nil) -} - -// --- NAT (dnat_and_snat for public IPs) --- - -// handleAddNAT installs a dnat_and_snat rule for an EIP. Idempotent: if a rule -// already exists for the external IP and its state matches the target -// (LogicalIP, VpcId, plus ExternalMAC/LogicalPort under distributed-NAT mode), -// the handler returns without touching NB. This avoids the delete-then-add -// gap during which a chassis briefly has no flow for the EIP — a gap that -// concurrent duplicate publishes for the same EIP would otherwise reopen -// (mulga-siv-124). -func (h *TopologyHandler) handleAddNAT(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt NATEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.add-nat event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - routerName := "vpc-" + evt.VpcId - - // NAT mode depends on bridge type: - // - Direct bridge: distributed NAT (ExternalMAC + LogicalPort set). DNAT - // is processed on the VM's own chassis — no hairpin through the gateway - // chassis. This is the preferred mode for multi-node performance. - natRule := &nbdb.NAT{ - Type: "dnat_and_snat", - ExternalIP: evt.ExternalIP, - LogicalIP: evt.LogicalIP, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": evt.VpcId, - "spinifex:public_ip": evt.ExternalIP, - }, - } - distributed := !h.useCentralizedNAT() && evt.MAC != "" && evt.PortName != "" - if distributed { - natRule.ExternalMAC = &evt.MAC - natRule.LogicalPort = &evt.PortName - slog.Debug("vpcd: using distributed NAT (direct bridge)", - "external_ip", evt.ExternalIP, "port", evt.PortName, "mac", evt.MAC) - } - - // Idempotency: if the existing rule already matches the target, do not - // run delete-then-add. Concurrent duplicate vpc.add-nat publishes for the - // same EIP would otherwise tear down the rule the previous invocation - // just installed, creating a flow-install gap on the chassis. - if existing, err := h.ovn.FindNATByExternalIP(ctx, "dnat_and_snat", evt.ExternalIP); err != nil { - slog.Warn("vpcd: failed to look up existing NAT for idempotency check", "external_ip", evt.ExternalIP, "err", err) - } else if existing != nil && existing.LogicalIP == evt.LogicalIP && - existing.ExternalIDs["spinifex:vpc_id"] == evt.VpcId && - (!distributed || - (existing.ExternalMAC != nil && *existing.ExternalMAC == evt.MAC && - existing.LogicalPort != nil && *existing.LogicalPort == evt.PortName)) { - slog.Info("vpcd: dnat_and_snat already current, skipping", - "router", routerName, - "external_ip", evt.ExternalIP, - "logical_ip", evt.LogicalIP, - "port", evt.PortName, - ) - respond(msg, nil) - return - } - - // Remove any stale NAT rule for the same external IP before adding the new - // one. Search ALL routers, not just the target — stale rules may exist on a - // different VPC's router when vpc.delete-nat (fire-and-forget) hasn't been - // processed before the IP was reused by a new VPC. - if removed, err := h.ovn.DeleteAllNATsByExternalIP(ctx, "dnat_and_snat", evt.ExternalIP); err != nil { - slog.Warn("vpcd: failed to clean up stale NAT rules for external IP", "external_ip", evt.ExternalIP, "err", err) - } else if removed > 0 { - slog.Info("vpcd: cleaned up stale NAT rules before re-add", "external_ip", evt.ExternalIP, "removed", removed) - } - - if err := h.ovn.AddNAT(ctx, routerName, natRule); err != nil { - slog.Error("vpcd: failed to add dnat_and_snat rule", "router", routerName, "externalIP", evt.ExternalIP, "logicalIP", evt.LogicalIP, "err", err) - respond(msg, err) - return - } - - slog.Info("vpcd: added dnat_and_snat rule", - "router", routerName, - "external_ip", evt.ExternalIP, - "logical_ip", evt.LogicalIP, - "port", evt.PortName, - ) - - // Block until ovn-northd compiles the new NAT into SB and every chassis - // has installed the resulting flows. handleIGWAttach has the same - // barrier (mulga-siv-105); without one here, the publisher (typically - // ec2d/RunInstances) returns success while the DNAT/SNAT flows for the - // VM's public IP are still being installed on the gateway chassis, so - // the freshly launched VM is unreachable on its public IP for whatever - // flow-install latency the chassis happens to incur. Reproduced in CI - // run 26072432957 Phase8b — Port_Binding showed up=true at the 3min - // timeout, but SSH never completed handshake. - _ = waitForFlowsHV() - respond(msg, nil) -} - -func (h *TopologyHandler) handleDeleteNAT(msg *nats.Msg) { - if h.ovn == nil { - respond(msg, fmt.Errorf("OVN client not connected")) - return - } - - var evt NATEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.delete-nat event", "err", err) - respond(msg, err) - return - } - - ctx := context.Background() - routerName := "vpc-" + evt.VpcId - - if err := h.ovn.DeleteNAT(ctx, routerName, "dnat_and_snat", evt.LogicalIP); err != nil { - // vpc.delete-nat is published from multiple cleanup paths (per-VM - // ReleasePublicIP, ENI delete, VPC delete sweep), so a duplicate - // delete for the same logical_ip is expected. Treat "already gone" - // as success — the caller's intent ("ensure this NAT is removed") - // is satisfied whether we did the delete or someone else did. - if errors.Is(err, ErrNATNotFound) { - slog.Info("vpcd: dnat_and_snat rule already absent (idempotent)", - "router", routerName, "logical_ip", evt.LogicalIP) - respond(msg, nil) - return - } - slog.Error("vpcd: failed to delete dnat_and_snat rule", "router", routerName, "logicalIP", evt.LogicalIP, "err", err) - respond(msg, err) - return - } - - slog.Info("vpcd: deleted dnat_and_snat rule", - "router", routerName, - "logical_ip", evt.LogicalIP, - ) - respond(msg, nil) -} - -// --- Reconciliation (called on startup, not via NATS) --- - -// reconcileVPC creates the OVN logical router for a VPC if it doesn't exist. -func (h *TopologyHandler) reconcileVPC(ctx context.Context, vpcId, cidr string) error { - routerName := "vpc-" + vpcId - - lr := &nbdb.LogicalRouter{ - Name: routerName, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": vpcId, - "spinifex:cidr": cidr, - }, - } - if err := h.ovn.CreateLogicalRouter(ctx, lr); err != nil { - return fmt.Errorf("create router %s: %w", routerName, err) - } - - slog.Info("vpcd reconcile: created VPC router", "router", routerName, "vpc_id", vpcId) - return nil -} - -// reconcileSubnet creates the OVN logical switch, router port, and DHCP options for a subnet. -func (h *TopologyHandler) reconcileSubnet(ctx context.Context, subnetId, vpcId, cidr string) error { - switchName := "subnet-" + subnetId - routerName := "vpc-" + vpcId - routerPortName := "rtr-" + subnetId - switchRouterPortName := "rtr-port-" + subnetId - - gwIP, mask, err := subnetGateway(cidr) - if err != nil { - return fmt.Errorf("compute gateway for %s: %w", cidr, err) - } - gwCIDR := fmt.Sprintf("%s/%d", gwIP, mask) - routerMAC := generateMAC(subnetId) - - // 1. Create LogicalSwitch - ls := &nbdb.LogicalSwitch{ - Name: switchName, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": subnetId, - "spinifex:vpc_id": vpcId, - }, - } - if err := h.ovn.CreateLogicalSwitch(ctx, ls); err != nil { - return fmt.Errorf("create switch %s: %w", switchName, err) - } - - // 2. Create LogicalRouterPort - lrp := &nbdb.LogicalRouterPort{ - Name: routerPortName, - MAC: routerMAC, - Networks: []string{gwCIDR}, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": subnetId, - "spinifex:vpc_id": vpcId, - }, - } - if err := h.ovn.CreateLogicalRouterPort(ctx, routerName, lrp); err != nil { - _ = h.ovn.DeleteLogicalSwitch(ctx, switchName) - return fmt.Errorf("create router port %s: %w", routerPortName, err) - } - - // 3. Create LogicalSwitchPort (type=router) - lsp := &nbdb.LogicalSwitchPort{ - Name: switchRouterPortName, - Type: "router", - Addresses: []string{"router"}, - Options: map[string]string{"router-port": routerPortName}, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": subnetId, - "spinifex:vpc_id": vpcId, - }, - } - if err := h.ovn.CreateLogicalSwitchPort(ctx, switchName, lsp); err != nil { - _ = h.ovn.DeleteLogicalRouterPort(ctx, routerName, routerPortName) - _ = h.ovn.DeleteLogicalSwitch(ctx, switchName) - return fmt.Errorf("create switch router port %s: %w", switchRouterPortName, err) - } - - // 4. Create DHCP options - dhcpOpts := &nbdb.DHCPOptions{ - CIDR: cidr, - Options: map[string]string{ - "server_id": gwIP, - "server_mac": routerMAC, - "lease_time": "3600", - "router": gwIP, - "dns_server": h.dnsServer(), - "mtu": "1442", - }, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": subnetId, - "spinifex:vpc_id": vpcId, - }, - } - if _, err := h.ovn.CreateDHCPOptions(ctx, dhcpOpts); err != nil { - slog.Warn("vpcd reconcile: failed to create DHCP options (non-fatal)", "cidr", cidr, "err", err) - } - - slog.Info("vpcd reconcile: created subnet topology", - "switch", switchName, "router_port", routerPortName, "gateway", gwCIDR) - return nil -} - -// reconcileIGW ensures the OVN external switch, localnet port, gateway router -// port, router-bridging LSP, default route, and gateway chassis bindings exist -// for a VPC's internet gateway. Every step is Get-then-Create idempotent, with -// no cascade rollback on failure: partial topology is safe — the next -// reconcile pass heals whatever is missing. -func (h *TopologyHandler) reconcileIGW(ctx context.Context, vpcId, igwId string) error { - routerName := "vpc-" + vpcId - extSwitchName := "ext-" + vpcId - extPortName := "ext-port-" + vpcId - gwPortName := "gw-" + vpcId - switchGWPortName := "gw-port-" + vpcId - - pool := h.findExternalPool("", "") - wanGateway := "169.254.0.2" - if pool != nil { - wanGateway = pool.Gateway - } - - // 1. External logical switch - if _, err := h.ovn.GetLogicalSwitch(ctx, extSwitchName); err != nil { - extIDs := map[string]string{ - "spinifex:vpc_id": vpcId, - "spinifex:role": "external", - } - if igwId != "" { - extIDs["spinifex:igw_id"] = igwId - } - if err := h.ovn.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{ - Name: extSwitchName, - ExternalIDs: extIDs, - }); err != nil { - return fmt.Errorf("create external switch %s: %w", extSwitchName, err) - } - } - - // 2. Localnet port. ensureLocalnetOptions runs unconditionally — it is a - // read-before-write no-op when the options are already correct, and the - // recovery path (port exists with stale/missing options) needs it. - if _, err := h.ovn.GetLogicalSwitchPort(ctx, extPortName); err != nil { - portExtIDs := map[string]string{"spinifex:vpc_id": vpcId} - if igwId != "" { - portExtIDs["spinifex:igw_id"] = igwId - } - opts := map[string]string{"network_name": "external"} - if h.useCentralizedNAT() { - opts["nat-addresses"] = "router" - } - if err := h.ovn.CreateLogicalSwitchPort(ctx, extSwitchName, &nbdb.LogicalSwitchPort{ - Name: extPortName, - Type: "localnet", - Addresses: []string{"unknown"}, - Options: opts, - ExternalIDs: portExtIDs, - }); err != nil { - return fmt.Errorf("create localnet port %s: %w", extPortName, err) - } - } - if err := h.ensureLocalnetOptions(ctx, extPortName); err != nil { - return fmt.Errorf("retrofit localnet options %s: %w", extPortName, err) - } - - // 3. Gateway router port. expectedGatewayPortNetwork is only consulted - // when we actually create the LRP — RetrofitAllGatewayPortNetworks - // handles drift on pre-existing ports separately. - if _, err := h.ovn.GetLogicalRouterPort(ctx, gwPortName); err != nil { - gatewayNetwork := gatewayPortNetwork - gwLrpIP := "" - if pool != nil && h.useCentralizedNAT() { - network, ip, allocErr := h.expectedGatewayPortNetwork(ctx, vpcId) - if allocErr != nil { - return fmt.Errorf("allocate gw LRP IP for %s: %w", vpcId, allocErr) - } - gwLrpIP = ip - gatewayNetwork = network - } - lrpExtIDs := map[string]string{ - "spinifex:vpc_id": vpcId, - "spinifex:role": "gateway", - } - if gwLrpIP != "" { - lrpExtIDs[gatewayIPExtID] = gwLrpIP - } - if err := h.ovn.CreateLogicalRouterPort(ctx, routerName, &nbdb.LogicalRouterPort{ - Name: gwPortName, - MAC: generateMAC("gw-" + vpcId), - Networks: []string{gatewayNetwork}, - ExternalIDs: lrpExtIDs, - }); err != nil { - return fmt.Errorf("create gateway router port %s: %w", gwPortName, err) - } - } - - // 4. Switch port connecting external switch to router - if _, err := h.ovn.GetLogicalSwitchPort(ctx, switchGWPortName); err != nil { - if err := h.ovn.CreateLogicalSwitchPort(ctx, extSwitchName, &nbdb.LogicalSwitchPort{ - Name: switchGWPortName, - Type: "router", - Addresses: []string{"router"}, - Options: map[string]string{"router-port": gwPortName}, - ExternalIDs: map[string]string{"spinifex:vpc_id": vpcId}, - }); err != nil { - return fmt.Errorf("create switch gateway port %s: %w", switchGWPortName, err) - } - } - - // 5. No blanket SNAT — per-VM dnat_and_snat rules handle public instances. - // See handleIGWAttach comment for rationale (AWS parity). - - // 6. Default route. AddStaticRoute is non-idempotent (every retry leaves - // a fresh duplicate row), so look up first via FindStaticRoute. An - // existing row with mismatched nexthop/output is treated as an operator - // override and left alone. - existing, err := h.ovn.FindStaticRoute(ctx, routerName, "0.0.0.0/0") - if err != nil { - slog.Warn("vpcd reconcile: failed to query default route", "router", routerName, "err", err) - } - switch { - case existing == nil && err == nil: - if addErr := h.ovn.AddStaticRoute(ctx, routerName, &nbdb.LogicalRouterStaticRoute{ - IPPrefix: "0.0.0.0/0", - Nexthop: wanGateway, - OutputPort: &gwPortName, - ExternalIDs: map[string]string{"spinifex:vpc_id": vpcId}, - }); addErr != nil { - slog.Warn("vpcd reconcile: failed to add default route", "err", addErr) - } - case existing != nil: - existingPort := "" - if existing.OutputPort != nil { - existingPort = *existing.OutputPort - } - if existing.Nexthop != wanGateway || existingPort != gwPortName { - slog.Warn("vpcd reconcile: default route differs from expected, leaving existing entry in place", - "router", routerName, - "existing_nexthop", existing.Nexthop, "want_nexthop", wanGateway, - "existing_output_port", existingPort, "want_output_port", gwPortName) - } - } - - // 7. Schedule gateway chassis (already idempotent via SetGatewayChassis). - for i, chassis := range h.chassisNames { - priority := max(20-(i*5), 1) - if err := h.ovn.SetGatewayChassis(ctx, gwPortName, chassis, priority); err != nil { - slog.Warn("vpcd reconcile: failed to set gateway chassis", "chassis", chassis, "err", err) - } - } - - slog.Info("vpcd reconcile: ensured IGW topology", - "ext_switch", extSwitchName, "gw_port", gwPortName, "wan_gateway", wanGateway) - return nil -} - -// reconcileGatewayChassis brings every gateway LRP's HA-scheduling state into -// agreement with the live SBDB chassis list. Two-phase: -// -// 1. Delete Gateway_Chassis rows whose chassis_name no longer matches any -// entry in validNames. These rows are inevitable when the OVS system-id -// changes across a reboot (mulga-999): NB still references the old -// chassis-* name, but the SB Chassis row is now under a UUID, so OVN -// can't resolve the binding and cr-gw* port-bindings sit empty. -// -// 2. Re-bind every LRP tagged spinifex:role=gateway against validNames via -// the idempotent SetGatewayChassis. Existing rows with matching priority -// are no-ops; mismatched priorities are mutated; missing rows are -// created. -// -// Called as the first step of ReconcileFromKV so chassis state is correct -// before any reconcileVPC / reconcileIGW work. Failures are logged and -// returned but do not abort the rest of the reconcile loop — partial -// progress is better than none. -func (h *TopologyHandler) reconcileGatewayChassis(ctx context.Context, validNames []string) error { - valid := make(map[string]struct{}, len(validNames)) - for _, n := range validNames { - valid[n] = struct{}{} - } - - rows, err := h.ovn.ListGatewayChassis(ctx) - if err != nil { - return fmt.Errorf("list gateway_chassis: %w", err) - } - for _, gc := range rows { - if _, ok := valid[gc.ChassisName]; ok { - continue - } - // Gateway_Chassis.Name is "lrpName-chassisName" (see SetGatewayChassis). - // The chassis suffix is unique because chassis names are globally - // unique; trimming it leaves the owning LRP name. - lrpName := strings.TrimSuffix(gc.Name, "-"+gc.ChassisName) - if err := h.ovn.DeleteGatewayChassis(ctx, lrpName, gc.UUID); err != nil { - slog.Warn("vpcd: failed to delete stale gateway_chassis", - "row", gc.Name, "chassis_name", gc.ChassisName, "err", err) - continue - } - slog.Warn("vpcd: deleted stale gateway_chassis", - "row", gc.Name, "chassis_name", gc.ChassisName, "lrp", lrpName) - } - - if len(validNames) == 0 { - return nil - } - - lrps, err := h.ovn.ListLogicalRouterPorts(ctx) - if err != nil { - return fmt.Errorf("list logical_router_port: %w", err) - } - for _, lrp := range lrps { - if lrp.ExternalIDs["spinifex:role"] != "gateway" { - continue - } - for i, name := range validNames { - priority := max(20-(i*5), 1) - if err := h.ovn.SetGatewayChassis(ctx, lrp.Name, name, priority); err != nil { - slog.Warn("vpcd: failed to set gateway chassis on rebind", - "lrp", lrp.Name, "chassis", name, "priority", priority, "err", err) - } - } - } - return nil -} - -// --- Helpers --- - -// gwLrpRange returns the per-VPC gateway LRP IP range for a pool. Priority: -// -// 1. Explicit pool.GwLrpRangeStart/End set in spinifex.toml. -// 2. Auto-derived from pool.Gateway + pool.PrefixLen — last 16 host IPs of -// the WAN subnet (broadcast - 16 .. broadcast - 1). When that range -// overlaps the per-VM EIP range (RangeStart..RangeEnd), shift to the -// 16 IPs immediately below RangeStart instead. -// -// Returns ok=false only when the pool is missing or the gateway/prefix is -// unparseable — link-local has no role here, the WAN-subnet IP is the only -// thing the upstream router will ARP-resolve. -func gwLrpRange(pool *ExternalPoolConfig) (start, end net.IP, prefix int, ok bool) { - if pool == nil { - return nil, nil, 0, false - } - prefix = pool.PrefixLen - if prefix <= 0 || prefix > 32 { - prefix = 24 - } - - // 1. Explicit operator config wins. - if pool.GwLrpRangeStart != "" || pool.GwLrpRangeEnd != "" { - s := net.ParseIP(pool.GwLrpRangeStart).To4() - e := net.ParseIP(pool.GwLrpRangeEnd).To4() - if s != nil && e != nil && ipv4ToUint32(s) <= ipv4ToUint32(e) { - return s, e, prefix, true - } - slog.Warn("vpcd: invalid explicit gw_lrp_range, attempting auto-derive", - "pool", pool.Name, "start", pool.GwLrpRangeStart, "end", pool.GwLrpRangeEnd) - } - - // 2. Auto-derive from subnet. - gw := net.ParseIP(pool.Gateway).To4() - if gw == nil { - return nil, nil, 0, false - } - mask := net.CIDRMask(prefix, 32) - network := gw.Mask(mask) - bcast := make(net.IP, 4) - for i := range 4 { - bcast[i] = network[i] | ^mask[i] - } - bcastU := ipv4ToUint32(bcast) - if bcastU < 17 { - return nil, nil, 0, false - } - autoEndU := bcastU - 1 // skip broadcast itself - autoStartU := bcastU - 16 // 16-IP range - - // Shift below per-VM EIP range when overlap. - if pool.RangeStart != "" && pool.RangeEnd != "" { - rs := net.ParseIP(pool.RangeStart).To4() - re := net.ParseIP(pool.RangeEnd).To4() - if rs != nil && re != nil { - rsU := ipv4ToUint32(rs) - reU := ipv4ToUint32(re) - if autoEndU >= rsU && autoStartU <= reU { - if rsU < 17 { - return nil, nil, 0, false - } - autoEndU = rsU - 1 - autoStartU = rsU - 16 - } - } - } - - // Clamp inside the subnet (network+1 .. broadcast-1) — never hand out - // the network address, the broadcast, or an off-subnet IP. - netU := ipv4ToUint32(network) - if autoStartU <= netU { - autoStartU = netU + 1 - } - if autoEndU >= bcastU { - autoEndU = bcastU - 1 - } - if autoStartU > autoEndU { - return nil, nil, 0, false - } - - // Guard the gateway IP itself — never give a VPC the upstream nexthop. - gwU := ipv4ToUint32(gw) - if gwU >= autoStartU && gwU <= autoEndU { - // Gateway is inside the auto range — shrink to exclude it. - switch gwU { - case autoStartU: - autoStartU++ - case autoEndU: - autoEndU-- - default: - // Gateway in the middle: prefer the upper half. - autoStartU = gwU + 1 - } - if autoStartU > autoEndU { - return nil, nil, 0, false - } - } - - return uint32ToIPv4(autoStartU), uint32ToIPv4(autoEndU), prefix, true -} - -// gwLrpClientID is the DHCP client-id used for a VPC's gateway LRP lease. -// Stable across reboots — DHCPManager reuses the same lease on idempotent -// re-attach so the LRP IP doesn't change unless the upstream server -// reassigns it. -func gwLrpClientID(vpcId string) string { return "gw-lrp-" + vpcId } - -// allocateGatewayLRPIPViaDHCP requests a DHCP lease from vpcd's DHCPManager -// for the gateway LRP of vpcId. The handler-side acquire is idempotent on -// client-id so retries return the same IP without a fresh DORA. Prefix -// comes from the lease's SubnetMask, falling back to pool.PrefixLen when -// the server omits option 1 (rare; should not happen in practice). -func (h *TopologyHandler) allocateGatewayLRPIPViaDHCP(ctx context.Context, vpcId string, pool *ExternalPoolConfig) (ip string, prefix int, err error) { - _ = ctx - if h.nc == nil { - return "", 0, fmt.Errorf("vpcd: no NATS conn for DHCP gw LRP allocation (vpc %s, pool %s)", vpcId, pool.Name) - } - clientID := gwLrpClientID(vpcId) - hostname := "spinifex-gw-" + vpcId - vendorClass := "mulga-spinifex-gw-lrp" - lease, dhcpErr := dhcp.RequestAcquire(h.nc, pool.DhcpBindBridge, clientID, hostname, vendorClass, pool.Name, "") - if dhcpErr != nil { - return "", 0, fmt.Errorf("dhcp acquire gw LRP IP for vpc %s: %w", vpcId, dhcpErr) - } - prefix = prefixFromMask(lease.SubnetMask) - if prefix == 0 { - prefix = pool.PrefixLen - } - if prefix <= 0 || prefix > 32 { - return "", 0, fmt.Errorf("dhcp gw LRP for vpc %s: cannot determine prefix (mask=%q pool prefix=%d)", vpcId, lease.SubnetMask, pool.PrefixLen) - } - return lease.IP, prefix, nil -} - -// releaseGatewayLRPLease asks vpcd's DHCPManager to drop the lease held for -// vpcId's gateway LRP. Best-effort — log on failure but never block IGW -// detach, since the upstream server will eventually expire the lease on -// its own. -func (h *TopologyHandler) releaseGatewayLRPLease(vpcId string) { - if h.nc == nil { - return - } - if err := dhcp.RequestRelease(h.nc, gwLrpClientID(vpcId)); err != nil { - slog.Warn("vpcd: dhcp release for gw LRP failed", "vpc_id", vpcId, "err", err) - } -} - -// prefixFromMask converts a dotted-decimal mask ("255.255.255.0") to a -// prefix length. Returns 0 when the mask is empty or unparseable. -func prefixFromMask(mask string) int { - if mask == "" { - return 0 - } - ip := net.ParseIP(mask).To4() - if ip == nil { - return 0 - } - ones, bits := net.IPMask(ip).Size() - if bits != 32 { - return 0 - } - return ones -} - -// allocateGatewayLRPIP picks the next free IP in pool.GwLrpRange for the -// given VPC, scanning every existing LRP's spinifex:gateway_ip external_id -// to compute the used set. If gw- already has an allocation the -// existing IP is returned (idempotent reconcile). Returns ok=false when -// the pool has no gw_lrp_range — caller falls back to link-local. -func (h *TopologyHandler) allocateGatewayLRPIP(ctx context.Context, vpcId string, pool *ExternalPoolConfig) (ip string, prefix int, ok bool, err error) { - start, end, prefix, ok := gwLrpRange(pool) - if !ok { - return "", 0, false, nil - } - - gwPortName := "gw-" + vpcId - lrps, err := h.ovn.ListLogicalRouterPorts(ctx) - if err != nil { - return "", 0, false, fmt.Errorf("list LRPs for gw IP allocation: %w", err) - } - used := make(map[uint32]struct{}, len(lrps)) - for _, lrp := range lrps { - existing := lrp.ExternalIDs[gatewayIPExtID] - if existing == "" { - continue - } - if lrp.Name == gwPortName { - // Idempotent: already allocated to this VPC. - return existing, prefix, true, nil - } - if v := net.ParseIP(existing).To4(); v != nil { - used[ipv4ToUint32(v)] = struct{}{} - } - } - - startU := ipv4ToUint32(start) - endU := ipv4ToUint32(end) - for n := startU; n <= endU; n++ { - if _, taken := used[n]; taken { - continue - } - return uint32ToIPv4(n).String(), prefix, true, nil - } - return "", 0, false, fmt.Errorf("gw_lrp_range exhausted for pool %q (%s-%s)", pool.Name, pool.GwLrpRangeStart, pool.GwLrpRangeEnd) -} - -func ipv4ToUint32(ip net.IP) uint32 { - v := ip.To4() - return uint32(v[0])<<24 | uint32(v[1])<<16 | uint32(v[2])<<8 | uint32(v[3]) -} - -func uint32ToIPv4(n uint32) net.IP { - return net.IPv4(byte(n>>24&0xff), byte(n>>16&0xff), byte(n>>8&0xff), byte(n&0xff)).To4() -} - -// subnetGateway computes the gateway IP (.1) from a CIDR string. -// Returns the gateway IP string and the prefix length. -func subnetGateway(cidr string) (string, int, error) { - ip, ipNet, err := net.ParseCIDR(cidr) - if err != nil { - return "", 0, fmt.Errorf("parse CIDR %q: %w", cidr, err) - } - - // Use the network address, increment last octet to .1 - gw := ipNet.IP.To4() - if gw == nil { - return "", 0, fmt.Errorf("only IPv4 supported, got %s", ip) - } - gw = make(net.IP, len(ipNet.IP.To4())) - copy(gw, ipNet.IP.To4()) - gw[3]++ - - ones, _ := ipNet.Mask.Size() - return gw.String(), ones, nil -} - -// generateMAC creates a deterministic locally-administered unicast MAC -// from a resource ID via utils.HashMAC. Inputs are vpcd-owned ids -// (subnet-..., gw-vpc-..., eni-...) which are unique on their own. -func generateMAC(resourceID string) string { - return utils.HashMAC(resourceID) -} - -// respond sends a simple JSON response to a NATS request. -func respond(msg *nats.Msg, err error) { - if msg.Reply == "" { - return // fire-and-forget, no reply expected - } - - type response struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - } - - resp := response{Success: true} - if err != nil { - resp.Success = false - resp.Error = err.Error() - } - - data, marshalErr := json.Marshal(resp) - if marshalErr != nil { - slog.Error("vpcd: failed to marshal NATS response", "err", marshalErr) - data = []byte(`{"success":false,"error":"internal marshal failure"}`) - } - if err := msg.Respond(data); err != nil { - slog.Error("vpcd: failed to respond to NATS request", "err", err) - } -} - -// handleAddNATGateway creates an OVN SNAT rule for a private subnet via a NAT Gateway. -// The SNAT rule rewrites source IPs from the private subnet CIDR to the NAT GW's public IP. -func (h *TopologyHandler) handleAddNATGateway(msg *nats.Msg) { - var evt NATGatewayEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.add-nat-gateway event", "err", err) - return - } - - slog.Info("vpcd: adding NAT Gateway SNAT rule", - "vpcId", evt.VpcId, "natGatewayId", evt.NatGatewayId, - "publicIp", evt.PublicIp, "subnetCidr", evt.SubnetCidr) - - ctx := context.Background() - routerName := "vpc-" + evt.VpcId - - snatRule := &nbdb.NAT{ - Type: "snat", - ExternalIP: evt.PublicIp, - LogicalIP: evt.SubnetCidr, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": evt.VpcId, - "spinifex:nat_gateway_id": evt.NatGatewayId, - }, - } - - if err := h.ovn.AddNAT(ctx, routerName, snatRule); err != nil { - slog.Error("vpcd: failed to add NAT Gateway SNAT rule", - "router", routerName, "publicIp", evt.PublicIp, - "subnetCidr", evt.SubnetCidr, "err", err) - return - } - - slog.Info("vpcd: NAT Gateway SNAT rule added", - "router", routerName, "publicIp", evt.PublicIp, "subnetCidr", evt.SubnetCidr) -} - -// handleDeleteNATGateway removes the OVN SNAT rule for a private subnet's NAT Gateway. -func (h *TopologyHandler) handleDeleteNATGateway(msg *nats.Msg) { - var evt NATGatewayEvent - if err := json.Unmarshal(msg.Data, &evt); err != nil { - slog.Error("vpcd: failed to unmarshal vpc.delete-nat-gateway event", "err", err) - return - } - - slog.Info("vpcd: removing NAT Gateway SNAT rule", - "vpcId", evt.VpcId, "natGatewayId", evt.NatGatewayId, "subnetCidr", evt.SubnetCidr) - - ctx := context.Background() - routerName := "vpc-" + evt.VpcId - - if err := h.ovn.DeleteNAT(ctx, routerName, "snat", evt.SubnetCidr); err != nil { - slog.Warn("vpcd: failed to delete NAT Gateway SNAT rule", - "router", routerName, "subnetCidr", evt.SubnetCidr, "err", err) - return - } - - slog.Info("vpcd: NAT Gateway SNAT rule removed", - "router", routerName, "subnetCidr", evt.SubnetCidr) -} diff --git a/spinifex/services/vpcd/topology_test.go b/spinifex/services/vpcd/topology_test.go deleted file mode 100644 index ec4075c0c..000000000 --- a/spinifex/services/vpcd/topology_test.go +++ /dev/null @@ -1,4187 +0,0 @@ -package vpcd - -import ( - "context" - "encoding/json" - "fmt" - "net" - "slices" - "strings" - "testing" - "time" - - "github.com/mulgadc/spinifex/spinifex/services/vpcd/dhcp" - "github.com/mulgadc/spinifex/spinifex/services/vpcd/nbdb" - "github.com/mulgadc/spinifex/spinifex/testutil" - "github.com/mulgadc/spinifex/spinifex/types" - "github.com/nats-io/nats-server/v2/server" - "github.com/nats-io/nats.go" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// startTestNATS starts an embedded NATS server for testing. -func startTestNATS(t *testing.T) (*server.Server, *nats.Conn) { - t.Helper() - return testutil.StartTestNATS(t) -} - -func TestTopologyHandler_VPCCreate(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - evt := VPCEvent{VpcId: "vpc-abc123", CidrBlock: "10.0.0.0/16", VNI: 100} - data, _ := json.Marshal(evt) - - resp, err := nc.Request(TopicVPCCreate, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.create: %v", err) - } - - var result struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - } - if err := json.Unmarshal(resp.Data, &result); err != nil { - t.Fatalf("unmarshal response: %v", err) - } - if !result.Success { - t.Fatalf("vpc.create failed: %s", result.Error) - } - - // Verify router was created in OVN - ctx := context.Background() - lr, err := mock.GetLogicalRouter(ctx, "vpc-vpc-abc123") - if err != nil { - t.Fatalf("expected logical router: %v", err) - } - if lr.ExternalIDs["spinifex:vpc_id"] != "vpc-abc123" { - t.Errorf("expected vpc_id external_id, got %v", lr.ExternalIDs) - } - if lr.ExternalIDs["spinifex:vni"] != "100" { - t.Errorf("expected vni external_id=100, got %v", lr.ExternalIDs["spinifex:vni"]) - } - if lr.ExternalIDs["spinifex:cidr"] != "10.0.0.0/16" { - t.Errorf("expected cidr external_id=10.0.0.0/16, got %v", lr.ExternalIDs["spinifex:cidr"]) - } -} - -func TestTopologyHandler_VPCDelete(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create a router - _ = mock.CreateLogicalRouter(ctx, nbdbLogicalRouter("vpc-vpc-xyz", "vpc-xyz")) - - // Delete via NATS - evt := VPCEvent{VpcId: "vpc-xyz"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicVPCDelete, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.delete: %v", err) - } - - var result struct { - Success bool `json:"Success"` - } - _ = json.Unmarshal(resp.Data, &result) - if !result.Success { - t.Fatal("vpc.delete failed") - } - - // Verify router is gone - _, err = mock.GetLogicalRouter(ctx, "vpc-vpc-xyz") - if err == nil { - t.Error("expected router to be deleted") - } -} - -func TestTopologyHandler_SubnetCreate(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create the VPC router - _ = mock.CreateLogicalRouter(ctx, nbdbLogicalRouter("vpc-vpc-sub1", "vpc-sub1")) - - // Create subnet - evt := SubnetEvent{SubnetId: "subnet-aaa", VpcId: "vpc-sub1", CidrBlock: "10.0.1.0/24"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicSubnetCreate, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.create-subnet: %v", err) - } - - var result struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - } - _ = json.Unmarshal(resp.Data, &result) - if !result.Success { - t.Fatalf("vpc.create-subnet failed: %s", result.Error) - } - - // Verify logical switch created - ls, err := mock.GetLogicalSwitch(ctx, "subnet-subnet-aaa") - if err != nil { - t.Fatalf("expected logical switch: %v", err) - } - if ls.ExternalIDs["spinifex:subnet_id"] != "subnet-aaa" { - t.Errorf("expected subnet_id external_id, got %v", ls.ExternalIDs) - } - - // Verify router port created - lr, err := mock.GetLogicalRouter(ctx, "vpc-vpc-sub1") - if err != nil { - t.Fatalf("expected router: %v", err) - } - if len(lr.Ports) != 1 { - t.Errorf("expected 1 router port, got %d", len(lr.Ports)) - } - - // Verify switch has 1 port (the router port) - if len(ls.Ports) != 1 { - t.Errorf("expected 1 switch port, got %d", len(ls.Ports)) - } - - // Verify DHCP options created - dhcpOpts, err := mock.FindDHCPOptionsByCIDR(ctx, "10.0.1.0/24") - if err != nil { - t.Fatalf("expected DHCP options: %v", err) - } - if dhcpOpts.Options["router"] != "10.0.1.1" { - t.Errorf("expected router=10.0.1.1, got %s", dhcpOpts.Options["router"]) - } - if dhcpOpts.Options["lease_time"] != "3600" { - t.Errorf("expected lease_time=3600, got %s", dhcpOpts.Options["lease_time"]) - } - if dhcpOpts.Options["mtu"] != "1442" { - t.Errorf("expected mtu=1442, got %s", dhcpOpts.Options["mtu"]) - } -} - -func TestTopologyHandler_SubnetDelete(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Setup: create VPC router + subnet topology manually - _ = mock.CreateLogicalRouter(ctx, nbdbLogicalRouter("vpc-vpc-del", "vpc-del")) - _ = mock.CreateLogicalSwitch(ctx, nbdbLogicalSwitch("subnet-subnet-del", "subnet-del", "vpc-del")) - _ = mock.CreateLogicalRouterPort(ctx, "vpc-vpc-del", nbdbLogicalRouterPort("rtr-subnet-del", "subnet-del", "vpc-del")) - _ = mock.CreateLogicalSwitchPort(ctx, "subnet-subnet-del", nbdbLogicalSwitchPortRouter("rtr-port-subnet-del", "rtr-subnet-del", "subnet-del", "vpc-del")) - _, _ = mock.CreateDHCPOptions(ctx, nbdbDHCPOptions("10.0.2.0/24", "subnet-del", "vpc-del")) - - // Delete subnet via NATS - evt := SubnetEvent{SubnetId: "subnet-del", VpcId: "vpc-del", CidrBlock: "10.0.2.0/24"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicSubnetDelete, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.delete-subnet: %v", err) - } - - var result struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - } - _ = json.Unmarshal(resp.Data, &result) - if !result.Success { - t.Fatalf("vpc.delete-subnet failed: %s", result.Error) - } - - // Verify switch is deleted - _, err = mock.GetLogicalSwitch(ctx, "subnet-subnet-del") - if err == nil { - t.Error("expected switch to be deleted") - } - - // Verify DHCP options are deleted - _, err = mock.FindDHCPOptionsByCIDR(ctx, "10.0.2.0/24") - if err == nil { - t.Error("expected DHCP options to be deleted") - } - - // Verify router still exists (only subnet topology deleted, not VPC) - _, err = mock.GetLogicalRouter(ctx, "vpc-vpc-del") - if err != nil { - t.Error("expected VPC router to still exist") - } -} - -func TestTopologyHandler_FullLifecycle(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // 1. Create VPC - vpcEvt := VPCEvent{VpcId: "vpc-full", CidrBlock: "10.0.0.0/16", VNI: 200} - data, _ := json.Marshal(vpcEvt) - resp, _ := nc.Request(TopicVPCCreate, data, 5_000_000_000) - assertSuccess(t, resp, "create VPC") - - // 2. Create Subnet 1 - subEvt1 := SubnetEvent{SubnetId: "subnet-a", VpcId: "vpc-full", CidrBlock: "10.0.1.0/24"} - data, _ = json.Marshal(subEvt1) - resp, _ = nc.Request(TopicSubnetCreate, data, 5_000_000_000) - assertSuccess(t, resp, "create subnet-a") - - // 3. Create Subnet 2 - subEvt2 := SubnetEvent{SubnetId: "subnet-b", VpcId: "vpc-full", CidrBlock: "10.0.2.0/24"} - data, _ = json.Marshal(subEvt2) - resp, _ = nc.Request(TopicSubnetCreate, data, 5_000_000_000) - assertSuccess(t, resp, "create subnet-b") - - // Verify: 1 router with 2 ports, 2 switches, 2 DHCP option sets - routers, _ := mock.ListLogicalRouters(ctx) - if len(routers) != 1 { - t.Errorf("expected 1 router, got %d", len(routers)) - } - switches, _ := mock.ListLogicalSwitches(ctx) - if len(switches) != 2 { - t.Errorf("expected 2 switches, got %d", len(switches)) - } - dhcpList, _ := mock.ListDHCPOptions(ctx) - if len(dhcpList) != 2 { - t.Errorf("expected 2 DHCP option sets, got %d", len(dhcpList)) - } - - // 4. Delete Subnet 1 - data, _ = json.Marshal(subEvt1) - resp, _ = nc.Request(TopicSubnetDelete, data, 5_000_000_000) - assertSuccess(t, resp, "delete subnet-a") - - switches, _ = mock.ListLogicalSwitches(ctx) - if len(switches) != 1 { - t.Errorf("expected 1 switch after delete, got %d", len(switches)) - } - - // 5. Delete Subnet 2 - data, _ = json.Marshal(subEvt2) - resp, _ = nc.Request(TopicSubnetDelete, data, 5_000_000_000) - assertSuccess(t, resp, "delete subnet-b") - - // 6. Delete VPC - delEvt := VPCEvent{VpcId: "vpc-full"} - data, _ = json.Marshal(delEvt) - resp, _ = nc.Request(TopicVPCDelete, data, 5_000_000_000) - assertSuccess(t, resp, "delete VPC") - - // Verify everything is gone - routers, _ = mock.ListLogicalRouters(ctx) - if len(routers) != 0 { - t.Errorf("expected 0 routers, got %d", len(routers)) - } - switches, _ = mock.ListLogicalSwitches(ctx) - if len(switches) != 0 { - t.Errorf("expected 0 switches, got %d", len(switches)) - } - dhcpList, _ = mock.ListDHCPOptions(ctx) - if len(dhcpList) != 0 { - t.Errorf("expected 0 DHCP options, got %d", len(dhcpList)) - } -} - -func TestTopologyHandler_VPCDeleteCascade(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Create VPC + subnet directly in mock (simulating pre-existing state) - _ = mock.CreateLogicalRouter(ctx, nbdbLogicalRouter("vpc-vpc-cas", "vpc-cas")) - _ = mock.CreateLogicalSwitch(ctx, nbdbLogicalSwitch("subnet-sub-cas", "sub-cas", "vpc-cas")) - _, _ = mock.CreateDHCPOptions(ctx, nbdbDHCPOptions("10.0.3.0/24", "sub-cas", "vpc-cas")) - - // Delete VPC should cascade-delete switches and DHCP - evt := VPCEvent{VpcId: "vpc-cas"} - data, _ := json.Marshal(evt) - resp, _ := nc.Request(TopicVPCDelete, data, 5_000_000_000) - assertSuccess(t, resp, "cascade delete VPC") - - switches, _ := mock.ListLogicalSwitches(ctx) - if len(switches) != 0 { - t.Errorf("expected 0 switches after cascade delete, got %d", len(switches)) - } - dhcpList, _ := mock.ListDHCPOptions(ctx) - if len(dhcpList) != 0 { - t.Errorf("expected 0 DHCP options after cascade delete, got %d", len(dhcpList)) - } -} - -func TestSubnetGateway(t *testing.T) { - tests := []struct { - cidr string - wantIP string - wantMask int - wantErr bool - }{ - {"10.0.1.0/24", "10.0.1.1", 24, false}, - {"192.168.0.0/16", "192.168.0.1", 16, false}, - {"172.16.0.0/20", "172.16.0.1", 20, false}, - {"invalid", "", 0, true}, - } - - for _, tt := range tests { - t.Run(tt.cidr, func(t *testing.T) { - ip, mask, err := subnetGateway(tt.cidr) - if tt.wantErr { - if err == nil { - t.Error("expected error") - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ip != tt.wantIP { - t.Errorf("expected IP %s, got %s", tt.wantIP, ip) - } - if mask != tt.wantMask { - t.Errorf("expected mask %d, got %d", tt.wantMask, mask) - } - }) - } -} - -func TestGenerateMAC(t *testing.T) { - mac1 := generateMAC("subnet-aaa") - mac2 := generateMAC("subnet-bbb") - - // Must be a valid locally-administered unicast MAC. First octet is - // hash-derived, not the literal "02:00:00" of the old 24-bit impl. - hw, err := net.ParseMAC(mac1) - if err != nil { - t.Fatalf("invalid MAC %q: %v", mac1, err) - } - if hw[0]&0x03 != 0x02 { - t.Errorf("expected unicast+LAA bits on first octet, got %#x", hw[0]) - } - - // Different inputs produce different MACs - if mac1 == mac2 { - t.Error("expected different MACs for different inputs") - } - - // Same input produces same MAC (deterministic) - mac1b := generateMAC("subnet-aaa") - if mac1 != mac1b { - t.Error("expected deterministic MAC") - } - - // Output must use full 46-bit hash space, not the old 24-bit space that - // pinned the top 3 octets to "02:00:00". Sweep many inputs and assert - // the second octet varies — old impl always emitted 0x00 there. - seen := map[byte]struct{}{} - for i := range 1000 { - hw, err := net.ParseMAC(generateMAC(fmt.Sprintf("subnet-%d", i))) - if err != nil { - t.Fatalf("invalid MAC: %v", err) - } - seen[hw[1]] = struct{}{} - } - if len(seen) <= 1 { - t.Errorf("expected second octet to vary across inputs, got %d distinct values", len(seen)) - } -} - -func TestTopologyHandler_CreatePort(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Setup: create VPC router, subnet switch, and DHCP options - _ = mock.CreateLogicalRouter(ctx, nbdbLogicalRouter("vpc-vpc-port1", "vpc-port1")) - _ = mock.CreateLogicalSwitch(ctx, nbdbLogicalSwitch("subnet-subnet-port1", "subnet-port1", "vpc-port1")) - dhcpUUID, _ := mock.CreateDHCPOptions(ctx, nbdbDHCPOptions("10.0.1.0/24", "subnet-port1", "vpc-port1")) - - // Create port via NATS - evt := PortEvent{ - NetworkInterfaceId: "eni-aaa111", - SubnetId: "subnet-port1", - VpcId: "vpc-port1", - PrivateIpAddress: "10.0.1.4", - MacAddress: "02:00:00:11:22:33", - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicCreatePort, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.create-port: %v", err) - } - assertSuccess(t, resp, "create port") - - // Verify logical switch port created - lsp, err := mock.GetLogicalSwitchPort(ctx, "port-eni-aaa111") - if err != nil { - t.Fatalf("expected logical switch port: %v", err) - } - - // Verify addresses - expectedAddr := "02:00:00:11:22:33 10.0.1.4" - if len(lsp.Addresses) != 1 || lsp.Addresses[0] != expectedAddr { - t.Errorf("expected addresses [%s], got %v", expectedAddr, lsp.Addresses) - } - - // Verify port security - if len(lsp.PortSecurity) != 1 || lsp.PortSecurity[0] != expectedAddr { - t.Errorf("expected port_security [%s], got %v", expectedAddr, lsp.PortSecurity) - } - - // Verify DHCPv4 options - if lsp.DHCPv4Options == nil { - t.Fatal("expected DHCPv4Options to be set") - } - if *lsp.DHCPv4Options != dhcpUUID { - t.Errorf("expected DHCPv4Options UUID %s, got %s", dhcpUUID, *lsp.DHCPv4Options) - } - - // Verify external IDs - if lsp.ExternalIDs["spinifex:eni_id"] != "eni-aaa111" { - t.Errorf("expected eni_id=eni-aaa111, got %s", lsp.ExternalIDs["spinifex:eni_id"]) - } - if lsp.ExternalIDs["spinifex:subnet_id"] != "subnet-port1" { - t.Errorf("expected subnet_id=subnet-port1, got %s", lsp.ExternalIDs["spinifex:subnet_id"]) - } - if lsp.ExternalIDs["spinifex:vpc_id"] != "vpc-port1" { - t.Errorf("expected vpc_id=vpc-port1, got %s", lsp.ExternalIDs["spinifex:vpc_id"]) - } - - // Verify port was added to the switch - ls, err := mock.GetLogicalSwitch(ctx, "subnet-subnet-port1") - if err != nil { - t.Fatalf("get switch: %v", err) - } - if len(ls.Ports) != 1 { - t.Errorf("expected 1 port on switch, got %d", len(ls.Ports)) - } -} - -func TestTopologyHandler_DeletePort(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Setup: create switch and port - _ = mock.CreateLogicalSwitch(ctx, nbdbLogicalSwitch("subnet-subnet-del2", "subnet-del2", "vpc-del2")) - _ = mock.CreateLogicalSwitchPort(ctx, "subnet-subnet-del2", &nbdb.LogicalSwitchPort{ - Name: "port-eni-bbb222", - Addresses: []string{"02:00:00:44:55:66 10.0.2.4"}, - PortSecurity: []string{"02:00:00:44:55:66 10.0.2.4"}, - ExternalIDs: map[string]string{ - "spinifex:eni_id": "eni-bbb222", - "spinifex:subnet_id": "subnet-del2", - "spinifex:vpc_id": "vpc-del2", - }, - }) - - // Verify port exists before delete - _, err = mock.GetLogicalSwitchPort(ctx, "port-eni-bbb222") - if err != nil { - t.Fatalf("expected port to exist before delete: %v", err) - } - - // Delete port via NATS - evt := PortEvent{ - NetworkInterfaceId: "eni-bbb222", - SubnetId: "subnet-del2", - VpcId: "vpc-del2", - PrivateIpAddress: "10.0.2.4", - MacAddress: "02:00:00:44:55:66", - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicDeletePort, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.delete-port: %v", err) - } - assertSuccess(t, resp, "delete port") - - // Verify port is gone - _, err = mock.GetLogicalSwitchPort(ctx, "port-eni-bbb222") - if err == nil { - t.Error("expected port to be deleted") - } - - // Verify switch still exists but has no ports - ls, err := mock.GetLogicalSwitch(ctx, "subnet-subnet-del2") - if err != nil { - t.Fatalf("expected switch to still exist: %v", err) - } - if len(ls.Ports) != 0 { - t.Errorf("expected 0 ports on switch after delete, got %d", len(ls.Ports)) - } -} - -func TestTopologyHandler_CreatePortNoDHCP(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Setup: create switch but NO DHCP options - _ = mock.CreateLogicalSwitch(ctx, nbdbLogicalSwitch("subnet-subnet-nodhcp", "subnet-nodhcp", "vpc-nodhcp")) - - // Create port — should succeed but without DHCPv4Options - evt := PortEvent{ - NetworkInterfaceId: "eni-nodhcp", - SubnetId: "subnet-nodhcp", - VpcId: "vpc-nodhcp", - PrivateIpAddress: "10.0.3.4", - MacAddress: "02:00:00:77:88:99", - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicCreatePort, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.create-port: %v", err) - } - assertSuccess(t, resp, "create port without DHCP") - - // Port should exist but without DHCPv4Options - lsp, err := mock.GetLogicalSwitchPort(ctx, "port-eni-nodhcp") - if err != nil { - t.Fatalf("expected port: %v", err) - } - if lsp.DHCPv4Options != nil { - t.Errorf("expected nil DHCPv4Options when no DHCP configured, got %s", *lsp.DHCPv4Options) - } -} - -func TestTopologyHandler_PortLifecycle(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // 1. Create VPC - vpcEvt := VPCEvent{VpcId: "vpc-plc", CidrBlock: "10.0.0.0/16", VNI: 300} - data, _ := json.Marshal(vpcEvt) - resp, _ := nc.Request(TopicVPCCreate, data, 5_000_000_000) - assertSuccess(t, resp, "create VPC") - - // 2. Create Subnet - subEvt := SubnetEvent{SubnetId: "subnet-plc", VpcId: "vpc-plc", CidrBlock: "10.0.1.0/24"} - data, _ = json.Marshal(subEvt) - resp, _ = nc.Request(TopicSubnetCreate, data, 5_000_000_000) - assertSuccess(t, resp, "create subnet") - - // 3. Create two ports - portEvt1 := PortEvent{ - NetworkInterfaceId: "eni-plc-1", - SubnetId: "subnet-plc", - VpcId: "vpc-plc", - PrivateIpAddress: "10.0.1.4", - MacAddress: "02:00:00:aa:bb:01", - } - data, _ = json.Marshal(portEvt1) - resp, _ = nc.Request(TopicCreatePort, data, 5_000_000_000) - assertSuccess(t, resp, "create port 1") - - portEvt2 := PortEvent{ - NetworkInterfaceId: "eni-plc-2", - SubnetId: "subnet-plc", - VpcId: "vpc-plc", - PrivateIpAddress: "10.0.1.5", - MacAddress: "02:00:00:aa:bb:02", - } - data, _ = json.Marshal(portEvt2) - resp, _ = nc.Request(TopicCreatePort, data, 5_000_000_000) - assertSuccess(t, resp, "create port 2") - - // Verify switch has 3 ports (router port + 2 ENI ports) - ls, err := mock.GetLogicalSwitch(ctx, "subnet-subnet-plc") - if err != nil { - t.Fatalf("get switch: %v", err) - } - if len(ls.Ports) != 3 { - t.Errorf("expected 3 ports (1 router + 2 ENI), got %d", len(ls.Ports)) - } - - // 4. Delete port 1 - data, _ = json.Marshal(portEvt1) - resp, _ = nc.Request(TopicDeletePort, data, 5_000_000_000) - assertSuccess(t, resp, "delete port 1") - - // Verify switch has 2 ports now - ls, err = mock.GetLogicalSwitch(ctx, "subnet-subnet-plc") - if err != nil { - t.Fatalf("get switch after port delete: %v", err) - } - if len(ls.Ports) != 2 { - t.Errorf("expected 2 ports after delete, got %d", len(ls.Ports)) - } - - // Port 2 should still exist - _, err = mock.GetLogicalSwitchPort(ctx, "port-eni-plc-2") - if err != nil { - t.Error("expected port 2 to still exist") - } - - // Port 1 should be gone - _, err = mock.GetLogicalSwitchPort(ctx, "port-eni-plc-1") - if err == nil { - t.Error("expected port 1 to be deleted") - } -} - -func TestTopologyHandler_NilOVN(t *testing.T) { - _, nc := startTestNATS(t) - - // nil OVN client (OVN not connected) - topo := NewTopologyHandler(nil) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Should fail gracefully when OVN is nil - evt := VPCEvent{VpcId: "vpc-nil", CidrBlock: "10.0.0.0/16", VNI: 100} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicVPCCreate, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - - var result struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - } - _ = json.Unmarshal(resp.Data, &result) - if result.Success { - t.Error("expected failure when OVN is nil") - } - - // Port create should also fail gracefully - portEvt := PortEvent{ - NetworkInterfaceId: "eni-nil", - SubnetId: "subnet-nil", - VpcId: "vpc-nil", - PrivateIpAddress: "10.0.1.4", - MacAddress: "02:00:00:00:00:01", - } - data, _ = json.Marshal(portEvt) - resp, err = nc.Request(TopicCreatePort, data, 5_000_000_000) - if err != nil { - t.Fatalf("request create port: %v", err) - } - _ = json.Unmarshal(resp.Data, &result) - if result.Success { - t.Error("expected create-port failure when OVN is nil") - } - - // Port delete should also fail gracefully - resp, err = nc.Request(TopicDeletePort, data, 5_000_000_000) - if err != nil { - t.Fatalf("request delete port: %v", err) - } - _ = json.Unmarshal(resp.Data, &result) - if result.Success { - t.Error("expected delete-port failure when OVN is nil") - } -} - -func TestTopologyHandler_IGWAttach(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create VPC router - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-igw1", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-igw1", - "spinifex:cidr": "10.0.0.0/16", - }, - }) - - // Attach IGW - evt := types.IGWEvent{InternetGatewayId: "igw-test1", VpcId: "vpc-igw1"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.igw-attach: %v", err) - } - assertSuccess(t, resp, "attach IGW") - - // Verify external switch created - extSwitch, err := mock.GetLogicalSwitch(ctx, "ext-vpc-igw1") - if err != nil { - t.Fatalf("expected external switch: %v", err) - } - if extSwitch.ExternalIDs["spinifex:role"] != "external" { - t.Errorf("expected role=external, got %s", extSwitch.ExternalIDs["spinifex:role"]) - } - if extSwitch.ExternalIDs["spinifex:igw_id"] != "igw-test1" { - t.Errorf("expected igw_id=igw-test1, got %s", extSwitch.ExternalIDs["spinifex:igw_id"]) - } - - // Verify localnet port created on external switch - _, err = mock.GetLogicalSwitchPort(ctx, "ext-port-vpc-igw1") - if err != nil { - t.Fatalf("expected localnet port: %v", err) - } - - // Verify gateway router port created - router, err := mock.GetLogicalRouter(ctx, "vpc-vpc-igw1") - if err != nil { - t.Fatalf("expected router: %v", err) - } - if len(router.Ports) < 1 { - t.Error("expected at least 1 router port (gateway)") - } - - // Verify switch gateway port created - _, err = mock.GetLogicalSwitchPort(ctx, "gw-port-vpc-igw1") - if err != nil { - t.Fatalf("expected switch gateway port: %v", err) - } - - // Verify NO blanket SNAT rule — only per-VM dnat_and_snat rules should - // provide NAT (AWS parity: private subnet instances cannot route via IGW) - if len(router.NAT) != 0 { - t.Errorf("expected 0 NAT rules (no blanket SNAT), got %d", len(router.NAT)) - } - - // Verify default route added - if len(router.StaticRoutes) != 1 { - t.Errorf("expected 1 static route, got %d", len(router.StaticRoutes)) - } -} - -// TestTopologyHandler_IGWAttach_InvokesFlowBarrier asserts handleIGWAttach -// calls waitForFlowsHV after the OVN writes complete (mulga-siv-105). The -// barrier itself shells out to ovn-nbctl --wait=hv sync; stub it here so -// the test doesn't depend on a live OVN. -func TestTopologyHandler_IGWAttach_InvokesFlowBarrier(t *testing.T) { - orig := waitForFlowsHV - var called int - waitForFlowsHV = func() error { called++; return nil } - defer func() { waitForFlowsHV = orig }() - - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - require.NoError(t, mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-barrier1", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-barrier1", - "spinifex:cidr": "10.0.0.0/16", - }, - })) - - evt := types.IGWEvent{InternetGatewayId: "igw-barrier1", VpcId: "vpc-barrier1"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "attach IGW") - - if called != 1 { - t.Errorf("expected waitForFlowsHV to run exactly once after IGW attach, got %d", called) - } -} - -// TestTopologyHandler_AddNAT_InvokesFlowBarrier asserts handleAddNAT -// calls waitForFlowsHV after writing the dnat_and_snat row to NB. -// Without this barrier, RunInstances returns success while OF flows for -// the new public IP are still being installed on the gateway chassis, -// and the freshly launched VM is unreachable on its public IP for the -// flow-install latency. Reproduced in CI run 26072432957 Phase8b. -func TestTopologyHandler_AddNAT_InvokesFlowBarrier(t *testing.T) { - orig := waitForFlowsHV - var called int - waitForFlowsHV = func() error { called++; return nil } - defer func() { waitForFlowsHV = orig }() - - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - require.NoError(t, mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-natbarrier", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-natbarrier", - "spinifex:cidr": "10.0.0.0/16", - }, - })) - - evt := NATEvent{ - VpcId: "vpc-natbarrier", - ExternalIP: "192.168.0.250", - LogicalIP: "10.0.1.4", - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicAddNAT, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "add NAT") - - if called != 1 { - t.Errorf("expected waitForFlowsHV to run exactly once after add-nat, got %d", called) - } -} - -// TestTopologyHandler_AddNAT_Idempotent asserts handleAddNAT does NOT -// delete-then-re-add the dnat_and_snat rule when the second invocation -// targets a rule whose state already matches the event (mulga-siv-124). -// Concurrent duplicate publishes for the same EIP would otherwise tear -// down the rule the previous invocation just installed, creating a -// flow-install gap during which the EIP is unreachable. -// -// Witness: the row's UUID. If the handler re-ran delete+add the row would -// be recreated with a fresh UUID; if the handler short-circuits the UUID -// is preserved. We also confirm the second invocation skips the flow -// barrier (waitForFlowsHV) — the flow already exists. -func TestTopologyHandler_AddNAT_Idempotent(t *testing.T) { - orig := waitForFlowsHV - var barrier int - waitForFlowsHV = func() error { barrier++; return nil } - defer func() { waitForFlowsHV = orig }() - - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeVeth)) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - require.NoError(t, mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-idem", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-idem", - "spinifex:cidr": "10.0.0.0/16", - }, - })) - - evt := NATEvent{ - VpcId: "vpc-idem", - ExternalIP: "192.168.1.244", - LogicalIP: "10.0.1.7", - PortName: "port-eni-idem", - MAC: "02:00:00:cc:cc:cc", - } - data, _ := json.Marshal(evt) - - // First invocation: rule does not exist → installs + waits for flows. - resp, err := nc.Request(TopicAddNAT, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "first add-nat") - - router, err := mock.GetLogicalRouter(ctx, "vpc-vpc-idem") - require.NoError(t, err) - if len(router.NAT) != 1 { - t.Fatalf("after first add-nat: expected 1 NAT rule, got %d", len(router.NAT)) - } - firstUUID := router.NAT[0] - if barrier != 1 { - t.Fatalf("after first add-nat: waitForFlowsHV called %d times, want 1", barrier) - } - - // Second invocation with identical event: must short-circuit. The - // existing row must survive untouched (same UUID), and the flow - // barrier must NOT run again. - resp, err = nc.Request(TopicAddNAT, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "second add-nat (idempotent)") - - router, err = mock.GetLogicalRouter(ctx, "vpc-vpc-idem") - require.NoError(t, err) - if len(router.NAT) != 1 { - t.Fatalf("after second add-nat: expected 1 NAT rule, got %d", len(router.NAT)) - } - if router.NAT[0] != firstUUID { - t.Errorf("second add-nat recreated the NAT row (uuid %q -> %q); delete-then-add was NOT skipped", firstUUID, router.NAT[0]) - } - if barrier != 1 { - t.Errorf("second add-nat ran flow barrier: waitForFlowsHV called %d times, want 1", barrier) - } - - nat := mock.nats[router.NAT[0]] - if nat.LogicalIP != evt.LogicalIP { - t.Errorf("NAT logical_ip=%q want %q", nat.LogicalIP, evt.LogicalIP) - } -} - -func TestTopologyHandler_IGWAttach_WithExternalPool(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - pools := []ExternalPoolConfig{ - { - Name: "wan", - RangeStart: "192.168.1.150", - RangeEnd: "192.168.1.250", - Gateway: "192.168.1.1", - PrefixLen: 24, - }, - } - topo := NewTopologyHandler(mock, WithExternalNetwork("pool", pools)) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create VPC router - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-extpool", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-extpool", - "spinifex:cidr": "10.0.0.0/16", - }, - }) - - // Attach IGW - evt := types.IGWEvent{InternetGatewayId: "igw-ext1", VpcId: "vpc-extpool"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.igw-attach: %v", err) - } - assertSuccess(t, resp, "attach IGW with external pool") - - // Verify gateway router port uses real external IP, not link-local - router, err := mock.GetLogicalRouter(ctx, "vpc-vpc-extpool") - if err != nil { - t.Fatalf("expected router: %v", err) - } - if len(router.Ports) < 1 { - t.Fatal("expected at least 1 router port") - } - - gwPort, err := mock.GetLogicalRouterPort(ctx, "gw-vpc-extpool") - if err != nil { - t.Fatalf("expected gateway router port: %v", err) - } - // Centralized NAT: LRP gets a WAN-subnet IP auto-derived below the - // per-VM EIP range (192.168.1.150-250), so somewhere in - // 192.168.1.134-149 with /24. Pool range_start - 16 = .134, range_start - 1 = .149. - if len(gwPort.Networks) != 1 { - t.Fatalf("expected 1 LRP network, got %v", gwPort.Networks) - } - gotNet := gwPort.Networks[0] - if !strings.HasPrefix(gotNet, "192.168.1.1") || !strings.HasSuffix(gotNet, "/24") { - t.Errorf("expected gw LRP network in 192.168.1.134-149/24, got %s", gotNet) - } - if gwPort.ExternalIDs["spinifex:gateway_ip"] == "" { - t.Errorf("expected spinifex:gateway_ip on LRP, got empty") - } - - // Verify NO blanket SNAT rule (AWS parity) - if len(router.NAT) != 0 { - t.Errorf("expected 0 NAT rules (no blanket SNAT), got %d", len(router.NAT)) - } - - // Verify default route points to WAN gateway, not link-local - if len(router.StaticRoutes) != 1 { - t.Fatalf("expected 1 static route, got %d", len(router.StaticRoutes)) - } - route := mock.staticRoutes[router.StaticRoutes[0]] - if route == nil { - t.Fatal("static route not found in mock") - } - if route.Nexthop != "192.168.1.1" { - t.Errorf("expected nexthop 192.168.1.1, got %s", route.Nexthop) - } - gwPortName := "gw-vpc-extpool" - if route.OutputPort == nil || *route.OutputPort != gwPortName { - t.Errorf("expected OutputPort=%s, got %v", gwPortName, route.OutputPort) - } -} - -func TestTopologyHandler_IGWAttach_PoolWithGatewayIP(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - pools := []ExternalPoolConfig{ - { - Name: "dc1", - RangeStart: "203.0.113.8", - RangeEnd: "203.0.113.14", - Gateway: "203.0.113.1", - GatewayIP: "203.0.113.8", // Explicit gateway IP - PrefixLen: 28, - }, - } - topo := NewTopologyHandler(mock, WithExternalNetwork("pool", pools)) - subs, _ := topo.Subscribe(nc) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-gwip", - ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-gwip", "spinifex:cidr": "10.0.0.0/16"}, - }) - - evt := types.IGWEvent{InternetGatewayId: "igw-gwip", VpcId: "vpc-gwip"} - data, _ := json.Marshal(evt) - resp, _ := nc.Request(TopicIGWAttach, data, 5_000_000_000) - assertSuccess(t, resp, "attach IGW with explicit gateway IP") - - router, _ := mock.GetLogicalRouter(ctx, "vpc-vpc-gwip") - gwPort, _ := mock.GetLogicalRouterPort(ctx, "gw-vpc-gwip") - // /28 with full pool .2-.14 + gateway .1 leaves no room in-subnet — - // auto-derive shifts below RangeStart and clamps; whatever the result, - // it must be a /28 IP carrying spinifex:gateway_ip. - if !strings.HasSuffix(gwPort.Networks[0], "/28") { - t.Errorf("expected /28 LRP network, got %s", gwPort.Networks[0]) - } - if gwPort.ExternalIDs["spinifex:gateway_ip"] == "" { - t.Errorf("expected spinifex:gateway_ip on LRP, got empty") - } - // No blanket SNAT rule (AWS parity) - if len(router.NAT) != 0 { - t.Errorf("expected 0 NAT rules (no blanket SNAT), got %d", len(router.NAT)) - } - route := mock.staticRoutes[router.StaticRoutes[0]] - if route.Nexthop != "203.0.113.1" { - t.Errorf("expected nexthop 203.0.113.1, got %s", route.Nexthop) - } - gwPortName := "gw-vpc-gwip" - if route.OutputPort == nil || *route.OutputPort != gwPortName { - t.Errorf("expected OutputPort=%s, got %v", gwPortName, route.OutputPort) - } -} - -// TestTopologyHandler_IGWAttach_MultiVPC_NoIPCollision is the regression -// guard for mulga-siv-26: three VPCs sharing one external pool must produce -// three gateway LRPs with distinct WAN-subnet IPs, distinct MACs and distinct -// default routes. Pre-fix every LRP claimed pool.RangeStart and the VPCs -// ARP-collided on br-ext. -func TestTopologyHandler_IGWAttach_MultiVPC_NoIPCollision(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - pools := []ExternalPoolConfig{ - { - Name: "ci-multi-2", - RangeStart: "192.168.0.160", - RangeEnd: "192.168.0.169", - Gateway: "192.168.0.1", - PrefixLen: 24, - }, - } - topo := NewTopologyHandler(mock, WithExternalNetwork("pool", pools)) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - vpcIds := []string{"vpc-multi-a", "vpc-multi-b", "vpc-multi-c"} - for _, vpcId := range vpcIds { - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-" + vpcId, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": vpcId, - "spinifex:cidr": "10.0.0.0/16", - }, - }) - evt := types.IGWEvent{InternetGatewayId: "igw-" + vpcId, VpcId: vpcId} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.igw-attach %s: %v", vpcId, err) - } - assertSuccess(t, resp, "attach IGW "+vpcId) - } - - macs := make(map[string]string, len(vpcIds)) - ips := make(map[string]string, len(vpcIds)) - for _, vpcId := range vpcIds { - gwPortName := "gw-" + vpcId - gwPort, err := mock.GetLogicalRouterPort(ctx, gwPortName) - if err != nil { - t.Fatalf("get %s: %v", gwPortName, err) - } - if len(gwPort.Networks) != 1 { - t.Fatalf("%s: expected 1 network, got %v", gwPortName, gwPort.Networks) - } - ip, cidr, _ := strings.Cut(gwPort.Networks[0], "/") - if cidr != "24" { - t.Errorf("%s: expected /24, got /%s", gwPortName, cidr) - } - if !strings.HasPrefix(ip, "192.168.0.") { - t.Errorf("%s: expected 192.168.0.x, got %s", gwPortName, ip) - } - gwIP := gwPort.ExternalIDs["spinifex:gateway_ip"] - if gwIP != ip { - t.Errorf("%s: external_ids gateway_ip=%q, expected %q", gwPortName, gwIP, ip) - } - if other, dup := ips[ip]; dup { - t.Errorf("IP collision: %s and %s share %s", gwPortName, other, ip) - } - ips[ip] = gwPortName - if gwPort.MAC == "" { - t.Errorf("%s: empty MAC", gwPortName) - } - if other, dup := macs[gwPort.MAC]; dup { - t.Errorf("MAC collision: %s and %s share %s", gwPortName, other, gwPort.MAC) - } - macs[gwPort.MAC] = gwPortName - - router, err := mock.GetLogicalRouter(ctx, "vpc-"+vpcId) - if err != nil { - t.Fatalf("get router for %s: %v", vpcId, err) - } - if len(router.StaticRoutes) != 1 { - t.Fatalf("%s: expected 1 static route, got %d", vpcId, len(router.StaticRoutes)) - } - route := mock.staticRoutes[router.StaticRoutes[0]] - if route.Nexthop != "192.168.0.1" { - t.Errorf("%s: expected nexthop 192.168.0.1, got %s", vpcId, route.Nexthop) - } - if route.OutputPort == nil || *route.OutputPort != gwPortName { - t.Errorf("%s: expected OutputPort=%s, got %v", vpcId, gwPortName, route.OutputPort) - } - } - if len(macs) != len(vpcIds) { - t.Errorf("expected %d distinct MACs, got %d", len(vpcIds), len(macs)) - } - if len(ips) != len(vpcIds) { - t.Errorf("expected %d distinct IPs, got %d", len(vpcIds), len(ips)) - } -} - -// TestTopologyHandler_IGWAttach_CentralizedNATAllocatesGwLrpIP guards -// mulga-siv-36: in centralized NAT (veth/macvlan) the gateway LRP must -// hold a WAN-subnet IP from pool.GwLrpRange so it can ARP the upstream -// nexthop. Three VPCs sharing one pool must each get a distinct IP and -// persist it on external_ids:spinifex:gateway_ip. -func TestTopologyHandler_IGWAttach_CentralizedNATAllocatesGwLrpIP(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - pools := []ExternalPoolConfig{ - { - Name: "lan", - RangeStart: "192.168.3.100", - RangeEnd: "192.168.3.150", - Gateway: "192.168.3.1", - PrefixLen: 24, - GwLrpRangeStart: "192.168.3.20", - GwLrpRangeEnd: "192.168.3.29", - }, - } - // Default bridge mode is centralized (useCentralizedNAT returns true - // when bridgeMode != "direct"). Explicit veth here to be unambiguous. - topo := NewTopologyHandler(mock, - WithExternalNetwork("pool", pools), - WithBridgeMode(BridgeModeVeth), - ) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - vpcIds := []string{"vpc-cn-a", "vpc-cn-b", "vpc-cn-c"} - for _, vpcId := range vpcIds { - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-" + vpcId, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": vpcId, - "spinifex:cidr": "10.0.0.0/16", - }, - }) - evt := types.IGWEvent{InternetGatewayId: "igw-" + vpcId, VpcId: vpcId} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.igw-attach %s: %v", vpcId, err) - } - assertSuccess(t, resp, "attach IGW "+vpcId) - } - - seen := make(map[string]string, len(vpcIds)) - for _, vpcId := range vpcIds { - gwPortName := "gw-" + vpcId - gwPort, err := mock.GetLogicalRouterPort(ctx, gwPortName) - if err != nil { - t.Fatalf("get %s: %v", gwPortName, err) - } - if len(gwPort.Networks) != 1 { - t.Fatalf("%s: expected 1 Network, got %v", gwPortName, gwPort.Networks) - } - net := gwPort.Networks[0] - if !strings.HasPrefix(net, "192.168.3.") || !strings.HasSuffix(net, "/24") { - t.Errorf("%s: expected 192.168.3.X/24 from gw_lrp_range, got %s", gwPortName, net) - } - gwIP := gwPort.ExternalIDs["spinifex:gateway_ip"] - if gwIP == "" { - t.Errorf("%s: missing spinifex:gateway_ip external_id", gwPortName) - } - if other, dup := seen[gwIP]; dup { - t.Errorf("gw IP collision: %s and %s share %s", gwPortName, other, gwIP) - } - seen[gwIP] = gwPortName - - // Idempotent: re-attach must not change the assignment. - evt := types.IGWEvent{InternetGatewayId: "igw-" + vpcId, VpcId: vpcId} - data, _ := json.Marshal(evt) - _, _ = nc.Request(TopicIGWAttach, data, 5_000_000_000) - gwPort2, _ := mock.GetLogicalRouterPort(ctx, gwPortName) - if gwPort2.ExternalIDs["spinifex:gateway_ip"] != gwIP { - t.Errorf("%s: re-attach changed gateway_ip from %s to %s", - gwPortName, gwIP, gwPort2.ExternalIDs["spinifex:gateway_ip"]) - } - } -} - -// dhcpStub is a minimal in-test responder for vpc.dhcp.acquire / release. -// Each acquire returns a distinct lease IP keyed off ClientID so multi-VPC -// tests get unique IPs without standing up a full DHCPManager. -type dhcpStub struct { - t *testing.T - leases map[string]string // client_id -> ip - released map[string]bool - nextOct int -} - -func newDHCPStub(t *testing.T, nc *nats.Conn) *dhcpStub { - t.Helper() - s := &dhcpStub{ - t: t, - leases: map[string]string{}, - released: map[string]bool{}, - nextOct: 101, - } - subA, err := nc.Subscribe(dhcp.TopicAcquire, func(msg *nats.Msg) { - var req dhcp.AcquireRequestMsg - if err := json.Unmarshal(msg.Data, &req); err != nil { - data, _ := json.Marshal(dhcp.AcquireReplyMsg{Error: err.Error()}) - _ = msg.Respond(data) - return - } - ip, ok := s.leases[req.ClientID] - if !ok { - ip = fmt.Sprintf("192.168.3.%d", s.nextOct) - s.nextOct++ - s.leases[req.ClientID] = ip - } - reply := dhcp.AcquireReplyMsg{ - IP: ip, - SubnetMask: "255.255.255.0", - ServerID: "192.168.3.1", - Routers: []string{"192.168.3.1"}, - HWAddr: "02:00:00:00:00:" + fmt.Sprintf("%02x", s.nextOct%256), - ExpiresUnix: 0, - } - data, _ := json.Marshal(reply) - _ = msg.Respond(data) - }) - if err != nil { - t.Fatalf("subscribe acquire: %v", err) - } - subR, err := nc.Subscribe(dhcp.TopicRelease, func(msg *nats.Msg) { - var req dhcp.ReleaseRequestMsg - _ = json.Unmarshal(msg.Data, &req) - s.released[req.ClientID] = true - delete(s.leases, req.ClientID) - data, _ := json.Marshal(dhcp.ReleaseReplyMsg{}) - _ = msg.Respond(data) - }) - if err != nil { - t.Fatalf("subscribe release: %v", err) - } - t.Cleanup(func() { - _ = subA.Unsubscribe() - _ = subR.Unsubscribe() - }) - return s -} - -// TestTopologyHandler_IGWAttach_DHCPGwLrpIP guards mulga-siv-38: with -// pool.Source="dhcp" the gateway LRP IP must come from a DHCP lease -// (vpc.dhcp.acquire), not the static gw_lrp_range allocator. Three VPCs -// sharing one DHCP pool must each get a distinct lease IP and persist it -// on external_ids:spinifex:gateway_ip. IGW detach must release the lease. -func TestTopologyHandler_IGWAttach_DHCPGwLrpIP(t *testing.T) { - _, nc := startTestNATS(t) - stub := newDHCPStub(t, nc) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - pools := []ExternalPoolConfig{ - { - Name: "lan-dhcp", - Source: "dhcp", - Gateway: "192.168.3.1", - PrefixLen: 24, - DhcpBindBridge: "br-wan", - }, - } - topo := NewTopologyHandler(mock, - WithExternalNetwork("pool", pools), - WithBridgeMode(BridgeModeVeth), - WithNATSConn(nc), - ) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - vpcIds := []string{"vpc-dh-a", "vpc-dh-b", "vpc-dh-c"} - for _, vpcId := range vpcIds { - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-" + vpcId, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": vpcId, - "spinifex:cidr": "10.0.0.0/16", - }, - }) - evt := types.IGWEvent{InternetGatewayId: "igw-" + vpcId, VpcId: vpcId} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.igw-attach %s: %v", vpcId, err) - } - assertSuccess(t, resp, "attach IGW "+vpcId) - } - - seen := make(map[string]string, len(vpcIds)) - for _, vpcId := range vpcIds { - gwPortName := "gw-" + vpcId - gwPort, err := mock.GetLogicalRouterPort(ctx, gwPortName) - if err != nil { - t.Fatalf("get %s: %v", gwPortName, err) - } - if len(gwPort.Networks) != 1 { - t.Fatalf("%s: expected 1 Network, got %v", gwPortName, gwPort.Networks) - } - ip, cidr, _ := strings.Cut(gwPort.Networks[0], "/") - if cidr != "24" { - t.Errorf("%s: expected /24 from DHCP lease mask, got /%s", gwPortName, cidr) - } - if !strings.HasPrefix(ip, "192.168.3.") { - t.Errorf("%s: expected 192.168.3.X from stub, got %s", gwPortName, ip) - } - gwIP := gwPort.ExternalIDs["spinifex:gateway_ip"] - if gwIP != ip { - t.Errorf("%s: external_ids gateway_ip=%q, expected %q", gwPortName, gwIP, ip) - } - if other, dup := seen[gwIP]; dup { - t.Errorf("gw IP collision: %s and %s share %s", gwPortName, other, gwIP) - } - seen[gwIP] = gwPortName - - // stub recorded the lease keyed by gw-lrp- - clientID := "gw-lrp-" + vpcId - if stub.leases[clientID] != ip { - t.Errorf("%s: stub lease %q != lrp IP %q", clientID, stub.leases[clientID], ip) - } - } - - // Detach one VPC, assert release reaches the stub. - detachVpc := vpcIds[0] - devt := types.IGWEvent{InternetGatewayId: "igw-" + detachVpc, VpcId: detachVpc} - ddata, _ := json.Marshal(devt) - resp, err := nc.Request(TopicIGWDetach, ddata, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.igw-detach: %v", err) - } - assertSuccess(t, resp, "detach IGW "+detachVpc) - if !stub.released["gw-lrp-"+detachVpc] { - t.Errorf("expected DHCP release for gw-lrp-%s, stub released=%v", detachVpc, stub.released) - } -} - -// TestTopologyHandler_IGWAttach_DHCPAcquireFailureAborts ensures that when -// vpcd's DHCPManager replies with an error, IGW attach surfaces the failure -// loud instead of falling back to link-local. Otherwise we silently restore -// the broken pre-siv-36 behavior. -func TestTopologyHandler_IGWAttach_DHCPAcquireFailureAborts(t *testing.T) { - _, nc := startTestNATS(t) - sub, err := nc.Subscribe(dhcp.TopicAcquire, func(msg *nats.Msg) { - reply := dhcp.AcquireReplyMsg{Error: "no leases available"} - data, _ := json.Marshal(reply) - _ = msg.Respond(data) - }) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { _ = sub.Unsubscribe() }() - - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - pools := []ExternalPoolConfig{ - { - Name: "lan-dhcp", - Source: "dhcp", - Gateway: "192.168.3.1", - PrefixLen: 24, - DhcpBindBridge: "br-wan", - }, - } - topo := NewTopologyHandler(mock, - WithExternalNetwork("pool", pools), - WithBridgeMode(BridgeModeVeth), - WithNATSConn(nc), - ) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-dh-fail", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-dh-fail", - "spinifex:cidr": "10.0.0.0/16", - }, - }) - evt := types.IGWEvent{InternetGatewayId: "igw-fail", VpcId: "vpc-dh-fail"} - data, _ := json.Marshal(evt) - resp, _ := nc.Request(TopicIGWAttach, data, 5_000_000_000) - var reply struct { - Error string `json:"error"` - } - _ = json.Unmarshal(resp.Data, &reply) - if reply.Error == "" { - t.Fatal("expected attach to fail when DHCP acquire errors, got success") - } - if !strings.Contains(reply.Error, "dhcp acquire") && !strings.Contains(reply.Error, "no leases available") { - t.Errorf("expected dhcp-related error, got %q", reply.Error) - } - // External switch must be rolled back. - if _, err := mock.GetLogicalSwitch(ctx, "ext-vpc-dh-fail"); err == nil { - t.Error("expected external switch rolled back after DHCP failure") - } -} - -// TestPrefixFromMask checks dotted-mask → prefix conversion across the -// boundary cases gw LRP DHCP relies on (option 1 missing, /32, /0, -// non-contiguous masks, IPv6). -func TestPrefixFromMask(t *testing.T) { - cases := []struct { - mask string - want int - }{ - {"", 0}, - {"255.255.255.0", 24}, - {"255.255.255.255", 32}, - {"0.0.0.0", 0}, - {"255.255.0.0", 16}, - {"not-an-ip", 0}, - } - for _, tc := range cases { - t.Run(tc.mask, func(t *testing.T) { - got := prefixFromMask(tc.mask) - if got != tc.want { - t.Errorf("prefixFromMask(%q)=%d, want %d", tc.mask, got, tc.want) - } - }) - } -} - -func TestTopologyHandler_FindExternalPool(t *testing.T) { - pools := []ExternalPoolConfig{ - {Name: "az-a", Region: "us-east-1", AZ: "us-east-1a", RangeStart: "1.1.1.1"}, - {Name: "region", Region: "us-east-1", RangeStart: "2.2.2.2"}, - {Name: "global", RangeStart: "3.3.3.3"}, - } - topo := NewTopologyHandler(nil, WithExternalNetwork("pool", pools)) - - // AZ match - p := topo.findExternalPool("us-east-1", "us-east-1a") - if p == nil || p.Name != "az-a" { - t.Errorf("expected az-a pool, got %v", p) - } - - // Region fallback (no matching AZ) - p = topo.findExternalPool("us-east-1", "us-east-1b") - if p == nil || p.Name != "region" { - t.Errorf("expected region pool, got %v", p) - } - - // Global fallback - p = topo.findExternalPool("eu-west-1", "eu-west-1a") - if p == nil || p.Name != "global" { - t.Errorf("expected global pool, got %v", p) - } - - // No pools - topo2 := NewTopologyHandler(nil) - p = topo2.findExternalPool("us-east-1", "us-east-1a") - if p != nil { - t.Errorf("expected nil pool, got %v", p) - } -} - -func TestTopologyHandler_IGWDetach(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create VPC router - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-igw2", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-igw2", - "spinifex:cidr": "10.0.0.0/16", - }, - }) - - // Attach IGW first - attachEvt := types.IGWEvent{InternetGatewayId: "igw-test2", VpcId: "vpc-igw2"} - data, _ := json.Marshal(attachEvt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.igw-attach: %v", err) - } - assertSuccess(t, resp, "attach IGW") - - // Verify resources exist before detach - _, err = mock.GetLogicalSwitch(ctx, "ext-vpc-igw2") - if err != nil { - t.Fatal("expected external switch before detach") - } - - // Detach IGW - detachEvt := types.IGWEvent{InternetGatewayId: "igw-test2", VpcId: "vpc-igw2"} - data, _ = json.Marshal(detachEvt) - resp, err = nc.Request(TopicIGWDetach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.igw-detach: %v", err) - } - assertSuccess(t, resp, "detach IGW") - - // Verify external switch deleted - _, err = mock.GetLogicalSwitch(ctx, "ext-vpc-igw2") - if err == nil { - t.Error("expected external switch to be deleted") - } - - // Verify router still exists but NAT and routes are cleaned up - router, err := mock.GetLogicalRouter(ctx, "vpc-vpc-igw2") - if err != nil { - t.Fatal("expected VPC router to still exist") - } - if len(router.NAT) != 0 { - t.Errorf("expected 0 NAT rules after detach, got %d", len(router.NAT)) - } - if len(router.StaticRoutes) != 0 { - t.Errorf("expected 0 static routes after detach, got %d", len(router.StaticRoutes)) - } - if len(router.Ports) != 0 { - t.Errorf("expected 0 router ports after detach, got %d", len(router.Ports)) - } -} - -func TestTopologyHandler_IGWAttachDetachLifecycle(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // 1. Create VPC - vpcEvt := VPCEvent{VpcId: "vpc-igwlc", CidrBlock: "10.0.0.0/16", VNI: 400} - data, _ := json.Marshal(vpcEvt) - resp, _ := nc.Request(TopicVPCCreate, data, 5_000_000_000) - assertSuccess(t, resp, "create VPC") - - // 2. Create subnet - subEvt := SubnetEvent{SubnetId: "subnet-igwlc", VpcId: "vpc-igwlc", CidrBlock: "10.0.1.0/24"} - data, _ = json.Marshal(subEvt) - resp, _ = nc.Request(TopicSubnetCreate, data, 5_000_000_000) - assertSuccess(t, resp, "create subnet") - - // 3. Attach IGW - igwEvt := types.IGWEvent{InternetGatewayId: "igw-lc1", VpcId: "vpc-igwlc"} - data, _ = json.Marshal(igwEvt) - resp, _ = nc.Request(TopicIGWAttach, data, 5_000_000_000) - assertSuccess(t, resp, "attach IGW") - - // Verify full topology: 2 switches (subnet + external), 1 router with ports+NAT+routes - switches, _ := mock.ListLogicalSwitches(ctx) - if len(switches) != 2 { - t.Errorf("expected 2 switches (subnet + external), got %d", len(switches)) - } - - router, _ := mock.GetLogicalRouter(ctx, "vpc-vpc-igwlc") - if len(router.Ports) != 2 { - t.Errorf("expected 2 router ports (subnet + gateway), got %d", len(router.Ports)) - } - - // 4. Detach IGW - data, _ = json.Marshal(igwEvt) - resp, _ = nc.Request(TopicIGWDetach, data, 5_000_000_000) - assertSuccess(t, resp, "detach IGW") - - // Only subnet switch should remain - switches, _ = mock.ListLogicalSwitches(ctx) - if len(switches) != 1 { - t.Errorf("expected 1 switch after IGW detach, got %d", len(switches)) - } - - // Router should still have subnet port but no gateway port - router, _ = mock.GetLogicalRouter(ctx, "vpc-vpc-igwlc") - if len(router.Ports) != 1 { - t.Errorf("expected 1 router port after IGW detach, got %d", len(router.Ports)) - } - - // 5. Delete subnet and VPC - data, _ = json.Marshal(subEvt) - resp, _ = nc.Request(TopicSubnetDelete, data, 5_000_000_000) - assertSuccess(t, resp, "delete subnet") - - data, _ = json.Marshal(VPCEvent{VpcId: "vpc-igwlc"}) - resp, _ = nc.Request(TopicVPCDelete, data, 5_000_000_000) - assertSuccess(t, resp, "delete VPC") - - // Everything should be gone - routers, _ := mock.ListLogicalRouters(ctx) - if len(routers) != 0 { - t.Errorf("expected 0 routers, got %d", len(routers)) - } - switches, _ = mock.ListLogicalSwitches(ctx) - if len(switches) != 0 { - t.Errorf("expected 0 switches, got %d", len(switches)) - } -} - -func TestTopologyHandler_IGWNilOVN(t *testing.T) { - _, nc := startTestNATS(t) - - topo := NewTopologyHandler(nil) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Should fail gracefully when OVN is nil - evt := types.IGWEvent{InternetGatewayId: "igw-nil", VpcId: "vpc-nil"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - - var result struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - } - _ = json.Unmarshal(resp.Data, &result) - if result.Success { - t.Error("expected failure when OVN is nil") - } - - // Detach should also fail gracefully - resp, err = nc.Request(TopicIGWDetach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - _ = json.Unmarshal(resp.Data, &result) - if result.Success { - t.Error("expected detach failure when OVN is nil") - } -} - -// --- Error path tests (Phase 3) --- - -func assertFailure(t *testing.T, msg *nats.Msg, label string) { - t.Helper() - var result struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - } - if err := json.Unmarshal(msg.Data, &result); err != nil { - t.Fatalf("%s: unmarshal: %v", label, err) - } - if result.Success { - t.Fatalf("%s: expected failure, got success", label) - } -} - -func TestTopologyHandler_BadJSON(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - badData := []byte("{invalid json") - - // Each topic should return an error response on bad JSON - topics := []string{ - TopicVPCCreate, TopicVPCDelete, - TopicSubnetCreate, TopicSubnetDelete, - TopicCreatePort, TopicDeletePort, TopicUpdatePortSGs, - TopicIGWAttach, TopicIGWDetach, - } - for _, topic := range topics { - resp, err := nc.Request(topic, badData, 5_000_000_000) - if err != nil { - t.Fatalf("request %s: %v", topic, err) - } - assertFailure(t, resp, "bad JSON on "+topic) - } -} - -func TestTopologyHandler_VPCCreate_DuplicateRouter(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create router to trigger idempotent path - _ = mock.CreateLogicalRouter(ctx, nbdbLogicalRouter("vpc-vpc-dup", "vpc-dup")) - - evt := VPCEvent{VpcId: "vpc-dup", CidrBlock: "10.0.0.0/16", VNI: 100} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicVPCCreate, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertSuccess(t, resp, "idempotent VPC create") - - // Verify exactly 1 router (no duplicate created) - routers, _ := mock.ListLogicalRouters(ctx) - if len(routers) != 1 { - t.Errorf("expected 1 router (idempotent), got %d", len(routers)) - } -} - -func TestTopologyHandler_VPCDelete_RouterNotFound(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Delete VPC that doesn't exist — DeleteLogicalRouter will fail - evt := VPCEvent{VpcId: "vpc-nonexistent"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicVPCDelete, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertFailure(t, resp, "delete nonexistent VPC") -} - -func TestTopologyHandler_SubnetCreate_SwitchAlreadyExists(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create switch to trigger idempotent path - _ = mock.CreateLogicalSwitch(ctx, nbdbLogicalSwitch("subnet-subnet-exists", "subnet-exists", "vpc-exists")) - - evt := SubnetEvent{SubnetId: "subnet-exists", VpcId: "vpc-exists", CidrBlock: "10.0.1.0/24"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicSubnetCreate, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertSuccess(t, resp, "idempotent subnet create") - - // Verify exactly 1 switch (no duplicate) - switches, _ := mock.ListLogicalSwitches(ctx) - if len(switches) != 1 { - t.Errorf("expected 1 switch (idempotent), got %d", len(switches)) - } -} - -func TestTopologyHandler_SubnetCreate_RouterPortFails(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Don't create the VPC router — CreateLogicalRouterPort will fail - evt := SubnetEvent{SubnetId: "subnet-norouter", VpcId: "vpc-norouter", CidrBlock: "10.0.1.0/24"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicSubnetCreate, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertFailure(t, resp, "subnet create without router") - - // Verify switch was cleaned up (best-effort cleanup path) - _, err = mock.GetLogicalSwitch(ctx, "subnet-subnet-norouter") - if err == nil { - t.Error("expected switch to be cleaned up after router port failure") - } -} - -func TestTopologyHandler_SubnetCreate_InvalidCIDR(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - evt := SubnetEvent{SubnetId: "subnet-badcidr", VpcId: "vpc-badcidr", CidrBlock: "not-a-cidr"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicSubnetCreate, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertFailure(t, resp, "subnet create with invalid CIDR") -} - -func TestTopologyHandler_SubnetDelete_SwitchNotFound(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Delete subnet where nothing exists — switch delete fails at step 4 - evt := SubnetEvent{SubnetId: "subnet-ghost", VpcId: "vpc-ghost", CidrBlock: "10.0.99.0/24"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicSubnetDelete, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertFailure(t, resp, "delete nonexistent subnet") -} - -func TestTopologyHandler_CreatePort_SwitchNotFound(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Create port on non-existent switch - evt := PortEvent{ - NetworkInterfaceId: "eni-orphan", - SubnetId: "subnet-missing", - VpcId: "vpc-missing", - PrivateIpAddress: "10.0.1.4", - MacAddress: "02:00:00:00:00:01", - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicCreatePort, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertFailure(t, resp, "create port on missing switch") -} - -func TestTopologyHandler_CreatePort_Idempotent(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Setup: create switch and port directly - _ = mock.CreateLogicalSwitch(ctx, nbdbLogicalSwitch("subnet-subnet-idem", "subnet-idem", "vpc-idem")) - _ = mock.CreateLogicalSwitchPort(ctx, "subnet-subnet-idem", &nbdb.LogicalSwitchPort{ - Name: "port-eni-idem", - Addresses: []string{"02:00:00:11:22:33 10.0.1.4"}, - ExternalIDs: map[string]string{ - "spinifex:eni_id": "eni-idem", - }, - }) - - // Send create for same port — should succeed (idempotent skip) - evt := PortEvent{ - NetworkInterfaceId: "eni-idem", - SubnetId: "subnet-idem", - VpcId: "vpc-idem", - PrivateIpAddress: "10.0.1.4", - MacAddress: "02:00:00:11:22:33", - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicCreatePort, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertSuccess(t, resp, "idempotent port create") -} - -func TestTopologyHandler_DeletePort_PortNotFound(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Delete port that doesn't exist - evt := PortEvent{ - NetworkInterfaceId: "eni-ghost", - SubnetId: "subnet-ghost", - VpcId: "vpc-ghost", - PrivateIpAddress: "10.0.1.99", - MacAddress: "02:00:00:ff:ff:ff", - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicDeletePort, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertFailure(t, resp, "delete nonexistent port") -} - -func TestTopologyHandler_IGWAttach_RouterNotFound(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Attach IGW without VPC router — CreateLogicalRouterPort fails - evt := types.IGWEvent{InternetGatewayId: "igw-orphan", VpcId: "vpc-norouter"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertFailure(t, resp, "IGW attach without router") - - // Verify external switch was cleaned up - _, err = mock.GetLogicalSwitch(ctx, "ext-vpc-norouter") - if err == nil { - t.Error("expected external switch to be cleaned up after router port failure") - } -} - -func TestTopologyHandler_IGWAttach_Idempotent(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create external switch to trigger idempotent path - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{ - Name: "ext-vpc-igw-idem", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-igw-idem", - "spinifex:role": "external", - }, - }) - - evt := types.IGWEvent{InternetGatewayId: "igw-idem", VpcId: "vpc-igw-idem"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertSuccess(t, resp, "idempotent IGW attach") -} - -func TestTopologyHandler_IGWDetach_PartialCleanup(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Only create the external switch (not the full IGW topology). - // All intermediate deletes will warn but switch delete should succeed. - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{ - Name: "ext-vpc-partial", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-partial", - "spinifex:role": "external", - }, - }) - // Create router so NAT cleanup path is exercised (but NAT won't be found) - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-partial", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-partial", - "spinifex:cidr": "10.0.0.0/16", - }, - }) - - evt := types.IGWEvent{InternetGatewayId: "igw-partial", VpcId: "vpc-partial"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWDetach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - // Should succeed — intermediate deletes warn but final switch delete succeeds - assertSuccess(t, resp, "partial IGW detach") - - // Verify external switch was deleted - _, err = mock.GetLogicalSwitch(ctx, "ext-vpc-partial") - if err == nil { - t.Error("expected external switch to be deleted") - } -} - -func TestTopologyHandler_IGWDetach_NoRouter(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Create external switch but no router — exercises the GetLogicalRouter warn path - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{ - Name: "ext-vpc-nortr", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-nortr", - "spinifex:role": "external", - }, - }) - - evt := types.IGWEvent{InternetGatewayId: "igw-nortr", VpcId: "vpc-nortr"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWDetach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - // Should succeed — router not found is a warn path, switch delete succeeds - assertSuccess(t, resp, "IGW detach without router") - - _, err = mock.GetLogicalSwitch(ctx, "ext-vpc-nortr") - if err == nil { - t.Error("expected external switch to be deleted") - } -} - -func TestTopologyHandler_IGWDetach_SwitchNotFound(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // No external switch — final DeleteLogicalSwitch fails - evt := types.IGWEvent{InternetGatewayId: "igw-nosw", VpcId: "vpc-nosw"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWDetach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertFailure(t, resp, "IGW detach without external switch") -} - -func TestTopologyHandler_VPCDeleteCascade_WithPorts(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Create VPC router + subnet switch with ports (simulating pre-existing state) - _ = mock.CreateLogicalRouter(ctx, nbdbLogicalRouter("vpc-vpc-casp", "vpc-casp")) - _ = mock.CreateLogicalSwitch(ctx, nbdbLogicalSwitch("subnet-sub-casp", "sub-casp", "vpc-casp")) - _ = mock.CreateLogicalSwitchPort(ctx, "subnet-sub-casp", &nbdb.LogicalSwitchPort{ - Name: "port-eni-casp1", - Addresses: []string{"02:00:00:11:22:33 10.0.1.4"}, - ExternalIDs: map[string]string{ - "spinifex:eni_id": "eni-casp1", - "spinifex:subnet_id": "sub-casp", - "spinifex:vpc_id": "vpc-casp", - }, - }) - _ = mock.CreateLogicalSwitchPort(ctx, "subnet-sub-casp", &nbdb.LogicalSwitchPort{ - Name: "port-eni-casp2", - Addresses: []string{"02:00:00:44:55:66 10.0.1.5"}, - ExternalIDs: map[string]string{ - "spinifex:eni_id": "eni-casp2", - "spinifex:subnet_id": "sub-casp", - "spinifex:vpc_id": "vpc-casp", - }, - }) - _, _ = mock.CreateDHCPOptions(ctx, nbdbDHCPOptions("10.0.1.0/24", "sub-casp", "vpc-casp")) - - // Verify switch has 2 ports before delete - ls, err := mock.GetLogicalSwitch(ctx, "subnet-sub-casp") - if err != nil { - t.Fatalf("expected switch: %v", err) - } - if len(ls.Ports) != 2 { - t.Errorf("expected 2 ports before cascade, got %d", len(ls.Ports)) - } - - // Delete VPC should cascade-delete switches (with ports) and DHCP - evt := VPCEvent{VpcId: "vpc-casp"} - data, _ := json.Marshal(evt) - resp, _ := nc.Request(TopicVPCDelete, data, 5_000_000_000) - assertSuccess(t, resp, "cascade delete VPC with ports") - - // Verify switch is gone - _, err = mock.GetLogicalSwitch(ctx, "subnet-sub-casp") - if err == nil { - t.Error("expected switch to be deleted after cascade") - } - // Verify DHCP options gone - dhcpList, _ := mock.ListDHCPOptions(ctx) - if len(dhcpList) != 0 { - t.Errorf("expected 0 DHCP options after cascade, got %d", len(dhcpList)) - } - // Verify router gone - _, err = mock.GetLogicalRouter(ctx, "vpc-vpc-casp") - if err == nil { - t.Error("expected router to be deleted after cascade") - } -} - -func TestTopologyHandler_SubnetDelete_NilOVN(t *testing.T) { - _, nc := startTestNATS(t) - - topo := NewTopologyHandler(nil) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - evt := SubnetEvent{SubnetId: "subnet-nil", VpcId: "vpc-nil", CidrBlock: "10.0.0.0/24"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicSubnetDelete, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - - var result struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - } - _ = json.Unmarshal(resp.Data, &result) - if result.Success { - t.Error("expected failure when OVN is nil") - } -} - -// --- Test helpers --- - -func assertSuccess(t *testing.T, msg *nats.Msg, label string) { - t.Helper() - var result struct { - Success bool `json:"success"` - Error string `json:"error,omitempty"` - } - if err := json.Unmarshal(msg.Data, &result); err != nil { - t.Fatalf("%s: unmarshal: %v", label, err) - } - if !result.Success { - t.Fatalf("%s: failed: %s", label, result.Error) - } -} - -// nbdb helper factories for tests - -func nbdbLogicalRouter(name, vpcId string) *nbdb.LogicalRouter { - return &nbdb.LogicalRouter{ - Name: name, - ExternalIDs: map[string]string{ - "spinifex:vpc_id": vpcId, - }, - } -} - -func nbdbLogicalSwitch(name, subnetId, vpcId string) *nbdb.LogicalSwitch { - return &nbdb.LogicalSwitch{ - Name: name, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": subnetId, - "spinifex:vpc_id": vpcId, - }, - } -} - -func nbdbLogicalRouterPort(name, subnetId, vpcId string) *nbdb.LogicalRouterPort { - return &nbdb.LogicalRouterPort{ - Name: name, - MAC: "02:00:00:aa:bb:cc", - Networks: []string{"10.0.2.1/24"}, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": subnetId, - "spinifex:vpc_id": vpcId, - }, - } -} - -func nbdbLogicalSwitchPortRouter(name, routerPort, subnetId, vpcId string) *nbdb.LogicalSwitchPort { - return &nbdb.LogicalSwitchPort{ - Name: name, - Type: "router", - Addresses: []string{"router"}, - Options: map[string]string{ - "router-port": routerPort, - }, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": subnetId, - "spinifex:vpc_id": vpcId, - }, - } -} - -func nbdbDHCPOptions(cidr, subnetId, vpcId string) *nbdb.DHCPOptions { - return &nbdb.DHCPOptions{ - CIDR: cidr, - Options: map[string]string{ - "router": "10.0.2.1", - "lease_time": "3600", - }, - ExternalIDs: map[string]string{ - "spinifex:subnet_id": subnetId, - "spinifex:vpc_id": vpcId, - }, - } -} - -// --- Bridge mode tests --- - -func TestTopologyHandler_IGWAttach_DirectBridge_NoNatAddresses(t *testing.T) { - // In direct bridge mode, the localnet port should NOT have nat-addresses=router - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeDirect)) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create VPC router - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-direct1", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-direct1", - "spinifex:cidr": "10.0.0.0/16", - }, - }) - - // Attach IGW - evt := types.IGWEvent{InternetGatewayId: "igw-d1", VpcId: "vpc-direct1"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.igw-attach: %v", err) - } - assertSuccess(t, resp, "attach IGW direct bridge") - - // Verify localnet port does NOT have nat-addresses - port, err := mock.GetLogicalSwitchPort(ctx, "ext-port-vpc-direct1") - if err != nil { - t.Fatalf("expected localnet port: %v", err) - } - if _, ok := port.Options["nat-addresses"]; ok { - t.Errorf("direct bridge mode should NOT have nat-addresses option, got %q", port.Options["nat-addresses"]) - } -} - -func TestTopologyHandler_IGWAttach_MacvlanMode_HasNatAddresses(t *testing.T) { - // In macvlan mode (default), the localnet port SHOULD have nat-addresses=router - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeMacvlan)) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create VPC router - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-mv1", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-mv1", - "spinifex:cidr": "10.0.0.0/16", - }, - }) - - // Attach IGW - evt := types.IGWEvent{InternetGatewayId: "igw-mv1", VpcId: "vpc-mv1"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.igw-attach: %v", err) - } - assertSuccess(t, resp, "attach IGW macvlan") - - // Verify localnet port HAS nat-addresses=router - port, err := mock.GetLogicalSwitchPort(ctx, "ext-port-vpc-mv1") - if err != nil { - t.Fatalf("expected localnet port: %v", err) - } - if port.Options["nat-addresses"] != "router" { - t.Errorf("macvlan mode should have nat-addresses=router, got %q", port.Options["nat-addresses"]) - } -} - -func TestTopologyHandler_AddNAT_DirectBridge_DistributedNAT(t *testing.T) { - // In direct bridge mode, DNAT rules should have ExternalMAC and LogicalPort set - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeDirect)) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create VPC router - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-dnat1", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-dnat1", - "spinifex:cidr": "10.0.0.0/16", - }, - }) - - // Add NAT with MAC and port (simulating a VM with public IP) - evt := NATEvent{ - VpcId: "vpc-dnat1", - ExternalIP: "192.168.1.201", - LogicalIP: "10.0.1.5", - PortName: "port-eni-1234", - MAC: "02:00:00:ab:cd:ef", - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicAddNAT, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.add-nat: %v", err) - } - assertSuccess(t, resp, "add NAT direct bridge") - - // Verify the NAT rule has ExternalMAC and LogicalPort (distributed NAT) - router, err := mock.GetLogicalRouter(ctx, "vpc-vpc-dnat1") - if err != nil { - t.Fatalf("expected router: %v", err) - } - if len(router.NAT) != 1 { - t.Fatalf("expected 1 NAT rule, got %d", len(router.NAT)) - } - nat := mock.nats[router.NAT[0]] - if nat.ExternalMAC == nil || *nat.ExternalMAC != "02:00:00:ab:cd:ef" { - t.Errorf("expected ExternalMAC=02:00:00:ab:cd:ef for distributed NAT, got %v", nat.ExternalMAC) - } - if nat.LogicalPort == nil || *nat.LogicalPort != "port-eni-1234" { - t.Errorf("expected LogicalPort=port-eni-1234 for distributed NAT, got %v", nat.LogicalPort) - } -} - -func TestTopologyHandler_AddNAT_MacvlanMode_CentralizedNAT(t *testing.T) { - // In macvlan mode, DNAT rules should NOT have ExternalMAC/LogicalPort - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeMacvlan)) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Pre-create VPC router - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-cnat1", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-cnat1", - "spinifex:cidr": "10.0.0.0/16", - }, - }) - - // Add NAT with MAC and port — macvlan mode should ignore them - evt := NATEvent{ - VpcId: "vpc-cnat1", - ExternalIP: "192.168.1.201", - LogicalIP: "10.0.1.5", - PortName: "port-eni-5678", - MAC: "02:00:00:11:22:33", - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicAddNAT, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.add-nat: %v", err) - } - assertSuccess(t, resp, "add NAT macvlan") - - // Verify the NAT rule does NOT have ExternalMAC/LogicalPort (centralized NAT) - router, err := mock.GetLogicalRouter(ctx, "vpc-vpc-cnat1") - if err != nil { - t.Fatalf("expected router: %v", err) - } - if len(router.NAT) != 1 { - t.Fatalf("expected 1 NAT rule, got %d", len(router.NAT)) - } - nat := mock.nats[router.NAT[0]] - if nat.ExternalMAC != nil { - t.Errorf("macvlan mode should NOT have ExternalMAC, got %v", *nat.ExternalMAC) - } - if nat.LogicalPort != nil { - t.Errorf("macvlan mode should NOT have LogicalPort, got %v", *nat.LogicalPort) - } -} - -func TestTopologyHandler_IGWAttach_VethMode_HasNatAddresses(t *testing.T) { - // In veth mode, the localnet port SHOULD have nat-addresses=router (centralized NAT) - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeVeth)) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-veth1", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-veth1", - "spinifex:cidr": "10.0.0.0/16", - }, - }) - - evt := types.IGWEvent{InternetGatewayId: "igw-veth1", VpcId: "vpc-veth1"} - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicIGWAttach, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.igw-attach: %v", err) - } - assertSuccess(t, resp, "attach IGW veth") - - port, err := mock.GetLogicalSwitchPort(ctx, "ext-port-vpc-veth1") - if err != nil { - t.Fatalf("expected localnet port: %v", err) - } - if port.Options["nat-addresses"] != "router" { - t.Errorf("veth mode should have nat-addresses=router, got %q", port.Options["nat-addresses"]) - } -} - -func TestTopologyHandler_AddNAT_VethMode_CentralizedNAT(t *testing.T) { - // In veth mode, DNAT rules should NOT have ExternalMAC/LogicalPort (centralized NAT) - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeVeth)) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-vnat1", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-vnat1", - "spinifex:cidr": "10.0.0.0/16", - }, - }) - - evt := NATEvent{ - VpcId: "vpc-vnat1", - ExternalIP: "192.168.1.201", - LogicalIP: "10.0.1.5", - PortName: "port-eni-veth1", - MAC: "02:00:00:aa:bb:cc", - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicAddNAT, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.add-nat: %v", err) - } - assertSuccess(t, resp, "add NAT veth") - - router, err := mock.GetLogicalRouter(ctx, "vpc-vpc-vnat1") - if err != nil { - t.Fatalf("expected router: %v", err) - } - if len(router.NAT) != 1 { - t.Fatalf("expected 1 NAT rule, got %d", len(router.NAT)) - } - nat := mock.nats[router.NAT[0]] - if nat.ExternalMAC != nil { - t.Errorf("veth mode should NOT have ExternalMAC, got %v", *nat.ExternalMAC) - } - if nat.LogicalPort != nil { - t.Errorf("veth mode should NOT have LogicalPort, got %v", *nat.LogicalPort) - } -} - -func TestTopologyHandler_AddNAT_CleansStaleRulesFromOtherVPCs(t *testing.T) { - // When a public IP is reused across VPCs (e.g. instance terminated in the - // default VPC, IP returned to pool, then LB allocates it in a new VPC), - // the fire-and-forget vpc.delete-nat for the old instance may not have been - // processed. handleAddNAT must clean up stale NAT rules from ALL routers, - // not just the target VPC's router. - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeVeth)) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Create two VPC routers (simulating default VPC and a new test VPC). - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-default", - ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-default", "spinifex:cidr": "10.0.0.0/16"}, - }) - _ = mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-new", - ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-new", "spinifex:cidr": "10.200.0.0/16"}, - }) - - // Simulate: instance in default VPC got public IP 192.168.1.243. - // Add NAT rule to default VPC's router (as if vpc.add-nat was processed). - oldEvt := NATEvent{ - VpcId: "vpc-default", - ExternalIP: "192.168.1.243", - LogicalIP: "10.0.1.5", - PortName: "port-eni-old", - MAC: "02:00:00:aa:aa:aa", - } - data, _ := json.Marshal(oldEvt) - resp, err := nc.Request(TopicAddNAT, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.add-nat (old): %v", err) - } - assertSuccess(t, resp, "add old NAT") - - // Verify the stale NAT exists on the default VPC's router. - defaultRouter, _ := mock.GetLogicalRouter(ctx, "vpc-vpc-default") - if len(defaultRouter.NAT) != 1 { - t.Fatalf("expected 1 NAT on default router, got %d", len(defaultRouter.NAT)) - } - - // Simulate: instance terminated, IP returned to pool, and a new LB in - // the new VPC re-allocates the same IP. vpc.delete-nat was NOT processed - // (fire-and-forget race). Now vpc.add-nat fires for the new VPC. - newEvt := NATEvent{ - VpcId: "vpc-new", - ExternalIP: "192.168.1.243", - LogicalIP: "10.200.1.10", - PortName: "port-eni-new", - MAC: "02:00:00:bb:bb:bb", - } - data, _ = json.Marshal(newEvt) - resp, err = nc.Request(TopicAddNAT, data, 5_000_000_000) - if err != nil { - t.Fatalf("request vpc.add-nat (new): %v", err) - } - assertSuccess(t, resp, "add new NAT (should clean stale)") - - // The stale NAT rule on the default VPC's router must be gone. - defaultRouter, _ = mock.GetLogicalRouter(ctx, "vpc-vpc-default") - if len(defaultRouter.NAT) != 0 { - t.Errorf("stale NAT rule was NOT cleaned from default VPC's router; got %d NAT rules", len(defaultRouter.NAT)) - } - - // The new VPC's router must have exactly 1 NAT rule with the new logical IP. - newRouter, _ := mock.GetLogicalRouter(ctx, "vpc-vpc-new") - if len(newRouter.NAT) != 1 { - t.Fatalf("expected 1 NAT on new router, got %d", len(newRouter.NAT)) - } - newNAT := mock.nats[newRouter.NAT[0]] - if newNAT.LogicalIP != "10.200.1.10" { - t.Errorf("new NAT logical IP = %q, want 10.200.1.10", newNAT.LogicalIP) - } -} - -// ensureLocalnetOptions retrofits options:nat-addresses on a localnet port -// whose mode no longer matches (e.g. created during a pre-Fix 1 run where -// detectBridgeMode returned direct because the veth had not been -// recreated after reboot). mulga-998.b Fix 3. - -func TestEnsureLocalnetOptions_AddsNatAddressesInCentralizedMode(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{Name: "ext-vpc-x"}) - _ = mock.CreateLogicalSwitchPort(ctx, "ext-vpc-x", &nbdb.LogicalSwitchPort{ - Name: "ext-port-vpc-x", - Type: "localnet", - Addresses: []string{"unknown"}, - Options: map[string]string{"network_name": "external"}, - }) - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeVeth)) - if err := topo.ensureLocalnetOptions(ctx, "ext-port-vpc-x"); err != nil { - t.Fatalf("ensureLocalnetOptions: %v", err) - } - port, _ := mock.GetLogicalSwitchPort(ctx, "ext-port-vpc-x") - if port.Options["nat-addresses"] != "router" { - t.Errorf("expected nat-addresses=router after retrofit, got %q", port.Options["nat-addresses"]) - } - if port.Options["network_name"] != "external" { - t.Errorf("network_name clobbered: got %q", port.Options["network_name"]) - } -} - -func TestEnsureLocalnetOptions_RemovesNatAddressesInDirectMode(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{Name: "ext-vpc-y"}) - _ = mock.CreateLogicalSwitchPort(ctx, "ext-vpc-y", &nbdb.LogicalSwitchPort{ - Name: "ext-port-vpc-y", - Type: "localnet", - Addresses: []string{"unknown"}, - Options: map[string]string{"network_name": "external", "nat-addresses": "router"}, - }) - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeDirect)) - if err := topo.ensureLocalnetOptions(ctx, "ext-port-vpc-y"); err != nil { - t.Fatalf("ensureLocalnetOptions: %v", err) - } - port, _ := mock.GetLogicalSwitchPort(ctx, "ext-port-vpc-y") - if _, ok := port.Options["nat-addresses"]; ok { - t.Errorf("nat-addresses should be removed in direct mode, got %q", port.Options["nat-addresses"]) - } - if port.Options["network_name"] != "external" { - t.Errorf("network_name clobbered: got %q", port.Options["network_name"]) - } -} - -func TestEnsureLocalnetOptions_NoOpWhenAlreadyCorrect(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{Name: "ext-vpc-z"}) - _ = mock.CreateLogicalSwitchPort(ctx, "ext-vpc-z", &nbdb.LogicalSwitchPort{ - Name: "ext-port-vpc-z", - Type: "localnet", - Addresses: []string{"unknown"}, - Options: map[string]string{"network_name": "external", "nat-addresses": "router"}, - }) - startUpdates := mock.UpdateLogicalSwitchPortCalls - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeVeth)) - if err := topo.ensureLocalnetOptions(ctx, "ext-port-vpc-z"); err != nil { - t.Fatalf("ensureLocalnetOptions: %v", err) - } - if mock.UpdateLogicalSwitchPortCalls != startUpdates { - t.Errorf("expected no UpdateLogicalSwitchPort call when already correct (idempotent); got %d new calls", - mock.UpdateLogicalSwitchPortCalls-startUpdates) - } -} - -func TestEnsureLocalnetOptions_NoOpInDirectModeWhenAbsent(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{Name: "ext-vpc-d"}) - _ = mock.CreateLogicalSwitchPort(ctx, "ext-vpc-d", &nbdb.LogicalSwitchPort{ - Name: "ext-port-vpc-d", - Type: "localnet", - Addresses: []string{"unknown"}, - Options: map[string]string{"network_name": "external"}, - }) - start := mock.UpdateLogicalSwitchPortCalls - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeDirect)) - if err := topo.ensureLocalnetOptions(ctx, "ext-port-vpc-d"); err != nil { - t.Fatalf("ensureLocalnetOptions: %v", err) - } - if mock.UpdateLogicalSwitchPortCalls != start { - t.Errorf("direct mode + no nat-addresses should be a no-op; got %d new calls", - mock.UpdateLogicalSwitchPortCalls-start) - } -} - -func TestEnsureLocalnetOptions_HandlesNilOptionsMap(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{Name: "ext-vpc-n"}) - _ = mock.CreateLogicalSwitchPort(ctx, "ext-vpc-n", &nbdb.LogicalSwitchPort{ - Name: "ext-port-vpc-n", - Type: "localnet", - Addresses: []string{"unknown"}, - }) - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeVeth)) - if err := topo.ensureLocalnetOptions(ctx, "ext-port-vpc-n"); err != nil { - t.Fatalf("ensureLocalnetOptions: %v", err) - } - port, _ := mock.GetLogicalSwitchPort(ctx, "ext-port-vpc-n") - if port.Options["nat-addresses"] != "router" { - t.Errorf("expected nat-addresses=router after nil-map init, got %q", port.Options["nat-addresses"]) - } -} - -func TestEnsureLocalnetOptions_RestoresNetworkName(t *testing.T) { - // Operator (or test drill) cleared options entirely — retrofit must - // restore both network_name=external and nat-addresses=router. - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{Name: "ext-vpc-r"}) - _ = mock.CreateLogicalSwitchPort(ctx, "ext-vpc-r", &nbdb.LogicalSwitchPort{ - Name: "ext-port-vpc-r", - Type: "localnet", - Addresses: []string{"unknown"}, - Options: map[string]string{}, - }) - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeVeth)) - if err := topo.ensureLocalnetOptions(ctx, "ext-port-vpc-r"); err != nil { - t.Fatalf("ensureLocalnetOptions: %v", err) - } - port, _ := mock.GetLogicalSwitchPort(ctx, "ext-port-vpc-r") - if port.Options["network_name"] != "external" { - t.Errorf("expected network_name=external after retrofit, got %q", port.Options["network_name"]) - } - if port.Options["nat-addresses"] != "router" { - t.Errorf("expected nat-addresses=router after retrofit, got %q", port.Options["nat-addresses"]) - } -} - -func TestEnsureLocalnetOptions_RestoresNetworkNameDirectMode(t *testing.T) { - // Direct mode: network_name still required, nat-addresses still absent. - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{Name: "ext-vpc-rd"}) - _ = mock.CreateLogicalSwitchPort(ctx, "ext-vpc-rd", &nbdb.LogicalSwitchPort{ - Name: "ext-port-vpc-rd", - Type: "localnet", - Addresses: []string{"unknown"}, - Options: map[string]string{}, - }) - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeDirect)) - if err := topo.ensureLocalnetOptions(ctx, "ext-port-vpc-rd"); err != nil { - t.Fatalf("ensureLocalnetOptions: %v", err) - } - port, _ := mock.GetLogicalSwitchPort(ctx, "ext-port-vpc-rd") - if port.Options["network_name"] != "external" { - t.Errorf("expected network_name=external after retrofit, got %q", port.Options["network_name"]) - } - if _, ok := port.Options["nat-addresses"]; ok { - t.Errorf("direct mode should not set nat-addresses, got %q", port.Options["nat-addresses"]) - } -} - -func TestRetrofitAllExternalLocalnetOptions_FixesClearedOptions(t *testing.T) { - // Walks every ext-* switch in OVN (authoritative) and retrofits options - // on its localnet port. Covers VPCs whose IGW KV record is missing or - // whose options were cleared manually (mulga-998.b Drill 2). - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - // Two external switches with cleared options, one subnet switch (must be - // skipped because role != external), one external switch already correct. - for _, vpcID := range []string{"vpc-a", "vpc-b"} { - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{ - Name: "ext-" + vpcID, - ExternalIDs: map[string]string{"spinifex:role": "external", "spinifex:vpc_id": vpcID}, - }) - _ = mock.CreateLogicalSwitchPort(ctx, "ext-"+vpcID, &nbdb.LogicalSwitchPort{ - Name: "ext-port-" + vpcID, - Type: "localnet", - Addresses: []string{"unknown"}, - Options: map[string]string{}, - }) - } - // A subnet switch that must NOT be retrofitted. - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{ - Name: "subnet-foo", - ExternalIDs: map[string]string{"spinifex:subnet_id": "subnet-foo"}, - }) - // An ext switch already correct — must remain a no-op. - _ = mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{ - Name: "ext-vpc-ok", - ExternalIDs: map[string]string{"spinifex:role": "external", "spinifex:vpc_id": "vpc-ok"}, - }) - _ = mock.CreateLogicalSwitchPort(ctx, "ext-vpc-ok", &nbdb.LogicalSwitchPort{ - Name: "ext-port-vpc-ok", - Type: "localnet", - Addresses: []string{"unknown"}, - Options: map[string]string{"network_name": "external", "nat-addresses": "router"}, - }) - startUpdates := mock.UpdateLogicalSwitchPortCalls - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeVeth)) - topo.RetrofitAllExternalLocalnetOptions(ctx) - - for _, vpcID := range []string{"vpc-a", "vpc-b"} { - port, err := mock.GetLogicalSwitchPort(ctx, "ext-port-"+vpcID) - if err != nil { - t.Fatalf("get port: %v", err) - } - if port.Options["network_name"] != "external" { - t.Errorf("%s: expected network_name=external, got %q", vpcID, port.Options["network_name"]) - } - if port.Options["nat-addresses"] != "router" { - t.Errorf("%s: expected nat-addresses=router, got %q", vpcID, port.Options["nat-addresses"]) - } - } - // Expect exactly 2 updates (vpc-a + vpc-b); vpc-ok must be idempotent no-op. - if got := mock.UpdateLogicalSwitchPortCalls - startUpdates; got != 2 { - t.Errorf("expected 2 UpdateLogicalSwitchPort calls (one per stale ext port); got %d", got) - } -} - -func TestEnsureLocalnetOptions_ErrorOnMissingPort(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock, WithBridgeMode(BridgeModeVeth)) - err := topo.ensureLocalnetOptions(ctx, "ext-port-missing") - if err == nil { - t.Fatal("expected error when port missing") - } - if !strings.Contains(err.Error(), "ext-port-missing") { - t.Errorf("error should name the port; got: %v", err) - } -} - -// reconcileIGW is now ensure-then-skip per child with no rollback paths, so -// running against an empty NB DB must produce a fully-formed IGW topology in -// one pass. -func TestReconcileIGW_FullCreate(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - topo := NewTopologyHandler(mock, - WithBridgeMode(BridgeModeDirect), - WithChassisNames([]string{"chassis-A"}), - ) - if err := mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-vpc-fc"}); err != nil { - t.Fatalf("seed router: %v", err) - } - - if err := topo.reconcileIGW(ctx, "vpc-fc", "igw-fc"); err != nil { - t.Fatalf("reconcileIGW: %v", err) - } - - if _, err := mock.GetLogicalSwitch(ctx, "ext-vpc-fc"); err != nil { - t.Errorf("ext switch missing: %v", err) - } - lsp, err := mock.GetLogicalSwitchPort(ctx, "ext-port-vpc-fc") - if err != nil { - t.Fatalf("localnet port missing: %v", err) - } - if lsp.Type != "localnet" || lsp.Options["network_name"] != "external" { - t.Errorf("localnet port misconfigured: type=%q options=%v", lsp.Type, lsp.Options) - } - if _, err := mock.GetLogicalRouterPort(ctx, "gw-vpc-fc"); err != nil { - t.Errorf("gateway LRP missing: %v", err) - } - if _, err := mock.GetLogicalSwitchPort(ctx, "gw-port-vpc-fc"); err != nil { - t.Errorf("router-LSP missing: %v", err) - } - route, err := mock.FindStaticRoute(ctx, "vpc-vpc-fc", "0.0.0.0/0") - if err != nil { - t.Fatalf("FindStaticRoute: %v", err) - } - if route == nil { - t.Errorf("default route missing") - } - gc, err := mock.GetGatewayChassisByName(ctx, "gw-vpc-fc-chassis-A") - if err != nil { - t.Fatalf("GetGatewayChassisByName: %v", err) - } - if gc == nil { - t.Errorf("gateway chassis binding missing") - } -} - -// Partial state: switch + LRP + router-LSP exist but localnet LSP is missing -// (the cell-10 nightly's bootstrap topology shape). reconcileIGW must add the -// missing localnet without touching the rest. -func TestReconcileIGW_PartialState_MissingLocalnet(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - topo := NewTopologyHandler(mock, - WithBridgeMode(BridgeModeDirect), - WithChassisNames([]string{"chassis-A"}), - ) - - if err := mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-vpc-pml"}); err != nil { - t.Fatalf("seed router: %v", err) - } - if err := mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{ - Name: "ext-vpc-pml", - ExternalIDs: map[string]string{ - "spinifex:vpc_id": "vpc-pml", - "spinifex:role": "external", - }, - }); err != nil { - t.Fatalf("seed ext switch: %v", err) - } - if err := mock.CreateLogicalRouterPort(ctx, "vpc-vpc-pml", &nbdb.LogicalRouterPort{ - Name: "gw-vpc-pml", - MAC: "02:00:00:aa:bb:01", - Networks: []string{"169.254.0.1/30"}, - ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-pml", "spinifex:role": "gateway"}, - }); err != nil { - t.Fatalf("seed LRP: %v", err) - } - if err := mock.CreateLogicalSwitchPort(ctx, "ext-vpc-pml", &nbdb.LogicalSwitchPort{ - Name: "gw-port-vpc-pml", - Type: "router", - Addresses: []string{"router"}, - Options: map[string]string{"router-port": "gw-vpc-pml"}, - ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-pml"}, - }); err != nil { - t.Fatalf("seed router-LSP: %v", err) - } - - if err := topo.reconcileIGW(ctx, "vpc-pml", "igw-pml"); err != nil { - t.Fatalf("reconcileIGW: %v", err) - } - - lsp, err := mock.GetLogicalSwitchPort(ctx, "ext-port-vpc-pml") - if err != nil { - t.Fatalf("localnet port still missing after self-heal: %v", err) - } - if lsp.Type != "localnet" { - t.Errorf("expected type=localnet, got %q", lsp.Type) - } - if lsp.Options["network_name"] != "external" { - t.Errorf("expected network_name=external, got %q", lsp.Options["network_name"]) - } -} - -// Partial state: switch + localnet exist but the gateway LRP and router-LSP -// are missing. reconcileIGW must finish what a previously-failed pass started. -func TestReconcileIGW_PartialState_MissingLRP(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - topo := NewTopologyHandler(mock, - WithBridgeMode(BridgeModeDirect), - WithChassisNames([]string{"chassis-A"}), - ) - - if err := mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-vpc-pmr"}); err != nil { - t.Fatalf("seed router: %v", err) - } - if err := mock.CreateLogicalSwitch(ctx, &nbdb.LogicalSwitch{ - Name: "ext-vpc-pmr", - ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-pmr", "spinifex:role": "external"}, - }); err != nil { - t.Fatalf("seed ext switch: %v", err) - } - if err := mock.CreateLogicalSwitchPort(ctx, "ext-vpc-pmr", &nbdb.LogicalSwitchPort{ - Name: "ext-port-vpc-pmr", - Type: "localnet", - Addresses: []string{"unknown"}, - Options: map[string]string{"network_name": "external"}, - ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-pmr"}, - }); err != nil { - t.Fatalf("seed localnet: %v", err) - } - - if err := topo.reconcileIGW(ctx, "vpc-pmr", "igw-pmr"); err != nil { - t.Fatalf("reconcileIGW: %v", err) - } - - if _, err := mock.GetLogicalRouterPort(ctx, "gw-vpc-pmr"); err != nil { - t.Errorf("gateway LRP not created: %v", err) - } - if _, err := mock.GetLogicalSwitchPort(ctx, "gw-port-vpc-pmr"); err != nil { - t.Errorf("router-LSP not created: %v", err) - } - gc, err := mock.GetGatewayChassisByName(ctx, "gw-vpc-pmr-chassis-A") - if err != nil { - t.Fatalf("GetGatewayChassisByName: %v", err) - } - if gc == nil { - t.Errorf("gateway chassis binding missing after heal") - } -} - -// Full pre-existing topology: reconcileIGW must be a no-op (no errors, no new -// entities, no duplicate default route). This is the steady-state self-heal -// path that runs on every vpcd start. -func TestReconcileIGW_FullExisting(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - topo := NewTopologyHandler(mock, - WithBridgeMode(BridgeModeDirect), - WithChassisNames([]string{"chassis-A"}), - ) - if err := mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-vpc-fe"}); err != nil { - t.Fatalf("seed router: %v", err) - } - - // First pass creates the topology end-to-end. - if err := topo.reconcileIGW(ctx, "vpc-fe", "igw-fe"); err != nil { - t.Fatalf("first reconcileIGW: %v", err) - } - extBefore, err := mock.GetLogicalSwitch(ctx, "ext-vpc-fe") - if err != nil { - t.Fatalf("ext switch missing after first pass: %v", err) - } - lrBefore, err := mock.GetLogicalRouter(ctx, "vpc-vpc-fe") - if err != nil { - t.Fatalf("router fetch: %v", err) - } - if got := len(lrBefore.StaticRoutes); got != 1 { - t.Fatalf("first pass: expected 1 default route, got %d", got) - } - - // Second pass against the now-complete topology. - if err := topo.reconcileIGW(ctx, "vpc-fe", "igw-fe"); err != nil { - t.Fatalf("second reconcileIGW: %v", err) - } - extAfter, err := mock.GetLogicalSwitch(ctx, "ext-vpc-fe") - if err != nil { - t.Fatalf("ext switch fetch: %v", err) - } - if extAfter.UUID != extBefore.UUID { - t.Errorf("ext switch UUID changed (was %q, now %q) — second pass must not recreate", extBefore.UUID, extAfter.UUID) - } - lrAfter, err := mock.GetLogicalRouter(ctx, "vpc-vpc-fe") - if err != nil { - t.Fatalf("router fetch: %v", err) - } - if got := len(lrAfter.StaticRoutes); got != 1 { - t.Errorf("second pass: expected 1 default route, got %d", got) - } -} - -// AddStaticRoute is non-idempotent on its own — without FindStaticRoute, every -// retry leaves a fresh duplicate 0.0.0.0/0 row. Pre-seed a complete topology -// (exercising the path where reconcileIGW finds an existing route) and verify -// no duplicate is added. -func TestReconcileIGW_DuplicateRoute(t *testing.T) { - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - topo := NewTopologyHandler(mock, - WithBridgeMode(BridgeModeDirect), - WithChassisNames([]string{"chassis-A"}), - ) - if err := mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-vpc-dr"}); err != nil { - t.Fatalf("seed router: %v", err) - } - if err := topo.reconcileIGW(ctx, "vpc-dr", "igw-dr"); err != nil { - t.Fatalf("first reconcileIGW: %v", err) - } - - // Multiple subsequent passes must keep the route count at exactly 1. - for i := range 3 { - if err := topo.reconcileIGW(ctx, "vpc-dr", "igw-dr"); err != nil { - t.Fatalf("reconcileIGW pass %d: %v", i+2, err) - } - lr, err := mock.GetLogicalRouter(ctx, "vpc-vpc-dr") - if err != nil { - t.Fatalf("router fetch pass %d: %v", i+2, err) - } - if got := len(lr.StaticRoutes); got != 1 { - t.Fatalf("pass %d: expected 1 default route, got %d", i+2, got) - } - } -} - -// TestGwLrpRange exercises the gateway LRP IP range derivation. mulga-siv-36 -// dropped the link-local fallback in centralized NAT — auto-derive must yield -// an in-subnet range whenever the operator did not configure one explicitly, -// or fail loudly so the IGW attach surfaces the misconfiguration. -func TestGwLrpRange(t *testing.T) { - cases := []struct { - name string - pool *ExternalPoolConfig - wantOK bool - wantStart string - wantEnd string - wantPfx int - }{ - { - name: "nil pool", - pool: nil, - wantOK: false, - }, - { - name: "explicit range wins over auto", - pool: &ExternalPoolConfig{ - Gateway: "192.168.1.1", - PrefixLen: 24, - RangeStart: "192.168.1.100", - RangeEnd: "192.168.1.110", - GwLrpRangeStart: "192.168.1.50", - GwLrpRangeEnd: "192.168.1.59", - }, - wantOK: true, - wantStart: "192.168.1.50", - wantEnd: "192.168.1.59", - wantPfx: 24, - }, - { - name: "explicit range invalid falls through to auto", - pool: &ExternalPoolConfig{ - Gateway: "192.168.1.1", - PrefixLen: 24, - RangeStart: "192.168.1.100", - RangeEnd: "192.168.1.110", - GwLrpRangeStart: "not-an-ip", - GwLrpRangeEnd: "192.168.1.59", - }, - wantOK: true, - wantStart: "192.168.1.239", - wantEnd: "192.168.1.254", - wantPfx: 24, - }, - { - name: "auto-derive /24 default end of subnet", - pool: &ExternalPoolConfig{ - Gateway: "192.168.1.1", - PrefixLen: 24, - RangeStart: "192.168.1.100", - RangeEnd: "192.168.1.110", - }, - wantOK: true, - wantStart: "192.168.1.239", - wantEnd: "192.168.1.254", - wantPfx: 24, - }, - { - name: "auto-derive overlap shifts below RangeStart", - pool: &ExternalPoolConfig{ - Gateway: "192.168.1.1", - PrefixLen: 24, - RangeStart: "192.168.1.245", - RangeEnd: "192.168.1.254", - }, - wantOK: true, - wantStart: "192.168.1.229", - wantEnd: "192.168.1.244", - wantPfx: 24, - }, - { - name: "auto-derive guards gateway at start", - pool: &ExternalPoolConfig{ - Gateway: "192.168.1.240", - PrefixLen: 24, - }, - wantOK: true, - wantStart: "192.168.1.241", - wantEnd: "192.168.1.254", - wantPfx: 24, - }, - { - name: "auto-derive guards gateway at end", - pool: &ExternalPoolConfig{ - Gateway: "192.168.1.254", - PrefixLen: 24, - }, - wantOK: true, - wantStart: "192.168.1.239", - wantEnd: "192.168.1.253", - wantPfx: 24, - }, - { - name: "auto-derive guards gateway in middle", - pool: &ExternalPoolConfig{ - Gateway: "192.168.1.245", - PrefixLen: 24, - }, - wantOK: true, - wantStart: "192.168.1.246", - wantEnd: "192.168.1.254", - wantPfx: 24, - }, - { - name: "default prefix 24 when missing", - pool: &ExternalPoolConfig{ - Gateway: "10.0.0.1", - }, - wantOK: true, - wantStart: "10.0.0.239", - wantEnd: "10.0.0.254", - wantPfx: 24, - }, - { - name: "unparseable gateway returns ok=false", - pool: &ExternalPoolConfig{ - Gateway: "not-an-ip", - PrefixLen: 24, - }, - wantOK: false, - }, - { - name: "/28 with full pool exhausts auto range", - pool: &ExternalPoolConfig{ - Gateway: "203.0.113.1", - PrefixLen: 28, - RangeStart: "203.0.113.2", - RangeEnd: "203.0.113.14", - }, - wantOK: false, - }, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - start, end, pfx, ok := gwLrpRange(tc.pool) - if ok != tc.wantOK { - t.Fatalf("ok=%v, want %v", ok, tc.wantOK) - } - if !ok { - return - } - if start.String() != tc.wantStart { - t.Errorf("start=%s, want %s", start, tc.wantStart) - } - if end.String() != tc.wantEnd { - t.Errorf("end=%s, want %s", end, tc.wantEnd) - } - if pfx != tc.wantPfx { - t.Errorf("prefix=%d, want %d", pfx, tc.wantPfx) - } - }) - } -} - -// --- Phase 4: SG enforcement port-group membership --- - -// setupPortSGFixture stands up a switch + DHCP options + the port groups for -// the named SGs so a vpc.create-port event can join them. -func setupPortSGFixture(t *testing.T, mock *MockOVNClient, ctx context.Context, vpcId, subnetId, cidr string, sgIds []string) { - t.Helper() - if err := mock.CreateLogicalRouter(ctx, nbdbLogicalRouter("vpc-"+vpcId, vpcId)); err != nil { - t.Fatalf("CreateLogicalRouter: %v", err) - } - if err := mock.CreateLogicalSwitch(ctx, nbdbLogicalSwitch("subnet-"+subnetId, subnetId, vpcId)); err != nil { - t.Fatalf("CreateLogicalSwitch: %v", err) - } - if _, err := mock.CreateDHCPOptions(ctx, nbdbDHCPOptions(cidr, subnetId, vpcId)); err != nil { - t.Fatalf("CreateDHCPOptions: %v", err) - } - for _, sgId := range sgIds { - pg := portGroupName(sgId) - if err := mock.CreatePortGroup(ctx, pg, nil); err != nil { - t.Fatalf("CreatePortGroup %s: %v", sgId, err) - } - } -} - -func TestTopologyHandler_CreatePort_JoinsPortGroups(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - setupPortSGFixture(t, mock, ctx, "vpc-sg1", "subnet-sg1", "10.0.1.0/24", []string{"sg-1111", "sg-2222"}) - - evt := PortEvent{ - NetworkInterfaceId: "eni-sg1", - SubnetId: "subnet-sg1", - VpcId: "vpc-sg1", - PrivateIpAddress: "10.0.1.10", - MacAddress: "02:00:00:00:00:01", - SecurityGroupIds: []string{"sg-1111", "sg-2222"}, - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicCreatePort, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertSuccess(t, resp, "create port with SGs") - - lsp, err := mock.GetLogicalSwitchPort(ctx, "port-eni-sg1") - if err != nil { - t.Fatalf("GetLogicalSwitchPort: %v", err) - } - - for _, sgId := range []string{"sg-1111", "sg-2222"} { - pg := mock.portGroups[portGroupName(sgId)] - if !slices.Contains(pg.Ports, lsp.UUID) { - t.Errorf("expected %s membership in port group %s, got %v", lsp.UUID, sgId, pg.Ports) - } - } -} - -func TestTopologyHandler_CreatePort_MissingPortGroup_NoLSPCreated(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // Switch + DHCP, but the SG's port group is intentionally absent. - setupPortSGFixture(t, mock, ctx, "vpc-sg2", "subnet-sg2", "10.0.2.0/24", nil) - - evt := PortEvent{ - NetworkInterfaceId: "eni-sg2", - SubnetId: "subnet-sg2", - VpcId: "vpc-sg2", - PrivateIpAddress: "10.0.2.10", - MacAddress: "02:00:00:00:00:02", - SecurityGroupIds: []string{"sg-missing"}, - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicCreatePort, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertFailure(t, resp, "create port with missing SG port group") - - if _, err := mock.GetLogicalSwitchPort(ctx, "port-eni-sg2"); err == nil { - t.Error("expected LSP not to be created when atomic transaction fails") - } -} - -func TestTopologyHandler_DeletePort_ClearsPortGroupMembership(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - setupPortSGFixture(t, mock, ctx, "vpc-del", "subnet-del", "10.0.3.0/24", []string{"sg-del"}) - - createEvt := PortEvent{ - NetworkInterfaceId: "eni-del", - SubnetId: "subnet-del", - VpcId: "vpc-del", - PrivateIpAddress: "10.0.3.10", - MacAddress: "02:00:00:00:00:03", - SecurityGroupIds: []string{"sg-del"}, - } - data, _ := json.Marshal(createEvt) - resp, _ := nc.Request(TopicCreatePort, data, 5_000_000_000) - assertSuccess(t, resp, "create port") - - if got := len(mock.portGroups[portGroupName("sg-del")].Ports); got != 1 { - t.Fatalf("setup: expected 1 member in port group, got %d", got) - } - - delEvt := PortEvent{ - NetworkInterfaceId: "eni-del", - SubnetId: "subnet-del", - VpcId: "vpc-del", - PrivateIpAddress: "10.0.3.10", - MacAddress: "02:00:00:00:00:03", - } - data, _ = json.Marshal(delEvt) - resp, _ = nc.Request(TopicDeletePort, data, 5_000_000_000) - assertSuccess(t, resp, "delete port") - - if got := len(mock.portGroups[portGroupName("sg-del")].Ports); got != 0 { - t.Errorf("expected 0 members after delete-port, got %d", got) - } -} - -func TestTopologyHandler_UpdatePortSGs_Transitions(t *testing.T) { - cases := []struct { - name string - initial []string - desired []string - expected []string - }{ - {name: "empty_to_A", initial: nil, desired: []string{"sg-A"}, expected: []string{"sg-A"}}, - {name: "A_to_B", initial: []string{"sg-A"}, desired: []string{"sg-B"}, expected: []string{"sg-B"}}, - {name: "A_to_AB", initial: []string{"sg-A"}, desired: []string{"sg-A", "sg-B"}, expected: []string{"sg-A", "sg-B"}}, - {name: "AB_to_B", initial: []string{"sg-A", "sg-B"}, desired: []string{"sg-B"}, expected: []string{"sg-B"}}, - {name: "AB_to_empty", initial: []string{"sg-A", "sg-B"}, desired: nil, expected: nil}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - if err != nil { - t.Fatalf("subscribe: %v", err) - } - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - // All possible SGs need their port groups in place — desired \ initial - // has to find them on the way in. - allSGs := make(map[string]struct{}) - for _, s := range tc.initial { - allSGs[s] = struct{}{} - } - for _, s := range tc.desired { - allSGs[s] = struct{}{} - } - sgList := make([]string, 0, len(allSGs)) - for s := range allSGs { - sgList = append(sgList, s) - } - setupPortSGFixture(t, mock, ctx, "vpc-tr", "subnet-tr", "10.0.4.0/24", sgList) - - createEvt := PortEvent{ - NetworkInterfaceId: "eni-tr", - SubnetId: "subnet-tr", - VpcId: "vpc-tr", - PrivateIpAddress: "10.0.4.10", - MacAddress: "02:00:00:00:00:04", - SecurityGroupIds: tc.initial, - } - data, _ := json.Marshal(createEvt) - resp, _ := nc.Request(TopicCreatePort, data, 5_000_000_000) - assertSuccess(t, resp, "create port") - - updEvt := UpdatePortSGsEvent{ - NetworkInterfaceId: "eni-tr", - PrivateIpAddress: "10.0.4.10", - SecurityGroupIds: tc.desired, - } - data, _ = json.Marshal(updEvt) - resp, err = nc.Request(TopicUpdatePortSGs, data, 5_000_000_000) - if err != nil { - t.Fatalf("request: %v", err) - } - assertSuccess(t, resp, "update port SGs") - - lsp, err := mock.GetLogicalSwitchPort(ctx, "port-eni-tr") - if err != nil { - t.Fatalf("GetLogicalSwitchPort: %v", err) - } - expected := make(map[string]struct{}, len(tc.expected)) - for _, s := range tc.expected { - expected[portGroupName(s)] = struct{}{} - } - for name, pg := range mock.portGroups { - inGroup := slices.Contains(pg.Ports, lsp.UUID) - _, want := expected[name] - if inGroup && !want { - t.Errorf("port unexpectedly in %s", name) - } - if !inGroup && want { - t.Errorf("port missing from expected %s", name) - } - } - }) - } -} - -// handleCreatePort must converge SG memberships even when the LSP already -// exists from a previous failed attempt — otherwise a port stranded outside -// its SG port groups would have unrestricted traffic until the next -// reconciler pass (~30s of zero-ACL exposure on the recovery path). -func TestTopologyHandler_CreatePort_ExistingLSP_ReconcilesSGMemberships(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - setupPortSGFixture(t, mock, ctx, "vpc-recover", "subnet-recover", "10.0.6.0/24", []string{"sg-recover"}) - - // Pre-create the LSP without joining any port group, simulating a - // previous create that crashed mid-transaction in some hypothetical - // non-atomic world. - require.NoError(t, mock.CreateLogicalSwitchPort(ctx, "subnet-subnet-recover", &nbdb.LogicalSwitchPort{ - Name: "port-eni-recover", - Addresses: []string{"02:00:00:00:00:99 10.0.6.50"}, - })) - pgName := portGroupName("sg-recover") - require.NotContains(t, mock.portGroups[pgName].Ports, mock.ports["port-eni-recover"].UUID) - - evt := PortEvent{ - NetworkInterfaceId: "eni-recover", - SubnetId: "subnet-recover", - VpcId: "vpc-recover", - PrivateIpAddress: "10.0.6.50", - MacAddress: "02:00:00:00:00:99", - SecurityGroupIds: []string{"sg-recover"}, - } - data, _ := json.Marshal(evt) - resp, err := nc.Request(TopicCreatePort, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "create port with pre-existing LSP") - - // The port is now in the SG port group. The matching `_ip4` - // Address_Set is auto-derived by ovn-northd in production from port - // group port addresses and intentionally not modelled in the mock. - lsp, err := mock.GetLogicalSwitchPort(ctx, "port-eni-recover") - require.NoError(t, err) - assert.Contains(t, mock.portGroups[pgName].Ports, lsp.UUID, "existing-LSP path must reconcile SG memberships") -} - -// --- handleAddNATGateway / handleDeleteNATGateway --- -// -// These handlers are fire-and-forget (no respond()), so tests publish + flush -// + poll OVN state instead of awaiting a NATS reply. - -// natGatewayProcessed waits for the SNAT rule for subnetCidr to (dis)appear on -// router routerName. Polls because handleAdd/DeleteNATGateway is fire-and- -// forget; without a wait the test races the dispatch goroutine. -func natGatewayProcessed(t *testing.T, mock *MockOVNClient, routerName, subnetCidr string, wantPresent bool) { - t.Helper() - deadline := time.Now().Add(2 * time.Second) - for time.Now().Before(deadline) { - mock.mu.Lock() - lr := mock.routers[routerName] - present := false - if lr != nil { - for _, uuid := range lr.NAT { - if n := mock.nats[uuid]; n != nil && n.Type == "snat" && n.LogicalIP == subnetCidr { - present = true - break - } - } - } - mock.mu.Unlock() - if present == wantPresent { - return - } - time.Sleep(20 * time.Millisecond) - } - t.Fatalf("NAT-Gateway SNAT (router=%s, cidr=%s) wantPresent=%v never converged", routerName, subnetCidr, wantPresent) -} - -func TestTopologyHandler_AddNATGateway_AddsSnat(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - require.NoError(t, mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{ - Name: "vpc-vpc-natgw", - ExternalIDs: map[string]string{"spinifex:vpc_id": "vpc-natgw"}, - })) - - evt := NATGatewayEvent{ - VpcId: "vpc-natgw", - NatGatewayId: "nat-001", - PublicIp: "203.0.113.10", - SubnetCidr: "10.0.50.0/24", - } - data, _ := json.Marshal(evt) - require.NoError(t, nc.Publish(TopicAddNATGateway, data)) - require.NoError(t, nc.Flush()) - - natGatewayProcessed(t, mock, "vpc-vpc-natgw", "10.0.50.0/24", true) - - mock.mu.Lock() - defer mock.mu.Unlock() - lr := mock.routers["vpc-vpc-natgw"] - require.Len(t, lr.NAT, 1) - nat := mock.nats[lr.NAT[0]] - assert.Equal(t, "snat", nat.Type) - assert.Equal(t, "203.0.113.10", nat.ExternalIP) - assert.Equal(t, "10.0.50.0/24", nat.LogicalIP) - assert.Equal(t, "nat-001", nat.ExternalIDs["spinifex:nat_gateway_id"]) - _ = ctx -} - -func TestTopologyHandler_DeleteNATGateway_RemovesSnat(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - require.NoError(t, mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-vpc-rm"})) - - evt := NATGatewayEvent{ - VpcId: "vpc-rm", - NatGatewayId: "nat-rm", - PublicIp: "203.0.113.20", - SubnetCidr: "10.0.51.0/24", - } - data, _ := json.Marshal(evt) - require.NoError(t, nc.Publish(TopicAddNATGateway, data)) - require.NoError(t, nc.Flush()) - natGatewayProcessed(t, mock, "vpc-vpc-rm", "10.0.51.0/24", true) - - require.NoError(t, nc.Publish(TopicDeleteNATGateway, data)) - require.NoError(t, nc.Flush()) - natGatewayProcessed(t, mock, "vpc-vpc-rm", "10.0.51.0/24", false) - - mock.mu.Lock() - natCount := len(mock.routers["vpc-vpc-rm"].NAT) - mock.mu.Unlock() - assert.Zero(t, natCount, "delete must remove the SNAT rule") - _ = ctx -} - -// --- handleDeleteNAT (1:1 dnat_and_snat removal) --- - -func TestTopologyHandler_DeleteNAT_OvnNotConnected(t *testing.T) { - _, nc := startTestNATS(t) - topo := NewTopologyHandler(nil) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - data, _ := json.Marshal(NATEvent{VpcId: "v", LogicalIP: "10.0.0.1"}) - resp, err := nc.Request(TopicDeleteNAT, data, 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "delete-nat must fail when ovn is nil") -} - -func TestTopologyHandler_DeleteNAT_BadJSON(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - resp, err := nc.Request(TopicDeleteNAT, []byte("not json"), 5_000_000_000) - require.NoError(t, err) - assertFailure(t, resp, "delete-nat with malformed payload must fail") -} - -func TestTopologyHandler_DeleteNAT_Success(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - require.NoError(t, mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-vpc-dnatrm"})) - - addEvt := NATEvent{VpcId: "vpc-dnatrm", ExternalIP: "203.0.113.30", LogicalIP: "10.0.60.5"} - addData, _ := json.Marshal(addEvt) - resp, err := nc.Request(TopicAddNAT, addData, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "add NAT") - - delEvt := NATEvent{VpcId: "vpc-dnatrm", LogicalIP: "10.0.60.5"} - delData, _ := json.Marshal(delEvt) - resp, err = nc.Request(TopicDeleteNAT, delData, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "delete NAT") - - lr, _ := mock.GetLogicalRouter(ctx, "vpc-vpc-dnatrm") - assert.Empty(t, lr.NAT, "NAT rule must be removed") -} - -// TestTopologyHandler_DeleteNAT_NotFound_Idempotent: instance termination -// fans out through multiple cleanup paths (ReleasePublicIP via -// instanceCleanerAdapter AND DeleteNetworkInterface in the VPC service), and -// both publish vpc.delete-nat for the same (vpc, logical_ip). The publish is -// fire-and-forget (utils.PublishNATEvent for vpc.delete-nat), so callers -// never branch on the response — a hard error on second delete just -// pollutes triage logs without surfacing any actionable failure. Treat -// "rule already absent" as success so the second publisher doesn't emit a -// spurious ERROR. The handler still returns an error for other failure -// modes (router missing, OVN unreachable, transact failure). -func TestTopologyHandler_DeleteNAT_NotFound_Idempotent(t *testing.T) { - _, nc := startTestNATS(t) - mock := NewMockOVNClient() - _ = mock.Connect(context.Background()) - ctx := context.Background() - - topo := NewTopologyHandler(mock) - subs, err := topo.Subscribe(nc) - require.NoError(t, err) - defer func() { - for _, s := range subs { - _ = s.Unsubscribe() - } - }() - - require.NoError(t, mock.CreateLogicalRouter(ctx, &nbdb.LogicalRouter{Name: "vpc-vpc-empty"})) - - delEvt := NATEvent{VpcId: "vpc-empty", LogicalIP: "10.99.99.99"} - data, _ := json.Marshal(delEvt) - resp, err := nc.Request(TopicDeleteNAT, data, 5_000_000_000) - require.NoError(t, err) - assertSuccess(t, resp, "delete-nat for non-existent rule must be idempotent (mulga-siv-107)") -} diff --git a/spinifex/utils/account.go b/spinifex/utils/account.go index 035ff2e2a..6f8e940b7 100644 --- a/spinifex/utils/account.go +++ b/spinifex/utils/account.go @@ -85,3 +85,17 @@ func GetOrCreateKVBucket(js nats.JetStreamContext, bucketName string, history in } return kv, nil } + +// DeleteKVBucketIfExists deletes a NATS KV bucket by name. Returns nil if the +// bucket does not exist (idempotent). Used by daemon startup to tombstone +// obsolete buckets after a subsystem is removed in a schema migration. +func DeleteKVBucketIfExists(js nats.JetStreamContext, bucketName string) error { + err := js.DeleteKeyValue(bucketName) + if err == nil { + return nil + } + if errors.Is(err, nats.ErrBucketNotFound) || errors.Is(err, nats.ErrStreamNotFound) { + return nil + } + return err +} diff --git a/spinifex/utils/account_test.go b/spinifex/utils/account_test.go index d1bd004a4..5c848d098 100644 --- a/spinifex/utils/account_test.go +++ b/spinifex/utils/account_test.go @@ -114,3 +114,21 @@ func TestVersionStateMachine(t *testing.T) { require.NoError(t, err) assert.Equal(t, "5", string(entry.Value())) } + +func TestDeleteKVBucketIfExists(t *testing.T) { + _, js := startJSNATSServer(t) + + // Missing bucket is a no-op. + require.NoError(t, DeleteKVBucketIfExists(js, "ghost-bucket")) + + // Existing bucket gets deleted. + _, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "doomed-bucket"}) + require.NoError(t, err) + require.NoError(t, DeleteKVBucketIfExists(js, "doomed-bucket")) + + _, err = js.KeyValue("doomed-bucket") + require.Error(t, err, "bucket should be gone") + + // Calling again on the now-missing bucket is still a no-op (idempotent). + require.NoError(t, DeleteKVBucketIfExists(js, "doomed-bucket")) +} diff --git a/spinifex/utils/sudo.go b/spinifex/utils/sudo.go new file mode 100644 index 000000000..f7f0e4929 --- /dev/null +++ b/spinifex/utils/sudo.go @@ -0,0 +1,34 @@ +package utils + +import ( + "os" + "os/exec" +) + +// sudoCommand is the runtime implementation. Kept private so tests in other +// packages can't reassign it directly — reassign linter rule. Use +// SetSudoCommandForTest from tests. +var sudoCommand = func(name string, args ...string) *exec.Cmd { + if os.Getuid() == 0 { + return exec.Command(name, args...) + } + return exec.Command("sudo", append([]string{name}, args...)...) +} + +// SudoCommand wraps exec.Command with sudo when running as non-root. +// OVS/OVN/ip require CAP_NET_ADMIN; production daemons run as root, but +// dev environments may not. +func SudoCommand(name string, args ...string) *exec.Cmd { + return sudoCommand(name, args...) +} + +// SetSudoCommandForTest swaps the underlying command builder for the duration +// of a test, returning a restore func that callers pass to t.Cleanup. The +// indirection prevents cross-package reassignment of an exported var, which +// the reassign linter flags. Running the live binary against the dev host's +// OVS would mutate external_ids on the running cluster, so tests must stub. +func SetSudoCommandForTest(stub func(name string, args ...string) *exec.Cmd) (restore func()) { + orig := sudoCommand + sudoCommand = stub + return func() { sudoCommand = orig } +} diff --git a/spinifex/utils/sudo_test.go b/spinifex/utils/sudo_test.go new file mode 100644 index 000000000..30559b509 --- /dev/null +++ b/spinifex/utils/sudo_test.go @@ -0,0 +1,27 @@ +package utils + +import ( + "os" + "testing" +) + +func TestSudoCommand_NonRoot(t *testing.T) { + cmd := SudoCommand("ovs-vsctl", "br-exists", "br-int") + args := cmd.Args + + if os.Getuid() == 0 { + if args[0] != "ovs-vsctl" { + t.Errorf("as root, expected args[0]='ovs-vsctl', got %q", args[0]) + } + return + } + if args[0] != "sudo" { + t.Errorf("as non-root, expected args[0]='sudo', got %q", args[0]) + } + if args[1] != "ovs-vsctl" { + t.Errorf("as non-root, expected args[1]='ovs-vsctl', got %q", args[1]) + } + if len(args) != 4 { + t.Errorf("expected 4 args [sudo ovs-vsctl br-exists br-int], got %d: %v", len(args), args) + } +} diff --git a/spinifex/vm/lifecycle_test.go b/spinifex/vm/lifecycle_test.go index 33e36eb7f..e84a4dbfa 100644 --- a/spinifex/vm/lifecycle_test.go +++ b/spinifex/vm/lifecycle_test.go @@ -193,6 +193,15 @@ func TestGenerateDevMAC_Stable(t *testing.T) { assert.NotEqual(t, GenerateDevMAC("i-abc123"), GenerateDevMAC("i-def456")) } +func TestGenerateMgmtMAC_Stable(t *testing.T) { + a := GenerateMgmtMAC("i-abc123") + b := GenerateMgmtMAC("i-abc123") + assert.Equal(t, a, b, "MAC must be deterministic for the same instance ID") + assert.NotEqual(t, GenerateMgmtMAC("i-abc123"), GenerateMgmtMAC("i-def456")) + // Dev and mgmt MACs for the same instance must differ — class separation. + assert.NotEqual(t, GenerateDevMAC("i-abc123"), GenerateMgmtMAC("i-abc123")) +} + func TestStartReturnsErrorWhenInstanceUnknown(t *testing.T) { m := NewManager() err := m.Start("i-missing") diff --git a/spinifex/vm/network_helpers.go b/spinifex/vm/network_helpers.go index a669ba892..42ca06a08 100644 --- a/spinifex/vm/network_helpers.go +++ b/spinifex/vm/network_helpers.go @@ -5,6 +5,7 @@ import ( "log/slog" "strings" + "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/utils" ) @@ -43,7 +44,7 @@ func MgmtTapName(instanceID string) string { // OVSIfaceID returns the OVS external_ids:iface-id value for an ENI. // This must match the OVN LogicalSwitchPort name for ovn-controller binding. func OVSIfaceID(eniID string) string { - return "port-" + eniID + return topology.Port(eniID) } // VPCTapSpec returns the TapSpec for a VPC ENI's tap on br-int. The @@ -66,6 +67,13 @@ func GenerateDevMAC(instanceID string) string { return utils.HashMAC("dev:" + instanceID) } +// GenerateMgmtMAC returns the locally-administered unicast MAC for the +// management NIC. The "mgmt:" tag disambiguates from the dev NIC of the +// same instance (which shares instanceID). +func GenerateMgmtMAC(instanceID string) string { + return utils.HashMAC("mgmt:" + instanceID) +} + // setupExtraENINICs creates tap devices on br-int and appends matching QEMU // virtio-net device entries to instance.Config for each additional ENI a // system VM spans. The primary ENI is handled separately by the launch diff --git a/spinifex/vm/network_plumber.go b/spinifex/vm/network_plumber.go index d52ff878b..455c89ae7 100644 --- a/spinifex/vm/network_plumber.go +++ b/spinifex/vm/network_plumber.go @@ -1,10 +1,10 @@ package vm // NetworkPlumber handles tap device and OVS bridge operations. Defined here -// so the manager can hold the collaborator without importing the daemon -// package; the daemon's OVSNetworkPlumber type satisfies it structurally. -// Both methods must be idempotent — callers rely on safe reinvocation across -// host reboots and terminate-during-pending races. +// so the manager can hold the collaborator without importing network/host; +// host.OVSPlumber satisfies it structurally. Both methods must be idempotent — +// callers rely on safe reinvocation across host reboots and terminate-during- +// pending races. type NetworkPlumber interface { SetupTap(spec TapSpec) error CleanupTap(name string) error diff --git a/spinifex/services/vpcd/vpcd.go b/spinifex/vpcd/vpcd.go similarity index 56% rename from spinifex/services/vpcd/vpcd.go rename to spinifex/vpcd/vpcd.go index f18b5dc90..1ec249742 100644 --- a/spinifex/services/vpcd/vpcd.go +++ b/spinifex/vpcd/vpcd.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "log/slog" + "net" + "net/netip" "os" "os/exec" "os/signal" @@ -13,11 +15,62 @@ import ( "time" "github.com/mulgadc/spinifex/spinifex/admin" - "github.com/mulgadc/spinifex/spinifex/services/vpcd/dhcp" + "github.com/mulgadc/spinifex/spinifex/network/external" + "github.com/mulgadc/spinifex/spinifex/network/host" + "github.com/mulgadc/spinifex/spinifex/network/ovn" + "github.com/mulgadc/spinifex/spinifex/network/policy" + "github.com/mulgadc/spinifex/spinifex/network/reconcile" + "github.com/mulgadc/spinifex/spinifex/network/subscribers" + "github.com/mulgadc/spinifex/spinifex/network/topology" "github.com/mulgadc/spinifex/spinifex/utils" - "github.com/nats-io/nats.go" ) +// Bridge mode constants for external connectivity. Bridge mode selects how +// the WAN NIC reaches the OVS integration bridge: +// - Direct: WAN NIC is added directly to br-external as an OVS port. +// Enables distributed NAT. Only safe when the WAN NIC is NOT the SSH/ +// management NIC. +// - Veth: a veth pair links a Linux bridge (br-wan) to the OVS bridge +// (br-ext). Requires centralized NAT — the Linux bridge intermediary +// breaks distributed NAT hairpin routing. +const ( + BridgeModeDirect = "direct" + BridgeModeVeth = "veth" + // OvnExternalBridge is the OVS bridge that ovn-bridge-mappings targets + // for the "external" localnet. Owned by setup-ovn.sh and independent of + // the WAN bridge (which is Linux-side in veth mode). + OvnExternalBridge = "br-ext" +) + +// waitForFlowsHV shells out to `ovn-nbctl --wait=hv sync`, bumping +// NB_Global.nb_cfg and blocking until every connected chassis has acknowledged +// the new sequence — i.e. ovn-northd has compiled NB→SB and ovn-controller +// has installed the resulting flows. Used as the NAT manager's flows barrier +// so newly-launched VMs aren't unreachable while their gateway chassis is +// still catching up. Bounded by ovn-nbctl --timeout=30; on overrun we log +// a Warn and return nil so the caller continues. +// +// Declared as a var so tests can stub it. +var waitForFlowsHV = func() error { + start := time.Now() + cmd := sudoCommand("ovn-nbctl", + "--no-leader-only", + "--timeout=30", + "--wait=hv", + "sync", + ) + if out, err := cmd.CombinedOutput(); err != nil { + slog.Warn("vpcd: OVN flows-ready barrier overran; continuing without confirmation", + "elapsed", time.Since(start), + "err", err, + "output", strings.TrimSpace(string(out)), + ) + return nil + } + slog.Debug("vpcd: OVN flows-ready barrier complete", "elapsed", time.Since(start)) + return nil +} + // sudoCommand wraps exec.Command with sudo when running as non-root. // OVS/OVN commands require elevated privileges; when running as root // (Docker, production) no wrapper is needed. @@ -64,25 +117,22 @@ type Config struct { ExternalPools []ExternalPoolConfig // Bootstrap holds the default VPC config from spinifex.toml for first-boot reconciliation. Bootstrap *BootstrapVPC - // ExternalInterface is the WAN NIC name (e.g., "enp0s3"). Used to align - // the macvlan MAC with the OVN gateway MAC for inbound traffic. + // ExternalInterface is the WAN NIC name (e.g., "enp0s3"). Used by + // detectBridgeMode for veth/direct auto-detection. ExternalInterface string - // DhcpBindBridge is the bridge where the DHCP client binds its AF_PACKET - // socket. In veth mode this is the Linux bridge that holds the WAN NIC - // (e.g. "br-wan"); in direct mode this is the OVS bridge holding the - // WAN NIC. Never the OVN-side "br-ext" — that never sees LAN DHCP. - DhcpBindBridge string - // BridgeMode is "direct", "macvlan", or "veth". Direct bridge adds the WAN - // NIC directly to the OVS bridge; macvlan creates a sub-interface; veth uses - // a veth pair to link a Linux bridge to OVS. When empty, auto-detected at - // startup. + // BridgeMode is "direct" or "veth". Direct bridge adds the WAN NIC + // directly to the OVS bridge; veth uses a veth pair to link a Linux bridge + // to OVS. When empty, auto-detected at startup. BridgeMode string + // AZ is the local availability zone identifier. The reconciler uses it + // to scope its IntentState scan to local-AZ KV records; new VPC records + // are stamped with this value at create time. + AZ string } // ExternalPoolConfig mirrors config.ExternalPool for vpcd's internal use. type ExternalPoolConfig struct { Name string - Source string // "static" (default) or "dhcp" RangeStart string RangeEnd string Gateway string @@ -91,16 +141,10 @@ type ExternalPoolConfig struct { DNSServers []string Region string AZ string - DhcpBindBridge string // Bridge where the DHCP AF_PACKET socket binds (e.g. "br-wan"). Required for source="dhcp". - GwLrpRangeStart string // Sub-range for OVN gateway LRP IPs in centralized NAT mode (mulga-siv-36). + GwLrpRangeStart string // Sub-range for OVN gateway LRP IPs in centralized NAT mode. GwLrpRangeEnd string } -// IsDHCP returns true if this pool obtains IPs from upstream DHCP. -func (p *ExternalPoolConfig) IsDHCP() bool { - return p.Source == "dhcp" -} - // Service implements the Spinifex service interface for vpcd. type Service struct { Config *Config @@ -287,6 +331,92 @@ func preflightOVN() error { return nil } +// externalCIDRFromBridge returns the first IPv4 CIDR assigned to the named +// bridge interface. Used to discover the host's uplink network at startup +// (the OS assigns this via netplan static config or systemd-networkd DHCP +// before Spinifex starts). +// +// Injected as a var so tests can stub it without requiring a real interface. +var externalCIDRFromBridge = func(bridge string) (netip.Prefix, error) { + iface, err := net.InterfaceByName(bridge) + if err != nil { + return netip.Prefix{}, fmt.Errorf("interface %q: %w", bridge, err) + } + addrs, err := iface.Addrs() + if err != nil { + return netip.Prefix{}, fmt.Errorf("addrs %q: %w", bridge, err) + } + for _, a := range addrs { + ipnet, ok := a.(*net.IPNet) + if !ok { + continue + } + v4 := ipnet.IP.To4() + if v4 == nil { + continue + } + ones, _ := ipnet.Mask.Size() + addr, _ := netip.AddrFromSlice(v4) + return netip.PrefixFrom(addr, ones), nil + } + return netip.Prefix{}, fmt.Errorf("no IPv4 address on %q", bridge) +} + +// resolveExternalCIDR blocks until the WAN bridge has an IPv4 address or the +// timeout elapses. This guards the boot race where vpcd starts before +// systemd-networkd finishes DHCP on br-wan, or before netplan applies a +// static config. Returns the resolved CIDR for downstream consumers. +// +// Forward-compatible with the ADR-0006 L0 contract: this becomes the backing +// implementation of HostWiring.ExternalCIDR() once the network/host package +// lands. Until then the value is only used to fail-start on missing uplink. +func resolveExternalCIDR(ctx context.Context, bridge string, timeout time.Duration) (netip.Prefix, error) { + const retryDelay = 500 * time.Millisecond + deadline := time.Now().Add(timeout) + attempt := 0 + for { + attempt++ + cidr, err := externalCIDRFromBridge(bridge) + if err == nil { + if attempt > 1 { + slog.Info("vpcd: external CIDR resolved", "bridge", bridge, "cidr", cidr.String(), "attempts", attempt) + } + return cidr, nil + } + if time.Now().After(deadline) { + return netip.Prefix{}, fmt.Errorf("external CIDR not resolved on %q after %s (%d attempts): %w", + bridge, timeout, attempt, err) + } + slog.Warn("vpcd: external CIDR not yet assigned, retrying", + "bridge", bridge, "err", err, "attempt", attempt, "retry_in", retryDelay) + select { + case <-ctx.Done(): + return netip.Prefix{}, fmt.Errorf("external CIDR resolution cancelled: %w", ctx.Err()) + case <-time.After(retryDelay): + } + } +} + +// ensureExternalCIDRReady blocks at startup until the WAN bridge has an IPv4 +// address, or returns an error if resolution fails within the timeout. The OS +// (netplan static or systemd-networkd DHCP) assigns the uplink address before +// Spinifex starts in steady-state; a missing address at this point means a +// boot race or misconfiguration. Bounded retry so systemd's +// Restart=on-failure does not flap through transient gaps. No-op when +// externalMode is empty (overlay-only deployments). +func ensureExternalCIDRReady(ctx context.Context, externalMode, bridge string) error { + if externalMode == "" { + return nil + } + cidr, err := resolveExternalCIDR(ctx, bridge, 30*time.Second) + if err != nil { + slog.Error("vpcd: external CIDR resolution failed", "bridge", bridge, "err", err) + return err + } + slog.Info("vpcd: external CIDR resolved at startup", "bridge", bridge, "cidr", cidr.String()) + return nil +} + func launchService(cfg *Config) error { slog.Info("Starting vpcd service", "ovn_nb_addr", cfg.OVNNBAddr, @@ -313,7 +443,7 @@ func launchService(cfg *Config) error { return fmt.Errorf("OVN NB DB address not configured (ovn_nb_addr is empty)") } - liveClient := NewLiveOVNClient(cfg.OVNNBAddr) + liveClient := ovn.NewLiveClient(cfg.OVNNBAddr) ctx := context.Background() if err := liveClient.Connect(ctx); err != nil { slog.Error("Failed to connect to OVN NB DB", "endpoint", cfg.OVNNBAddr, "err", err) @@ -322,19 +452,20 @@ func launchService(cfg *Config) error { defer liveClient.Close() slog.Info("Connected to OVN NB DB", "endpoint", cfg.OVNNBAddr) - bridgeMode, dhcpBindBridge := resolveBridgeConfig(cfg.BridgeMode, cfg.ExternalInterface, cfg.DhcpBindBridge) - slog.Info("External bridge mode", "mode", bridgeMode, "dhcp_bind_bridge", dhcpBindBridge) - if err := verifyBridgeMode(bridgeMode, cfg.ExternalInterface, dhcpBindBridge); err != nil { + bridgeMode, wanBridge := resolveBridgeConfig(cfg.BridgeMode, cfg.ExternalInterface) + slog.Info("External bridge mode", "mode", bridgeMode, "wan_bridge", wanBridge) + if err := verifyBridgeMode(bridgeMode, cfg.ExternalInterface, wanBridge); err != nil { slog.Error("vpcd: bridge mode sanity check failed", "err", err) return err } - // Reconcile OVN topology from bootstrap config before subscribing. - // This ensures the default VPC topology exists even if admin init ran - // before services were started (first-install case). - var topoOpts []TopologyOption + // Resolve external CIDR from the WAN bridge before any reconcile. Skipped + // when external networking is disabled (overlay-only deployments). + if err := ensureExternalCIDRReady(ctx, cfg.ExternalMode, wanBridge); err != nil { + return err + } + if cfg.ExternalMode != "" { - topoOpts = append(topoOpts, WithExternalNetwork(cfg.ExternalMode, cfg.ExternalPools)) slog.Info("External network enabled", "mode", cfg.ExternalMode, "pools", len(cfg.ExternalPools)) } // Discover chassis from OVN SBDB. The OVS-managed system-id (persisted at @@ -349,37 +480,78 @@ func launchService(cfg *Config) error { if len(chassisNames) == 0 { return fmt.Errorf("vpcd: no OVN chassis registered in SBDB — is ovn-controller running and connected?") } - topoOpts = append(topoOpts, WithChassisNames(chassisNames)) slog.Info("vpcd: gateway chassis discovered", "chassis", chassisNames) - topoOpts = append(topoOpts, WithBridgeMode(bridgeMode)) - topoOpts = append(topoOpts, WithNATSConn(nc)) - topo := NewTopologyHandler(liveClient, topoOpts...) - // Elect a single vpcd to run startup reconcile. Without this, N vpcds in a - // multi-node cluster all hit Get-then-Create on Logical_Router with no + uplinkMode := host.UplinkModePhysical + if bridgeMode == BridgeModeVeth { + uplinkMode = host.UplinkModeVeth + } + natMode := policy.NATModeFromUplinkMode(uplinkMode) + + // Construct the network manager stack. The reconciler is the single + // intent-driven pass that runs once at startup (leader-gated) and then + // on a 5-minute drift ticker; together they replace the five separate + // passes the old vpcd ran on boot. + var topoOpts []topology.Option + if dns := pickDNSServer(cfg.ExternalPools); dns != "" { + topoOpts = append(topoOpts, topology.WithDNSServer(func() string { return dns })) + } + topoMgr := topology.NewLiveManager(liveClient, topoOpts...) + + sgMgr := policy.NewSecurityGroupManager(liveClient) + natMgr, err := policy.NewNATManager(liveClient, natMode, policy.WithFlowsBarrier(waitForFlowsHV)) + if err != nil { + return fmt.Errorf("construct NAT manager: %w", err) + } + routeMgr := policy.NewRouteManager(liveClient) + + var igwPool *external.ExternalPoolConfig + if cfg.ExternalMode != "" && len(cfg.ExternalPools) > 0 { + p := externalPoolConfigToShared(cfg.ExternalPools[0]) + igwPool = &p + } + igwMgr, err := external.NewIGWManager(external.IGWManagerConfig{ + OVN: liveClient, + Routes: routeMgr, + NAT: natMgr, + Pool: igwPool, + Allocator: external.NewStaticRangeAllocator(liveClient), + Chassis: chassisNames, + NATMode: natMode, + FlowsBarrier: waitForFlowsHV, + }) + if err != nil { + return fmt.Errorf("construct IGW manager: %w", err) + } + eipMgr, err := external.NewEIPManager(natMgr, waitForFlowsHV) + if err != nil { + return fmt.Errorf("construct EIP manager: %w", err) + } + natgwMgr, err := external.NewNATGWManager(natMgr) + if err != nil { + return fmt.Errorf("construct NATGW manager: %w", err) + } + + // Elect a single vpcd to run startup reconcile. Without this, N vpcds in + // a multi-node cluster all hit Get-then-Create on Logical_Router with no // atomicity, producing duplicate rows that ovn-nbctl rejects with - // "Multiple logical routers named '...'. Use a UUID." (mulga-siv-29). + // "Multiple logical routers named '...'. Use a UUID." // Runtime VPC events still fan out via the vpcd-workers queue group, so // non-leaders remain functional after Subscribe below. holder, _ := os.Hostname() - releaseLeader, isLeader := AcquireReconcileLeader(nc, holder) + releaseLeader, isLeader := reconcile.AcquireLeader(nc, holder) - // Macvlan mode only: align macvlan MAC with the OVN gateway router MAC. - // The macvlan only delivers inbound unicast matching its own MAC. With - // centralized NAT, OVN uses the router MAC for all external ARP replies. - // Setting the macvlan MAC to the router MAC ensures inbound traffic reaches OVS. - // Direct bridge mode doesn't need this — OVS sees all traffic on the wire. - if bridgeMode == BridgeModeMacvlan && cfg.Bootstrap != nil && cfg.Bootstrap.VpcId != "" && cfg.ExternalInterface != "" { - gwMAC := generateMAC("gw-" + cfg.Bootstrap.VpcId) - macvlanName := "spx-ext-" + cfg.ExternalInterface - if err := setMacvlanMAC(macvlanName, gwMAC); err != nil { - slog.Warn("vpcd: failed to set macvlan MAC (inbound traffic may not work)", "iface", macvlanName, "mac", gwMAC, "err", err) - } else { - slog.Info("vpcd: macvlan MAC aligned with gateway", "iface", macvlanName, "mac", gwMAC) - } + subscriber, err := subscribers.New(subscribers.Config{ + Topology: topoMgr, + SG: sgMgr, + EIP: eipMgr, + NATGW: natgwMgr, + IGW: igwMgr, + }) + if err != nil { + return fmt.Errorf("construct subscriber: %w", err) } - - subs, err := topo.Subscribe(nc) + subs, err := subscriber.Subscribe(nc) if err != nil { slog.Error("Failed to subscribe to VPC topics", "err", err) return err @@ -390,100 +562,64 @@ func launchService(cfg *Config) error { } }() - // DHCP manager: services vpc.dhcp.acquire/release requests from the - // daemon-side ExternalIPAM handlers and (mulga-siv-38) topology's gw-LRP - // allocator. MUST subscribe before any reconcile pass — bootstrap - // reconcile, ReconcileFromKV, and retrofit all call into reconcileIGW / - // expectedGatewayPortNetwork which fan out vpc.dhcp.acquire on - // source="dhcp" pools. Without a live subscription those requests hit - // "no responders" and burn the 60-attempt cold-boot retry budget per - // existing IGW. Started unconditionally — pools with source="static" - // never issue acquire requests, so the Manager sits idle until - // something actually wants DHCP. js, err := nc.JetStream() if err != nil { - slog.Error("Failed to get JetStream context for DHCP manager", "err", err) + slog.Error("Failed to get JetStream context", "err", err) return err } - // timeout/retry args are legacy no-ops (mulga-siv-39): DHCPManager - // owns DORA retransmission via acquireWithBackoff. - // - // Retry on JetStream-not-ready: NewDHCPManager creates the DHCP-leases - // KV bucket, which fails immediately if the JetStream cluster has not - // reached quorum. On multi-node bring-up vpcd's launchService runs - // before all peers are reachable, so we mirror the daemon's - // initJetStream backoff (500ms→10s, 5 min cap) instead of crashing the - // service and aborting the rest of the node bring-up. - dhcpManager, err := newDHCPManagerWithRetry(ctx, nc, js) - if err != nil { - slog.Error("Failed to create DHCP manager", "err", err) - return err - } - defer dhcpManager.Close() - if err := dhcpManager.Bootstrap(ctx); err != nil { - slog.Error("DHCP manager bootstrap failed", "err", err) - return err - } - dhcpSubs, err := dhcpManager.Subscribe(nc) + rec, err := reconcile.New(reconcile.Config{ + OVN: liveClient, + SG: sgMgr, + NAT: natMgr, + Routes: routeMgr, + IGW: igwMgr, + Topology: topoMgr, + LocalAZ: cfg.AZ, + NodeHostname: holder, + Chassis: chassisNames, + }) if err != nil { - slog.Error("DHCP manager subscribe failed", "err", err) - return err + return fmt.Errorf("construct reconciler: %w", err) } - defer func() { - for _, s := range dhcpSubs { - _ = s.Unsubscribe() + + // Startup reconcile: leader-gated read of NATS KV intent state, applied + // once against OVN NB DB. The drift loop below handles the recovery case + // (KV not yet populated when this fires — daemon's EnsureDefaultVPC may + // race with vpcd's startup). Per-event NATS subscribers route through + // the same network/* managers, so the startup pass and the runtime fast + // path share one implementation. + // + // ReconcileApplyOnly (not Reconcile) — orphan pruning is unsafe at + // startup because intent may be stale: daemon EnsureDefaultVPC on a + // peer node can be mid-flight (KV write committed but not yet visible + // here) while a peer subscriber concurrently creates port groups from + // its vpc.create-sg event. A startup prune then sweeps those port + // groups as orphans, breaking every subsequent RunInstances until the + // 5-minute drift tick recreates them. Drift uses full Reconcile. + if isLeader { + intent, intentErr := reconcile.LoadIntentFromKV(ctx, js, cfg.AZ) + if intentErr != nil { + slog.Warn("vpcd: startup intent load failed", "err", intentErr) + } else if err := rec.ReconcileApplyOnly(ctx, intent); err != nil { + slog.Warn("vpcd: startup reconcile failed", "err", err) } - }() - - // Pass 1: Bootstrap-config reconcile. Creates the bootstrap VPC / IGW - // topology if missing. Leader-only. - if isLeader { - Reconcile(ctx, topo, cfg.Bootstrap) - } - - // Pass 2: Reconcile from NATS KV (handles reboots, OVN DB loss, missed events). - // Runs after subscribing so new events are not missed during reconciliation. - // Leader-gated (mulga-siv-29) to avoid concurrent Create races against OVN NB. - // - // ReconcileFromKV creates ENI LSPs with the non-atomic primitive (no SG - // port-group joins). Calling ReconcileSGsOnce immediately after — under the - // same leader lock — recreates any missing port groups and joins the - // fresh LSPs to them, closing the enforcement gap before the periodic - // loop's first 30s tick. - if isLeader { - ReconcileFromKV(ctx, nc, topo, chassisNames) - ReconcileSGsOnce(ctx, nc, topo) - } - - // Pass 3: Retrofit localnet options on every external switch. Walks OVN - // directly so stale/missing KV records can't hide a stale nat-addresses - // or cleared network_name. Idempotent; silent when all correct. - // Pass 4: Retrofit gateway-port Networks. Reconcile gates IGW work on - // "ext switch missing", so a cluster shipped with the pool-IP bug - // (mulga-siv-26) never re-enters reconcileIGW. Walk every gateway LRP - // directly and rewrite stale Networks to link-local in place. - // Both leader-gated (mulga-siv-29) — these read-modify-write OVN NB and - // would race against the leader's Reconcile/ReconcileFromKV otherwise. - if isLeader { - topo.RetrofitAllExternalLocalnetOptions(ctx) - topo.RetrofitAllGatewayPortNetworks(ctx) releaseLeader() } - // Periodic SG/port-group/address-set drift convergence. Leader-gated on the - // shared reconcile bucket so only one vpcd in the cluster runs the scans at - // a time; cancelled when the parent ctx is. + // Periodic drift detection. Leader-gated on the shared reconcile bucket + // so only one vpcd in the cluster scans at a time; cancelled when the + // parent ctx is. loopCtx, loopCancel := context.WithCancel(ctx) defer loopCancel() loopDone := make(chan struct{}) go func() { - ReconcileSGsLoop(loopCtx, nc, topo) + reconcile.DriftLoop(loopCtx, rec, nc, cfg.AZ, holder) close(loopDone) }() slog.Info("vpcd service started, waiting for VPC lifecycle events", - "subscriptions", len(subs), "dhcp_subscriptions", len(dhcpSubs)) + "subscriptions", len(subs)) // Wait for shutdown signal sigChan := make(chan os.Signal, 1) @@ -496,70 +632,50 @@ func launchService(cfg *Config) error { return nil } -// newDHCPManagerWithRetry constructs a DHCPManager, retrying on transient -// JetStream-not-ready errors with exponential backoff (500ms→10s, capped -// at 5 min). NewDHCPManager creates the spinifex-dhcp-leases KV bucket; -// on multi-node bring-up the JetStream cluster may not yet have quorum -// when vpcd starts, and a single-shot CreateKeyValue would otherwise crash -// the service before the rest of the node finishes coming up. -// -// Mirrors daemon.initJetStream's backoff so vpcd waits as patiently as the -// daemon does for the cluster to form. -func newDHCPManagerWithRetry(ctx context.Context, nc *nats.Conn, js nats.JetStreamContext) (*DHCPManager, error) { - const maxWait = 5 * time.Minute - retryDelay := 500 * time.Millisecond - start := time.Now() - attempt := 0 - - for { - attempt++ - mgr, err := NewDHCPManager(nc, js, dhcp.NewNClient4(0, 0)) - if err == nil { - if attempt > 1 { - slog.Info("vpcd: DHCP manager ready", "attempts", attempt, "elapsed", time.Since(start).Round(time.Second)) - } - return mgr, nil +// pickDNSServer returns the OVN dhcp_options dns_server string ("{a, b}") +// from the first unscoped (no region, no AZ) pool with DNS servers set. +// Empty string falls back to topology.NewLiveManager's default +// ("{8.8.8.8, 1.1.1.1}"). +func pickDNSServer(pools []ExternalPoolConfig) string { + for _, p := range pools { + if p.Region == "" && p.AZ == "" && len(p.DNSServers) > 0 { + return "{" + strings.Join(p.DNSServers, ", ") + "}" } + } + return "" +} - elapsed := time.Since(start) - if elapsed >= maxWait { - return nil, fmt.Errorf("DHCP manager init timed out after %s (%d attempts): %w", elapsed.Round(time.Second), attempt, err) - } - - slog.Warn("vpcd: DHCP manager not ready (waiting for JetStream cluster quorum)", - "err", err, "attempt", attempt, "elapsed", elapsed.Round(time.Second), "retryIn", retryDelay) - - select { - case <-ctx.Done(): - return nil, fmt.Errorf("DHCP manager init cancelled: %w", ctx.Err()) - case <-time.After(retryDelay): - } - retryDelay = min(retryDelay*2, 10*time.Second) +// externalPoolConfigToShared translates the vpcd-local ExternalPoolConfig +// into the network/external package's shared shape. The two will merge +// once topology.go is split (see external.go's type doc); until then +// this is the seam. +func externalPoolConfigToShared(p ExternalPoolConfig) external.ExternalPoolConfig { + return external.ExternalPoolConfig{ + Name: p.Name, + RangeStart: p.RangeStart, + RangeEnd: p.RangeEnd, + Gateway: p.Gateway, + GatewayIP: p.GatewayIP, + PrefixLen: p.PrefixLen, + DNSServers: p.DNSServers, + Region: p.Region, + AZ: p.AZ, + GwLrpRangeStart: p.GwLrpRangeStart, + GwLrpRangeEnd: p.GwLrpRangeEnd, } } -// resolveBridgeConfig picks the bridge mode and DHCP-bind-bridge to use, +// resolveBridgeConfig picks the bridge mode and WAN bridge name to use, // auto-detecting mode when unset. Empty mode stays empty — verifyBridgeMode -// rejects it with a list of supported values (D12). Empty bind bridge -// defaults to "br-wan", the consumer-router convention. -func resolveBridgeConfig(cfgBridgeMode, externalIface, cfgDhcpBindBridge string) (string, string) { +// rejects it with a list of supported values (D12). The WAN bridge name is +// always "br-wan" (the consumer-router convention) since vpcd no longer +// accepts a per-node override. +func resolveBridgeConfig(cfgBridgeMode, externalIface string) (string, string) { bridgeMode := cfgBridgeMode if bridgeMode == "" && externalIface != "" { bridgeMode = detectBridgeMode(externalIface) } - dhcpBindBridge := cfgDhcpBindBridge - if dhcpBindBridge == "" { - dhcpBindBridge = "br-wan" - } - return bridgeMode, dhcpBindBridge -} - -// ifaceIsMacvlan returns true when the given interface exists and is a -// macvlan sub-interface. Injected as a var so detectBridgeMode tests can stub -// it without launching ip(8). -var ifaceIsMacvlan = func(name string) bool { - out, err := exec.Command("ip", "-d", "link", "show", name).CombinedOutput() - return err == nil && strings.Contains(string(out), "macvlan") + return bridgeMode, "br-wan" } // ifaceExists returns true when the kernel reports the named link. @@ -568,7 +684,6 @@ var ifaceExists = func(name string) bool { } // detectBridgeMode checks how the WAN bridge is wired: -// - macvlan: spx-ext-{iface} macvlan sub-interface exists // - veth: veth-wan-ovs interface exists (Linux bridge linked to OVS via veth pair) // - direct: physical NIC is added directly to the OVS bridge // @@ -577,17 +692,12 @@ var ifaceExists = func(name string) bool { // silent Debug fall-through is what let the veth-persistence bug hide for // weeks (mulga-998.b Fix 2). func detectBridgeMode(externalIface string) string { - macvlanName := "spx-ext-" + externalIface - if ifaceIsMacvlan(macvlanName) { - slog.Info("vpcd: detected macvlan interface on WAN bridge", "iface", macvlanName, "mode", BridgeModeMacvlan) - return BridgeModeMacvlan - } if ifaceExists("veth-wan-ovs") { slog.Info("vpcd: detected veth pair linking Linux bridge to OVS", "mode", BridgeModeVeth) return BridgeModeVeth } - slog.Warn("vpcd: no macvlan or veth interface found, assuming direct bridge mode", - "external_iface", externalIface, "checked_macvlan", macvlanName, "checked_veth", "veth-wan-ovs", + slog.Warn("vpcd: no veth interface found, assuming direct bridge mode", + "external_iface", externalIface, "checked_veth", "veth-wan-ovs", "mode", BridgeModeDirect) return BridgeModeDirect } @@ -624,37 +734,36 @@ var readLinkMaster = func(iface string) (string, error) { // verifyBridgeMode is the post-detect sanity check. It refuses to start vpcd // when the chosen bridge mode does not match the host plumbing (D4+D18): // -// - direct: ExternalInterface must be an OVS port on DhcpBindBridge. That is -// the whole contract of direct mode. +// - direct: ExternalInterface must be an OVS port on the WAN bridge. That +// is the whole contract of direct mode. // - veth: (a) veth-wan-ovs must be an OVS port on OvnExternalBridge — the // OVN side, owned by setup-ovn.sh's ovn-bridge-mappings. (b) veth-wan-br -// must be enslaved to DhcpBindBridge — the Linux side, where the DHCP -// client sees LAN frames. +// must be enslaved to the WAN bridge — the Linux side. // - empty / unknown: fail with the list of supported values. // // Fail-start, not soft-degrade — the distributed-NAT-on-veth-host footgun is // exactly what this plan set out to kill. -func verifyBridgeMode(mode, externalIface, dhcpBindBridge string) error { +func verifyBridgeMode(mode, externalIface, wanBridge string) error { switch mode { case BridgeModeDirect: if externalIface == "" { return fmt.Errorf("vpcd: direct bridge mode requires external_interface (the WAN NIC name)") } - if dhcpBindBridge == "" { - return fmt.Errorf("vpcd: direct bridge mode requires dhcp_bind_bridge (the OVS bridge holding the WAN NIC)") + if wanBridge == "" { + return fmt.Errorf("vpcd: direct bridge mode requires the WAN bridge (the OVS bridge holding the WAN NIC)") } br, err := portToBr(externalIface) if err != nil { return fmt.Errorf("vpcd: direct bridge mode: %w", err) } - if br != dhcpBindBridge { - return fmt.Errorf("vpcd: direct bridge mode: %q is on OVS bridge %q, expected %q (dhcp_bind_bridge)", - externalIface, br, dhcpBindBridge) + if br != wanBridge { + return fmt.Errorf("vpcd: direct bridge mode: %q is on OVS bridge %q, expected %q", + externalIface, br, wanBridge) } return nil case BridgeModeVeth: - if dhcpBindBridge == "" { - return fmt.Errorf("vpcd: veth bridge mode requires dhcp_bind_bridge (the Linux bridge holding the WAN NIC)") + if wanBridge == "" { + return fmt.Errorf("vpcd: veth bridge mode requires wan_bridge (the Linux bridge holding the WAN NIC)") } br, err := portToBr("veth-wan-ovs") if err != nil { @@ -668,9 +777,9 @@ func verifyBridgeMode(mode, externalIface, dhcpBindBridge string) error { if err != nil { return fmt.Errorf("vpcd: veth bridge mode: veth-wan-br missing or has no master — is spinifex-veth-wan.service running? %w", err) } - if master != dhcpBindBridge { - return fmt.Errorf("vpcd: veth bridge mode: veth-wan-br master is %q, expected %q (dhcp_bind_bridge)", - master, dhcpBindBridge) + if master != wanBridge { + return fmt.Errorf("vpcd: veth bridge mode: veth-wan-br master is %q, expected %q (wan_bridge)", + master, wanBridge) } return nil default: @@ -678,19 +787,3 @@ func verifyBridgeMode(mode, externalIface, dhcpBindBridge string) error { mode, BridgeModeDirect, BridgeModeVeth) } } - -// setMacvlanMAC sets the MAC address on a macvlan interface. The interface is -// brought down, MAC changed, and brought back up. Requires sudo/NET_ADMIN. -func setMacvlanMAC(iface, mac string) error { - cmds := [][]string{ - {"sudo", "ip", "link", "set", iface, "down"}, - {"sudo", "ip", "link", "set", iface, "address", mac}, - {"sudo", "ip", "link", "set", iface, "up"}, - } - for _, args := range cmds { - if out, err := exec.Command(args[0], args[1:]...).CombinedOutput(); err != nil { - return fmt.Errorf("%s: %w (%s)", args[3], err, string(out)) - } - } - return nil -} diff --git a/spinifex/services/vpcd/vpcd_test.go b/spinifex/vpcd/vpcd_test.go similarity index 66% rename from spinifex/services/vpcd/vpcd_test.go rename to spinifex/vpcd/vpcd_test.go index 51fe7b329..986209594 100644 --- a/spinifex/services/vpcd/vpcd_test.go +++ b/spinifex/vpcd/vpcd_test.go @@ -1,9 +1,12 @@ package vpcd import ( + "context" "fmt" + "net/netip" "strings" "testing" + "time" ) func TestPreflightOVN_AllPass(t *testing.T) { @@ -230,7 +233,7 @@ func TestVerifyBridgeMode_VethWrongLinuxMaster(t *testing.T) { } func TestVerifyBridgeMode_UnknownModeLists(t *testing.T) { - err := verifyBridgeMode("macvlan", "enp0s3", "br-wan") + err := verifyBridgeMode("bogus", "enp0s3", "br-wan") if err == nil { t.Fatal("expected error for unsupported mode") } @@ -255,15 +258,15 @@ func TestVerifyBridgeMode_DirectMissingExternalIface(t *testing.T) { func TestVerifyBridgeMode_DirectMissingBindBridge(t *testing.T) { err := verifyBridgeMode(BridgeModeDirect, "enp0s3", "") - if err == nil || !strings.Contains(err.Error(), "dhcp_bind_bridge") { - t.Fatalf("expected dhcp_bind_bridge error, got: %v", err) + if err == nil || !strings.Contains(err.Error(), "WAN bridge") { + t.Fatalf("expected WAN bridge error, got: %v", err) } } func TestVerifyBridgeMode_VethMissingBindBridge(t *testing.T) { err := verifyBridgeMode(BridgeModeVeth, "", "") - if err == nil || !strings.Contains(err.Error(), "dhcp_bind_bridge") { - t.Fatalf("expected dhcp_bind_bridge error, got: %v", err) + if err == nil || !strings.Contains(err.Error(), "wan_bridge") { + t.Fatalf("expected wan_bridge error, got: %v", err) } } @@ -279,73 +282,179 @@ func TestVerifyBridgeMode_VethLinuxBrMissing(t *testing.T) { // detectBridgeMode — mulga-998.b Fix 2. -func stubDetectProbes(t *testing.T, macvlans []string, links []string) { +func stubDetectProbes(t *testing.T, links []string) { t.Helper() - origMac := ifaceIsMacvlan origExists := ifaceExists t.Cleanup(func() { - ifaceIsMacvlan = origMac ifaceExists = origExists }) - macSet := map[string]bool{} - for _, m := range macvlans { - macSet[m] = true - } linkSet := map[string]bool{} for _, l := range links { linkSet[l] = true } - ifaceIsMacvlan = func(name string) bool { return macSet[name] } ifaceExists = func(name string) bool { return linkSet[name] } } -func TestDetectBridgeMode_MacvlanWins(t *testing.T) { - stubDetectProbes(t, []string{"spx-ext-enp0s3"}, []string{"veth-wan-ovs"}) - if got := detectBridgeMode("enp0s3"); got != BridgeModeMacvlan { - t.Errorf("want %q, got %q", BridgeModeMacvlan, got) - } -} - -func TestDetectBridgeMode_VethWhenNoMacvlan(t *testing.T) { - stubDetectProbes(t, nil, []string{"veth-wan-ovs"}) +func TestDetectBridgeMode_VethWins(t *testing.T) { + stubDetectProbes(t, []string{"veth-wan-ovs"}) if got := detectBridgeMode("enp0s3"); got != BridgeModeVeth { t.Errorf("want %q, got %q", BridgeModeVeth, got) } } func TestDetectBridgeMode_FallthroughDirect(t *testing.T) { - stubDetectProbes(t, nil, nil) + stubDetectProbes(t, nil) if got := detectBridgeMode("enp0s3"); got != BridgeModeDirect { t.Errorf("want %q, got %q", BridgeModeDirect, got) } } func TestResolveBridgeConfig_UsesExplicitMode(t *testing.T) { - stubDetectProbes(t, nil, nil) - mode, br := resolveBridgeConfig(BridgeModeVeth, "enp0s3", "br-wan") + stubDetectProbes(t, nil) + mode, br := resolveBridgeConfig(BridgeModeVeth, "enp0s3") if mode != BridgeModeVeth || br != "br-wan" { t.Errorf("got (%q,%q), want (%q,br-wan)", mode, br, BridgeModeVeth) } } func TestResolveBridgeConfig_AutoDetects(t *testing.T) { - stubDetectProbes(t, nil, []string{"veth-wan-ovs"}) - mode, _ := resolveBridgeConfig("", "enp0s3", "br-wan") + stubDetectProbes(t, []string{"veth-wan-ovs"}) + mode, _ := resolveBridgeConfig("", "enp0s3") if mode != BridgeModeVeth { t.Errorf("want auto-detect veth, got %q", mode) } } func TestResolveBridgeConfig_EmptyStaysEmptyWithNoIface(t *testing.T) { - mode, _ := resolveBridgeConfig("", "", "") + mode, _ := resolveBridgeConfig("", "") if mode != "" { t.Errorf("empty mode + no iface should stay empty (D12); got %q", mode) } } func TestResolveBridgeConfig_DefaultsBindBridge(t *testing.T) { - _, br := resolveBridgeConfig(BridgeModeDirect, "enp0s3", "") + _, br := resolveBridgeConfig(BridgeModeDirect, "enp0s3") if br != "br-wan" { - t.Errorf("empty dhcp_bind_bridge should default to br-wan, got %q", br) + t.Errorf("WAN bridge should default to br-wan, got %q", br) + } +} + +// stubExternalCIDR replaces externalCIDRFromBridge with a sequence of +// responses. Each call consumes the next entry; the last entry repeats. +func stubExternalCIDR(t *testing.T, responses []externalCIDRResponse) { + t.Helper() + orig := externalCIDRFromBridge + t.Cleanup(func() { externalCIDRFromBridge = orig }) + i := 0 + externalCIDRFromBridge = func(bridge string) (netip.Prefix, error) { + r := responses[i] + if i < len(responses)-1 { + i++ + } + return r.prefix, r.err + } +} + +type externalCIDRResponse struct { + prefix netip.Prefix + err error +} + +func TestResolveExternalCIDR_ImmediateSuccess(t *testing.T) { + want := netip.MustParsePrefix("192.168.1.10/24") + stubExternalCIDR(t, []externalCIDRResponse{{prefix: want}}) + got, err := resolveExternalCIDR(context.Background(), "br-wan", 1*time.Second) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got != want { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestResolveExternalCIDR_RetriesThenSucceeds(t *testing.T) { + want := netip.MustParsePrefix("10.0.0.5/16") + stubExternalCIDR(t, []externalCIDRResponse{ + {err: fmt.Errorf("no IPv4")}, + {err: fmt.Errorf("no IPv4")}, + {prefix: want}, + }) + got, err := resolveExternalCIDR(context.Background(), "br-wan", 5*time.Second) + if err != nil { + t.Fatalf("unexpected error after retry: %v", err) + } + if got != want { + t.Errorf("got %v, want %v", got, want) + } +} + +func TestResolveExternalCIDR_TimeoutFailsStart(t *testing.T) { + stubExternalCIDR(t, []externalCIDRResponse{{err: fmt.Errorf("no IPv4")}}) + _, err := resolveExternalCIDR(context.Background(), "br-wan", 100*time.Millisecond) + if err == nil { + t.Fatal("expected timeout error") + } + if !strings.Contains(err.Error(), "not resolved") { + t.Errorf("expected 'not resolved' error, got: %v", err) + } +} + +func TestResolveExternalCIDR_ContextCancel(t *testing.T) { + stubExternalCIDR(t, []externalCIDRResponse{{err: fmt.Errorf("no IPv4")}}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := resolveExternalCIDR(ctx, "br-wan", 10*time.Second) + if err == nil { + t.Fatal("expected cancellation error") + } + if !strings.Contains(err.Error(), "cancelled") { + t.Errorf("expected 'cancelled' error, got: %v", err) + } +} + +// Exercises the original externalCIDRFromBridge body (not the test stub) by +// resolving against the loopback interface, which is guaranteed to exist and +// carry an IPv4 address on every supported host. +func TestExternalCIDRFromBridge_Loopback(t *testing.T) { + got, err := externalCIDRFromBridge("lo") + if err != nil { + t.Fatalf("loopback resolve: %v", err) + } + if !got.Addr().Is4() { + t.Errorf("expected IPv4 address, got %v", got) + } + if !got.Addr().IsLoopback() { + t.Errorf("expected loopback address, got %v", got.Addr()) + } +} + +func TestExternalCIDRFromBridge_UnknownInterface(t *testing.T) { + _, err := externalCIDRFromBridge("nonexistent-iface-zzz") + if err == nil { + t.Fatal("expected error for unknown interface") + } + if !strings.Contains(err.Error(), "interface") { + t.Errorf("expected 'interface' in error, got: %v", err) + } +} + +func TestEnsureExternalCIDRReady_NoExternalMode(t *testing.T) { + if err := ensureExternalCIDRReady(context.Background(), "", "br-wan"); err != nil { + t.Fatalf("empty externalMode must be a no-op, got: %v", err) + } +} + +func TestEnsureExternalCIDRReady_ResolvesSuccessfully(t *testing.T) { + stubExternalCIDR(t, []externalCIDRResponse{{prefix: netip.MustParsePrefix("10.0.0.1/24")}}) + if err := ensureExternalCIDRReady(context.Background(), "direct", "br-wan"); err != nil { + t.Fatalf("expected success, got: %v", err) + } +} + +func TestEnsureExternalCIDRReady_PropagatesError(t *testing.T) { + stubExternalCIDR(t, []externalCIDRResponse{{err: fmt.Errorf("no IPv4")}}) + err := ensureExternalCIDRReady(context.Background(), "direct", "br-wan") + if err == nil { + t.Fatal("expected error when resolveExternalCIDR fails") } } diff --git a/tests/e2e/TEST_COVERAGE.md b/tests/e2e/TEST_COVERAGE.md index 7be72808d..54e2dc3e9 100644 --- a/tests/e2e/TEST_COVERAGE.md +++ b/tests/e2e/TEST_COVERAGE.md @@ -301,7 +301,7 @@ Gated on `external_mode = "pool"` (skipped when only `dev_networking` is configu - `run-instances` into default subnet → auto-assigned `PublicIpAddress` - `ovn-nbctl lr-nat-list` shows per-instance `dnat_and_snat` rule for `PUB_PRIVATE_IP` - Asserts NO blanket VPC CIDR SNAT (mulga-754 AWS-parity regression) - - SSH via public IP succeeds (when bridge/macvlan reachable) + - SSH via public IP succeeds (when bridge reachable) - Outbound internet from inside the VM (via OVN SNAT) - Step 2b: In-guest TLS trust — cloud-init injects Spinifex CA into the guest, so `curl https://awsgw` and `https://predastore-s3` work without `--insecure` - Step 3: Private subnet isolation diff --git a/tests/e2e/harness/ec2helpers.go b/tests/e2e/harness/ec2helpers.go index a999f3d22..9490c5162 100644 --- a/tests/e2e/harness/ec2helpers.go +++ b/tests/e2e/harness/ec2helpers.go @@ -226,6 +226,40 @@ func AuthorizeSSHIngress(t *testing.T, c *AWSClient, sgID string) { t.Fatalf("authorize-security-group-ingress tcp/22 on %s: %v", sgID, err) } +// AuthorizeICMPIngress idempotently authorizes ICMP (all types) ingress from +// 0.0.0.0/0 on sgID. Same Duplicate-tolerant contract as AuthorizeSSHIngress. +func AuthorizeICMPIngress(t *testing.T, c *AWSClient, sgID string) { + t.Helper() + _, err := c.EC2.AuthorizeSecurityGroupIngress(&ec2.AuthorizeSecurityGroupIngressInput{ + GroupId: aws.String(sgID), + IpPermissions: []*ec2.IpPermission{{ + IpProtocol: aws.String("icmp"), + FromPort: aws.Int64(-1), + ToPort: aws.Int64(-1), + IpRanges: []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}}, + }}, + }) + if err == nil { + return + } + var aerr awserr.Error + if asErr(err, &aerr) && aerr.Code() == "InvalidPermission.Duplicate" { + return + } + t.Fatalf("authorize-security-group-ingress icmp on %s: %v", sgID, err) +} + +// EnsureDefaultSGOpen authorizes tcp/22 + ICMP on the default VPC's default +// SG. AWS default SGs only admit same-SG members; e2e probes come from the +// test runner's external IP, so without this every SSH/ping hits the OVN +// port-group ACL drop. Mirrors the Phase-5 block in run-e2e.sh. +func EnsureDefaultSGOpen(t *testing.T, c *AWSClient) { + t.Helper() + _, sgID, _ := DiscoverDefaultVPC(t, c) + AuthorizeSSHIngress(t, c, sgID) + AuthorizeICMPIngress(t, c, sgID) +} + // asErr is a thin errors.As that lets the helper stay testify-free. func asErr(err error, target *awserr.Error) bool { if a, ok := err.(awserr.Error); ok { diff --git a/tests/e2e/harness/witness.go b/tests/e2e/harness/witness.go index 23e702bfd..9a4474a8b 100644 --- a/tests/e2e/harness/witness.go +++ b/tests/e2e/harness/witness.go @@ -148,11 +148,11 @@ type WitnessVM struct { func LaunchWitnessVM(ctx context.Context, w *Witness, host Node) (*WitnessVM, error) { const maxPlacementAttempts = 3 - // Workaround for mulga-siv-79: daemon silently drops the shortcut SG - // ingress form, so the default SG starts with no :22 ingress and the - // awaitBaseline SSH tunnel to the witness EIP times out. Authorise - // :22 from anywhere via the structured form — once siv-79 lands this - // can drop along with the matching workaround in lb_test. + // Workaround: daemon silently drops the shortcut SG ingress form, so + // the default SG starts with no :22 ingress and the awaitBaseline SSH + // tunnel to the witness EIP times out. Authorise :22 from anywhere + // via the structured form — once the underlying bug lands this can + // drop along with the matching workaround in lb_test. if err := w.ensureDefaultSGSSHIngress(ctx); err != nil { return nil, fmt.Errorf("e2e harness: open witness SSH ingress: %w", err) } @@ -255,11 +255,12 @@ func AssertProgressed(ctx context.Context, t *testing.T, v *WitnessVM) { // --- internals ------------------------------------------------------------ -// ensureDefaultSGSSHIngress authorises tcp/22 from 0.0.0.0/0 on the default -// security group via the structured IpPermissions form. The daemon currently -// drops the top-level shortcut form silently (mulga-siv-79), so without this -// the witness EIP has no inbound :22 and awaitBaseline times out. Idempotent: -// duplicate-rule errors from a prior run are tolerated. +// ensureDefaultSGSSHIngress authorises tcp/22 + ICMP from 0.0.0.0/0 on the +// default security group via the structured IpPermissions form. The daemon +// currently drops the top-level shortcut form silently, so without this the +// witness EIP has no inbound :22 and awaitBaseline times out. ICMP is +// required for downstream gateway-ping probes. +// Idempotent: duplicate-rule errors from a prior run are tolerated. func (w *Witness) ensureDefaultSGSSHIngress(ctx context.Context) error { sgs, err := w.ec2.DescribeSecurityGroupsWithContext(ctx, &ec2.DescribeSecurityGroupsInput{ Filters: []*ec2.Filter{{ @@ -275,17 +276,29 @@ func (w *Witness) ensureDefaultSGSSHIngress(ctx context.Context) error { } groupID := aws.StringValue(sgs.SecurityGroups[0].GroupId) - _, err = w.ec2.AuthorizeSecurityGroupIngressWithContext(ctx, &ec2.AuthorizeSecurityGroupIngressInput{ - GroupId: aws.String(groupID), - IpPermissions: []*ec2.IpPermission{{ + rules := []*ec2.IpPermission{ + { IpProtocol: aws.String("tcp"), FromPort: aws.Int64(22), ToPort: aws.Int64(22), IpRanges: []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}}, - }}, - }) - if err != nil && !strings.Contains(err.Error(), "InvalidPermission.Duplicate") { - return fmt.Errorf("AuthorizeSecurityGroupIngress :22 on %s: %w", groupID, err) + }, + { + IpProtocol: aws.String("icmp"), + FromPort: aws.Int64(-1), + ToPort: aws.Int64(-1), + IpRanges: []*ec2.IpRange{{CidrIp: aws.String("0.0.0.0/0")}}, + }, + } + for _, rule := range rules { + _, err = w.ec2.AuthorizeSecurityGroupIngressWithContext(ctx, &ec2.AuthorizeSecurityGroupIngressInput{ + GroupId: aws.String(groupID), + IpPermissions: []*ec2.IpPermission{rule}, + }) + if err != nil && !strings.Contains(err.Error(), "InvalidPermission.Duplicate") { + return fmt.Errorf("AuthorizeSecurityGroupIngress %s on %s: %w", + aws.StringValue(rule.IpProtocol), groupID, err) + } } return nil } diff --git a/tests/e2e/lib/multinode-helpers.sh b/tests/e2e/lib/multinode-helpers.sh index 9857fad3c..db8da0d37 100755 --- a/tests/e2e/lib/multinode-helpers.sh +++ b/tests/e2e/lib/multinode-helpers.sh @@ -798,6 +798,9 @@ init_leader_node() { # Start init in background — formation server will wait for joins # shellcheck disable=SC2086 + # IPsec disabled: pseudo test runs as unprivileged tf-user and the charon + # trust-store install in `spx admin init` requires writing /etc/ipsec.d/cacerts. + # Loopback-only "nodes" don't exercise real Geneve encryption anyway. ./bin/spx admin init \ --node node1 \ --nodes 3 \ @@ -808,6 +811,7 @@ init_leader_node() { --az ap-southeast-2a \ --spinifex-dir "$HOME/node1/" \ --config-dir "$HOME/node1/config/" \ + --ipsec=false \ $external_flags & LEADER_INIT_PID=$! diff --git a/tests/e2e/multinode/ipsec_test.go b/tests/e2e/multinode/ipsec_test.go new file mode 100644 index 000000000..8300de0c9 --- /dev/null +++ b/tests/e2e/multinode/ipsec_test.go @@ -0,0 +1,114 @@ +//go:build e2e + +package multinode + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/mulgadc/spinifex/tests/e2e/harness" +) + +// runIPSec verifies OVN native IPsec is live on every node. Three checks: +// +// 1. OVS DB on each node carries the cert/key/CA pointers plus +// other_config:ipsec_encapsulation=true. This is what +// daemon.enableOVNIPSec writes; if any node lost it (rolling restart, +// manual ovs-vsctl unset, missing peer cert) the SAs below would +// silently drop to plaintext. +// +// 2. `ip xfrm state` on each node shows at least one AEAD SA negotiated +// in AES-GCM mode. ovs-monitor-ipsec drives strongSwan to bring SAs up +// against every peer ovn-controller has a Geneve tunnel to, so on a +// 3-node mesh we expect 2N SAs per node (in+out per peer) — but the +// test only asserts >=1 to stay tolerant of ovs-monitor-ipsec startup +// ordering quirks. +// +// 3. Best-effort tcpdump for ESP (proto 50) traffic on the underlay. +// Logged only — geneve_sys_6081 may be idle during the capture window +// if no overlay VMs are running yet. +// +// Skips cleanly if the cluster was bootstrapped with --ipsec=false (no +// other_config:ipsec_encapsulation key on the first node). +func runIPSec(t *testing.T, fix *Fixture) { + harness.Phase(t, "Multinode — OVN Native IPsec") + + ssh := harness.NewPeerSSH() + first := fix.Cluster.Nodes[0] + + probeCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + out, err := ssh.Run(probeCtx, + first.Addr, + "sudo ovs-vsctl --if-exists get Open_vSwitch . other_config:ipsec_encapsulation 2>/dev/null || true", + ) + if err != nil { + t.Fatalf("probe ovs-vsctl on %s: %v", first.Name, err) + } + if !strings.Contains(string(out), "true") { + t.Skipf("IPsec not enabled on cluster (%s reports %q): skip", first.Name, strings.TrimSpace(string(out))) + } + + harness.Step(t, "OVS DB carries cert pointers + ipsec_encapsulation=true on every node") + required := []string{"certificate=", "private_key=", "ca_cert=", "ipsec_encapsulation=\"true\""} + for _, n := range fix.Cluster.Nodes { + c, cancel := context.WithTimeout(context.Background(), 10*time.Second) + raw, err := ssh.Run(c, n.Addr, "sudo ovs-vsctl get Open_vSwitch . other_config") + cancel() + if err != nil { + t.Fatalf("%s ovs-vsctl get other_config: %v", n.Name, err) + } + s := strings.TrimSpace(string(raw)) + for _, key := range required { + if !strings.Contains(s, key) { + t.Fatalf("%s OVS other_config missing %q: %s", n.Name, key, s) + } + } + harness.Detail(t, "node", n.Name, "other_config", s) + } + + harness.Step(t, "xfrm SAs with AES-GCM established on every node") + harness.EventuallyErr(t, func() error { + for _, n := range fix.Cluster.Nodes { + c, cancel := context.WithTimeout(context.Background(), 10*time.Second) + raw, err := ssh.Run(c, n.Addr, "sudo ip xfrm state") + cancel() + if err != nil { + return fmt.Errorf("%s ip xfrm state: %w", n.Name, err) + } + s := string(raw) + if !strings.Contains(s, "aead") { + return fmt.Errorf("%s xfrm has no AEAD SAs:\n%s", n.Name, strings.TrimSpace(s)) + } + // Kernel renders the GCM AEAD as either `rfc4106(gcm(aes))` + // (RFC 4106 ESP AES-GCM, what strongSwan negotiates by default) + // or the bare `gcm(aes)` template name. Accept either. + if !strings.Contains(s, "gcm(aes)") { + return fmt.Errorf("%s xfrm SAs not AES-GCM:\n%s", n.Name, strings.TrimSpace(s)) + } + } + return nil + }, 90*time.Second, 5*time.Second) + + harness.Step(t, "tcpdump ESP traffic on underlay (best-effort)") + for _, n := range fix.Cluster.Nodes { + c, cancel := context.WithTimeout(context.Background(), 15*time.Second) + raw, err := ssh.Run(c, n.Addr, + "sudo timeout 5 tcpdump -i any -nn -c 5 'ip proto 50' 2>&1 || true", + ) + cancel() + if err != nil { + t.Logf("WARN: %s tcpdump ESP capture failed: %v", n.Name, err) + continue + } + s := strings.TrimSpace(string(raw)) + if strings.Contains(s, "ESP") { + harness.Detail(t, "node", n.Name, "esp_capture", "observed") + } else { + t.Logf("WARN: %s tcpdump saw no ESP traffic in 5s window (geneve may be idle):\n%s", n.Name, s) + } + } +} diff --git a/tests/e2e/multinode/tests_test.go b/tests/e2e/multinode/tests_test.go index c9425fca4..aa24f4d7c 100644 --- a/tests/e2e/multinode/tests_test.go +++ b/tests/e2e/multinode/tests_test.go @@ -13,7 +13,7 @@ import "testing" // sub-tests sharing one VPC + bastion + private trio + NAT GW setup chain. // Sub-test layout keeps JUnit granularity without paying 6× setup cost. // -// Parallelism (mulga-siv-127 Stage J): +// Parallelism: // // Bucket #1 (parallel): the read-only / independent Tests below call // t.Parallel(). They share the package-singleton trio (sync.Once gated) and @@ -98,3 +98,11 @@ func TestMultinodeVPCNetworking(t *testing.T) { t.Parallel() runVPCNetworking(t, requireMultiNodeFixture(t)) } + +// TestMultinodeIPSec is read-only over SSH — verifies the OVN native IPsec +// wiring (OVS DB cert pointers, xfrm SAs, ESP traffic) without launching +// any AWS resources. Safe to run alongside the parallel bucket. +func TestMultinodeIPSec(t *testing.T) { + t.Parallel() + runIPSec(t, requireMultiNodeFixture(t)) +} diff --git a/tests/e2e/reboot/reboot_test.go b/tests/e2e/reboot/reboot_test.go index 2d62f0037..12cf5bad2 100644 --- a/tests/e2e/reboot/reboot_test.go +++ b/tests/e2e/reboot/reboot_test.go @@ -75,6 +75,18 @@ WantedBy=multi-user.target UNIT systemctl daemon-reload systemctl enable --now reboot-e2e-httpd.service + +# Workaround for viperblock WAL <4MB skip-upload bug: +# WriteWALToChunk(false) is a no-op when the WAL is under 4MB so small +# user-data writes (the unit file, index.html) sit in the on-host WAL +# and are lost when the host reboots before the WAL hits the threshold. +# Write 8MB of zeros + sync to push WAL over the threshold so the +# unit-file write definitely lands in predastore before the host reboot. +# Drop when upstream fix lands. +dd if=/dev/zero of=/var/lib/spinifex-e2e-pad bs=1M count=8 status=none +sync +rm -f /var/lib/spinifex-e2e-pad +sync ` type fixture struct { diff --git a/tests/e2e/run-e2e.sh b/tests/e2e/run-e2e.sh index b8731f0c7..9a452df73 100755 --- a/tests/e2e/run-e2e.sh +++ b/tests/e2e/run-e2e.sh @@ -285,6 +285,21 @@ if [ $AUTH_EXIT -ne 0 ] && ! echo "$AUTH_OUTPUT" | grep -q 'InvalidPermission.Du fi echo " Default SG ingress: tcp/22 from 0.0.0.0/0" +# Phase 5f exercises ICMP egress through OVN; the reply path needs explicit +# ICMP ingress on the default SG even though SGs are stateful in AWS — keeps +# parity with the Go single-suite fixture. +set +e +ICMP_OUTPUT=$(aws ec2 authorize-security-group-ingress \ + --group-id "$DEFAULT_SG_PHASE5" \ + --protocol icmp --port -1 --cidr 0.0.0.0/0 2>&1) +ICMP_EXIT=$? +set -e +if [ $ICMP_EXIT -ne 0 ] && ! echo "$ICMP_OUTPUT" | grep -q 'InvalidPermission.Duplicate'; then + echo "Failed to authorize ICMP ingress on default SG: $ICMP_OUTPUT" + exit 1 +fi +echo " Default SG ingress: icmp from 0.0.0.0/0" + # Launch a VM (run-instances) echo "Running: aws ec2 run-instances --image-id $AMI_ID --instance-type $INSTANCE_TYPE --key-name test-key-1" # Capture full output for debugging @@ -3240,7 +3255,7 @@ if [ "$PUB_SSH_READY" = true ]; then exit 1 fi else - echo "WARN: SSH via public IP $PUB_IP not reachable (macvlan isolation or bridge not ready)" + echo "WARN: SSH via public IP $PUB_IP not reachable (bridge not ready)" echo "Falling back to API-only verification (OVN state already validated above)" fi @@ -3437,10 +3452,60 @@ echo " PASS: Bastion → private instance SSH works (hostname: $PRIV_HOSTNAME)" done if [ "$NAT_GW_OK" = false ]; then echo " FAIL: Private instance cannot reach internet WITH NAT GW after 10 attempts" - echo " Dumping OVN NAT rules for debugging:" - sudo ovn-nbctl --no-leader-only lr-nat-list "vpc-${DEFAULT_VPC}" 2>/dev/null || true - echo " OVN routes:" - sudo ovn-nbctl --no-leader-only lr-route-list "vpc-${DEFAULT_VPC}" 2>/dev/null || true + echo "" + echo "==================== Phase 8d failure diagnostics ====================" + echo "VPC: vpc-${DEFAULT_VPC}" + echo "Private subnet: ${PRIV_SUBNET_ID} (private VM IP: ${PRIV_PRIVATE_IP})" + echo "NATGW: ${NAT_GW_ID} → ${NAT_PUB_IP}" + echo "" + echo "--- ovn-nbctl lr-nat-list vpc-${DEFAULT_VPC} ---" + sudo ovn-nbctl --no-leader-only lr-nat-list "vpc-${DEFAULT_VPC}" 2>&1 || true + echo "" + echo "--- ovn-nbctl lr-route-list vpc-${DEFAULT_VPC} ---" + sudo ovn-nbctl --no-leader-only lr-route-list "vpc-${DEFAULT_VPC}" 2>&1 || true + echo "" + echo "--- ovn-nbctl show ---" + sudo ovn-nbctl --no-leader-only show 2>&1 || true + echo "" + echo "--- ovn-sbctl show ---" + sudo ovn-sbctl --no-leader-only show 2>&1 || true + echo "" + echo "--- ovn-sbctl list chassis ---" + sudo ovn-sbctl --no-leader-only list chassis 2>&1 || true + echo "" + echo "--- ovn-nbctl find Port_Group (membership + ACLs) ---" + sudo ovn-nbctl --no-leader-only --columns=name,ports,acls find Port_Group 2>&1 || true + echo "" + echo "--- ovn-nbctl list ACL (all VPC ACLs) ---" + sudo ovn-nbctl --no-leader-only list ACL 2>&1 || true + echo "" + echo "--- ovn-nbctl find Logical_Switch_Port name=*${PRIV_PRIVATE_IP//./_}* (private VM LSP) ---" + sudo ovn-nbctl --no-leader-only find Logical_Switch_Port addresses=\"*${PRIV_PRIVATE_IP}*\" 2>&1 || true + echo "" + echo "--- ovs-vsctl show ---" + sudo ovs-vsctl show 2>&1 || true + echo "" + echo "--- ovs-ofctl dump-flows br-int (filtered: NATGW IP/subnet/private IP) ---" + sudo ovs-ofctl dump-flows br-int 2>&1 \ + | grep -E "${NAT_PUB_IP//./\\.}|172\\.31\\.16\\.|${PRIV_PRIVATE_IP//./\\.}" \ + | head -200 || true + echo "" + echo "--- ovs-ofctl dump-flows br-int | wc -l (total flows) ---" + sudo ovs-ofctl dump-flows br-int 2>&1 | wc -l || true + echo "" + echo "--- ip route show ---" + ip route show 2>&1 || true + echo "" + echo "--- ip -br addr show ---" + ip -br addr show 2>&1 || true + echo "" + echo "--- private VM: ip addr / ip route / ip neigh ---" + $BASTION_SSH "$PRIV_SSH_CMD 'ip -br addr; echo ---; ip route; echo ---; ip neigh'" 2>&1 || true + echo "" + echo "--- private VM: ping -c 1 subnet gateway (172.31.16.1) ---" + $BASTION_SSH "$PRIV_SSH_CMD 'ping -c 1 -W 3 172.31.16.1; echo rc=\$?'" 2>&1 || true + echo "" + echo "==================== end Phase 8d diagnostics ====================" exit 1 fi diff --git a/tests/e2e/run-reboot-e2e.sh b/tests/e2e/run-reboot-e2e.sh index 3b55ada88..6470703c8 100755 --- a/tests/e2e/run-reboot-e2e.sh +++ b/tests/e2e/run-reboot-e2e.sh @@ -286,7 +286,7 @@ UNIT systemctl daemon-reload systemctl enable --now reboot-e2e-httpd.service -# Workaround for viperblock WAL <4MB skip-upload bug (mulga-siv-122): +# Workaround for viperblock WAL <4MB skip-upload bug: # WriteWALToChunk(false) is a no-op when the WAL is under 4MB so small # user-data writes (the unit file, index.html) sit in the on-host WAL # and are lost when the host reboots before the WAL hits the threshold. @@ -544,7 +544,7 @@ if [ "$IPS_PRESERVED" = true ]; then fi # 8.3: guest VMs relaunched post-reboot (LaunchTime > reboot time via describe-instances) -# Two-hop SSH to VPC private IPs is not possible in single-node macvlan mode — +# Two-hop SSH to VPC private IPs is not possible in single-node mode — # the host has no route to VPC subnets. LaunchTime from the daemon's recovery # path (reset to time.Now() on relaunch) is a reliable reboot signal. log "Verifying guest VMs relaunched post-reboot (LaunchTime check)..." diff --git a/tests/e2e/single/datapath_test.go b/tests/e2e/single/datapath_test.go index 3c50ae0f7..65aab9cc9 100644 --- a/tests/e2e/single/datapath_test.go +++ b/tests/e2e/single/datapath_test.go @@ -45,6 +45,7 @@ systemd-run --unit=sge-http --description="Phase 8e HTTP server" \ func runSGToSGDatapath(t *testing.T, fix *Fixture) { harness.Phase(t, "Single — Security Group SG-to-SG Datapath (OVN)") harness.SkipIfNoOVN(t) + requireSSHHealthy(t) // Bootstrap every prereq up front. runSGEInstance / primaryENI use // `fix.AMIID / InstanceType / KeyName / KeyPath` indirectly; resolve diff --git a/tests/e2e/single/lifecycle_test.go b/tests/e2e/single/lifecycle_test.go index 8a1bd8f5f..48cbe549d 100644 --- a/tests/e2e/single/lifecycle_test.go +++ b/tests/e2e/single/lifecycle_test.go @@ -3,11 +3,11 @@ package single import ( - "fmt" "net" "os/exec" "strconv" "strings" + "sync/atomic" "testing" "time" @@ -18,6 +18,23 @@ import ( "github.com/stretchr/testify/require" ) +// sshDatapathBroken is set the first time a single-node SSH probe times +// out. Downstream SSH-dependent tests call requireSSHHealthy(t) and skip +// rather than re-running the same 3-minute timeout against the same +// broken datapath. +var sshDatapathBroken atomic.Bool + +// requireSSHHealthy skips the calling test if a prior SSH probe on this +// single-node VM has already failed. Cuts the cascade cost of a broken +// datapath from 4×3min down to one fail + four skips. +func requireSSHHealthy(t *testing.T) { + t.Helper() + if sshDatapathBroken.Load() { + t.Skipf("skipping: earlier single-node SSH probe failed; " + + "not retrying to keep suite time bounded") + } +} + // runStopStart stops fix.Instance and waits for it to reach the // "stopped" state, then asserts that rebooting a stopped instance is // rejected with IncorrectInstanceState. Leaves the instance stopped so @@ -391,24 +408,17 @@ func runRunInstancesMultiCount(t *testing.T, fix *Fixture) { // TCP-port reachability alone is insufficient — sshd accepts the connect // while pam/cloud-init are still finishing, and the first real runSSH then // hits "Connection timed out during banner exchange" (CI run 26034322018). -// Proper hoist to harness.WaitForSSH is tracked under mulga-siv-101. func waitForSSHReady(t *testing.T, host string, port int, keyPath string) { t.Helper() + requireSSHHealthy(t) addr := net.JoinHostPort(host, strconv.Itoa(port)) harness.Step(t, "waiting for SSH handshake %s", addr) - harness.Eventually(t, func() bool { - cmd := exec.Command("ssh", - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "LogLevel=ERROR", - "-o", "ConnectTimeout=5", - "-o", "BatchMode=yes", - "-p", strconv.Itoa(port), - "-i", keyPath, - "ec2-user@"+host, - "true") - return cmd.Run() == nil - }, 3*time.Minute, 3*time.Second, fmt.Sprintf("SSH handshake %s never completed", addr)) + if !trySSHReady(host, port, keyPath, 3*time.Minute) { + sshDatapathBroken.Store(true) + t.Fatalf("Eventually: condition not met within 3m0s: "+ + "[SSH handshake %s never completed] "+ + "(sticky-skip enabled for downstream)", addr) + } } // waitForSSHHandshake is an alias kept for instance_test.go's call site. @@ -418,11 +428,14 @@ func waitForSSHHandshake(t *testing.T, host string, port int, keyPath string) { // trySSHReady mirrors waitForSSHReady but returns reachability as a bool // instead of t.Fatal-ing on timeout. Matches run-e2e.sh ~2850–2860, which -// treats SSH-via-public-IP timeout as a WARN ("macvlan isolation or bridge -// not ready") rather than a hard test failure — new-subnet IGW SNAT/DNAT +// treats SSH-via-public-IP timeout as a WARN ("bridge not ready") rather +// than a hard test failure — new-subnet IGW SNAT/DNAT // flow installation is timing-sensitive on shared CI runners and not part // of the EC2 surface contract this test exists to validate. func trySSHReady(host string, port int, keyPath string, timeout time.Duration) bool { + if sshDatapathBroken.Load() { + return false + } deadline := time.Now().Add(timeout) for { cmd := exec.Command("ssh", diff --git a/tests/e2e/single/main_test.go b/tests/e2e/single/main_test.go index af3fb71f2..15e7eaea8 100644 --- a/tests/e2e/single/main_test.go +++ b/tests/e2e/single/main_test.go @@ -79,6 +79,11 @@ func requireSingleNodeFixture(t *testing.T) *Fixture { TmpDir: tmpDir, } fix.PoolMode = detectPoolMode(env) + // admin init leaves the default SG closed (AWS parity); the e2e suite + // drives SSH + ICMP probes from the test runner's external IP, so open + // both on the default VPC's default SG once per process. Idempotent — + // mirrors run-e2e.sh Phase 5. + harness.EnsureDefaultSGOpen(t, awsCli) pkgFix = fix }) if pkgFixErr != nil {