feat: build an Oracle Linux appliance gold image that sets each unit up on first boot

A rerun on the create node restarts its NB and SB members, which can hand
the leadership to a peer. set-connection then went to the local socket
leader-only, failed, and the run exited 1 after the cluster had rejoined.

.github/e2e-cells.json

@@ -223,5 +223,27 @@
     "runner": "ci-single",
     "extra": "nat-single",
     "suite": ""
+  },
+  {
+    "cell": 37,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "dhcp",
+    "host": "oracle-10",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "",
+    "suite": ""
+  },
+  {
+    "cell": 38,
+    "install": "binary",
+    "topo": "real-multi",
+    "net": "nat",
+    "host": "oracle-10",
+    "arch": "x86_64",
+    "runner": "ci-multi",
+    "extra": "nat-multi",
+    "suite": ""
   }
 ]

.github/workflows/e2e-nightly.yml

@@ -10,7 +10,7 @@ on:
   workflow_dispatch:
     inputs:
       cells:
-        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 34-36 Oracle Linux single node (basic, reboot, routed NAT), 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
+        description: "Comma-separated cell numbers to run, no spaces (blank = all). 1-3, 8-10, 17-19 tofu permutation matrix, 34-36 Oracle Linux single node (basic, reboot, routed NAT), 37-38 Oracle Linux on three nodes (multi-node subset with IPsec, routed NAT), 20 baremetal, 21-22 ISO, 23 predastore throughput, 24 predastore fault injection, 25-27 service suites (eks, ecs, rds), 28 guest disk performance, 29 storage fault injection, 30 routed NAT on three nodes, 31 instance recovery on three nodes, 32 load balancer recovery on three nodes, 33 KV bucket quorum under node loss on three nodes"
         required: false
 
 permissions:

Makefile

@@ -186,13 +186,14 @@ test-build-scripts:
 		echo "-- $$t"; \
 		sh "$$t" || exit 1; \
 	done
-	@echo -e "\n....Running scripts/*_test.sh and scripts/lib/*_test.sh...."
-	@for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib -name '*_test.sh') | sort); do \
+	@echo -e "\n....Running scripts/*_test.sh, scripts/lib/*_test.sh and scripts/appliance/*_test.sh...."
+	@for t in $$( (find scripts -maxdepth 1 -name '*_test.sh'; find scripts/lib scripts/appliance -name '*_test.sh') | sort); do \
 		echo "-- $$t"; \
 		sh "$$t" || exit 1; \
 	done
 	@echo -e "\n....Running shellcheck over build/scripts/**/*.sh + the node scripts...."
 	shellcheck -S warning $$(find build/scripts -name '*.sh' | sort) $(NODE_SCRIPTS)
+	shellcheck -S warning -x scripts/build-oracle-appliance.sh $$(find scripts/appliance -name '*.sh' | sort)
 	@echo "  test-build-scripts ok"
 
 # E2E harness unit tests. Build-tagged `e2e` so they're skipped by the

docs/admin/host-firewall/README.md

@@ -158,6 +158,7 @@ Restarting the daemon is what re-arms it: the node rebuilds its peer list from t
 
 > [!WARNING]
 > Do not flush the whole nftables ruleset to clear the policy. The VPC daemon writes SNAT and per-elastic-IP forward rules into the `ip nat` and `ip filter` tables and reinstalls them only when it starts, so `nft flush ruleset` breaks elastic IPs silently until the next `systemctl restart spinifex-vpcd`. Use `disable`, which only ever touches `inet spinifex_filter`.
+> On Oracle Linux with IPsec on, a flush also removes `inet spinifex_ipsec_guard`, so Geneve to a peer without an SA goes out in clear until the daemon puts the table back, within 60 seconds.
 
 ## The Files
 

docs/install/install-multi-node/README.md

@@ -163,11 +163,14 @@ Open between all hosts:
 | Protocol and port | Used by |
 |---|---|
 | UDP 6081 | Geneve tunnels |
+| UDP 500, 4500 and IP protocol 50 (ESP) | IPsec for the Geneve tunnels, while IPsec is on |
 | TCP 4222, 4248 | NATS |
 | TCP 6641, 6642 | OVN northbound and southbound |
 | TCP 8443 | Predastore S3 gate |
 | UDP 6660, 7660 | Object shards, metadata consensus |
 
+UDP 4500 carries the tunnel traffic only where IKE detects NAT between two nodes; otherwise it travels as ESP, on Oracle Linux as on Debian.
+
 Open between the three OVN database servers (servers 1 to 3) only:
 
 | Protocol and port | Used by |
@@ -207,7 +210,10 @@ Spinifex's own **host** firewall is separate, and already carries this policy
 
 Choose one method and apply it to **every** server in the cluster.
 
-**Option A — existing OS.** On a server already running Ubuntu 26.04 or Debian 13 (Oracle Linux runs a single node only for now, see [Oracle Linux Hosts](/docs/install-oracle-linux)):
+**Option A — existing OS.** Use this on servers already running Ubuntu 26.04, Debian 13 or Oracle Linux 10, and prepare an Oracle Linux server with [Oracle Linux Hosts](/docs/install-oracle-linux) first.
+Every server in a cluster must run the same OS: Oracle Linux never shares a cluster with Debian or Ubuntu, and `spx admin join` does not check this for you.
+
+Run the installer on each server:
 
 ```bash
 curl -fsSL https://install.mulgadc.com | bash

docs/install/install-oracle-linux/README.md

@@ -1,12 +1,13 @@
 ---
 title: "Oracle Linux Hosts"
 seoTitle: "Install Spinifex on Oracle Linux 10 — Spinifex Docs"
-description: "Prepare an Oracle Linux 10 host for Spinifex: UEK R8, SELinux, a NetworkManager br-wan bridge, the CentOS NFV SIG repository, firewalld handover and updates."
+description: "Prepare Oracle Linux 10 hosts for Spinifex: UEK R8, SELinux, a NetworkManager br-wan bridge, the NFV SIG repository, IPsec on libreswan and host updates."
 category: "Install"
 tags:
   - install
   - oracle linux
   - single node
+  - multi node
 resources:
   - title: "Oracle Linux 10 Documentation"
     url: "https://docs.oracle.com/en/operating-systems/oracle-linux/10/"
@@ -35,13 +36,12 @@ Follow it alongside [Single-Node Install](/docs/install).
 
 **Supported on Oracle Linux:**
 
-- A single node.
+- A single node, or a cluster in which every node runs Oracle Linux; servers beyond the third join as compute nodes, as on Debian.
 - Guests, VPCs, routed NAT, Elastic IPs, and Application and Network Load Balancers.
 - Whole-GPU passthrough, on x86_64 GPU hosts (see [GPU Passthrough](/docs/gpu-passthrough)).
 
 **Not supported on Oracle Linux yet:**
 
-- Clusters of more than one node. They wait for IPsec on libreswan, so `spx admin init` turns IPsec off on Oracle Linux and refuses `--ipsec=true`.
 - Clusters that mix Oracle Linux with Debian or Ubuntu nodes, on any release.
 - SELinux enforcing, aarch64, Oracle Linux 9 or earlier, the Red Hat Compatible Kernel, a ZFS root, and NVIDIA MIG.
 
@@ -51,7 +51,7 @@ Spinifex installs `openvswitch3.5` and `ovn25.03`, the versions Debian 13 ships,
 QEMU comes from Oracle's KVM Utilities repository, because only that build has the `microvm` machine type Spinifex load balancers boot on.
 
 An Oracle Linux node must be a dedicated Spinifex host.
-Spinifex claims every `tap*` device and stops libreswan's `ipsec.service`, so VPNs, hand-run QEMU guests and libreswan tunnels of your own cannot share the host.
+Spinifex claims every `tap*` device and owns libreswan's `ipsec.service`, which it keeps stopped on a single node and runs for its own tunnels on a cluster node, so VPNs, hand-run QEMU guests and libreswan tunnels of your own cannot share the host.
 Crypto policies other than `DEFAULT` and `FIPS` are untested, and so is Secure Boot.
 
 ## Prerequisites
@@ -134,8 +134,28 @@ Run the installer as a user with `sudo`, exactly as on Debian:
 curl -fsSL https://install.mulgadc.com | bash
 ```
 
-Then continue with [Single-Node Install](/docs/install) from Step 2: `setup-ovn.sh --management`, `spx admin init` and `systemctl start spinifex.target`.
-`spx admin init` prints that IPsec is off on this host; on a single node there is no peer to encrypt traffic to.
+For a single node, continue with [Single-Node Install](/docs/install) from Step 2: `setup-ovn.sh --management`, `spx admin init` and `systemctl start spinifex.target`.
+IPsec stays off on a single node, because there is no peer to encrypt traffic to.
+
+For a cluster, prepare and install every node this way, then continue with [Multi-Node Install](/docs/install-multi-node) from Step 2.
+
+### Clusters
+
+Every node in an Oracle Linux cluster runs Oracle Linux.
+`install-node.sh` refuses a host list that mixes Oracle Linux with Debian or Ubuntu, but `spx admin join` does not check, so keep to one OS yourself when joining by hand.
+
+IPsec encrypts the Geneve tunnels between nodes and is on by default, as on Debian.
+On Oracle Linux it runs on libreswan, the IKE daemon Oracle ships, rather than strongSwan.
+Once the node's IPsec certificate exists, the daemon unmasks and starts `ipsec.service` and `openvswitch-ipsec.service`.
+`ovs-monitor-ipsec` then loads the node certificate, its key and the cluster CA into libreswan's NSS database at `/var/lib/ipsec/nss`, under nicknames that start with `ovs_cert`.
+
+The tunnels are plain ESP (IP protocol 50), as on Debian, negotiated over UDP 500.
+Open UDP 500, UDP 4500 and ESP between nodes: libreswan moves to UDP 4500 if it detects NAT between two nodes.
+The Spinifex host firewall opens all three to cluster peers; a firewall between the nodes must allow them too.
+While IKE to a peer fails, the node drops its Geneve traffic to that peer rather than send it in clear.
+The nft table `inet spinifex_ipsec_guard` drops outbound UDP 6081 from the encap address whose route carries no SA, which also covers the moments pluto flushes the xfrm policy table as it starts and stops.
+It is not persisted: `ipsec.service` puts it back at boot before ovn-controller builds the tunnels, and the daemon re-syncs it every 60 seconds.
+`spx admin init --ipsec=false` leaves the tunnels unencrypted, as on any OS.
 
 ## Step 2. Know What the Installer Changed
 
@@ -182,7 +202,9 @@ Those advisories do not cover the NFV SIG packages, so follow Open vSwitch and O
 
 ## Step 5. Uninstall
 
-`/usr/local/share/spinifex/uninstall-spx.sh` removes Spinifex as on Debian, and lists what it leaves in place on Oracle Linux: the repository changes and the SIG key, `OVS_USER_ID` and `OVN_USER_ID`, the OVS and OVN logrotate edits, the chrony include, `/etc/modules-load.d/spinifex.conf`, the SELinux mode, a masked `ipsec.service`, firewalld disabled, and any GPU passthrough configuration.
+`/usr/local/share/spinifex/uninstall-spx.sh` removes Spinifex as on Debian.
+On a cluster node it also stops `openvswitch-ipsec.service` and deletes the `ovs_cert*` entries from libreswan's NSS database, so the node can join a different cluster; other NSS identities and your connection files in `/etc/ipsec.d` stay.
+It lists what it leaves in place on Oracle Linux: the repository changes and the SIG key, `OVS_USER_ID` and `OVN_USER_ID`, the OVS and OVN logrotate edits, the chrony include, `/etc/modules-load.d/spinifex.conf`, the SELinux mode, a masked `ipsec.service`, firewalld disabled, and any GPU passthrough configuration.
 `--purge-deps` is refused on Oracle Linux, because dnf cannot tell the packages Spinifex installed from the host's own; remove them with `dnf remove` yourself.
 
 ## Troubleshooting
@@ -224,6 +246,20 @@ If it prints `yes`, run `sudo nmcli connection modify br-wan bridge.stp no` and
 
 Check that `/usr/bin/spx` links to `/usr/local/bin/spx`; run the installer again if it does not, or call `sudo /usr/local/bin/spx` meanwhile.
 
+### Cluster nodes form no IPsec tunnels
+
+Check that both IPsec services run, and that libreswan holds the node's certificate and the cluster CA:
+
+```bash
+systemctl is-active ipsec.service openvswitch-ipsec.service
+sudo certutil -L -d sql:/var/lib/ipsec/nss     # ovs_certkey_<node>, ovs_cert_cacert
+sudo ipsec trafficstatus                       # one line per peer tunnel
+```
+
+If the services are inactive, check the daemon log with `journalctl -u spinifex-daemon` for the IPsec reconcile.
+If the certificates are present but no tunnel is listed, check that UDP 500 and ESP pass between the nodes in both directions, and look for `SEC_ERROR_UNKNOWN_ISSUER` in `journalctl -u ipsec`, which means a peer's certificate is from another CA.
+Until the tunnel forms, guest traffic between those two nodes is dropped, which is the guard in [Clusters](#clusters) working.
+
 ### CA certificate not trusted
 
 `sudo spx admin init` installs the CA automatically. To install it again by hand on Oracle Linux:

docs/install/install/README.md

@@ -39,7 +39,7 @@ This guide installs Spinifex on a single server using the binary installer. For
 
 - Ubuntu 26.04
 - Debian 13
-- Oracle Linux 10.2 or a later 10.x, on UEK R8, x86_64 only, as a single node. Open vSwitch and OVN come from the CentOS NFV SIG, a community repository that Oracle does not support. Prepare the host with [Oracle Linux Hosts](/docs/install-oracle-linux) first.
+- Oracle Linux 10.2 or a later 10.x, on UEK R8, x86_64 only, as a single node or, with [Multi-Node Install](/docs/install-multi-node), a cluster. Open vSwitch and OVN come from the CentOS NFV SIG, a community repository that Oracle does not support. Prepare the host with [Oracle Linux Hosts](/docs/install-oracle-linux) first.
 
 **What Gets Installed:**
 

docs/security/network-connections/README.md

@@ -111,8 +111,8 @@ behavior in other words.
 | 6643 | OVN Northbound DB (RAFT) | OVSDB/TCP | Cluster | NB database RAFT replication between the 3 quorum nodes | Cluster network only; TLS planned |
 | 6644 | OVN Southbound DB (RAFT) | OVSDB/TCP | Cluster | SB database RAFT replication between the 3 quorum nodes | Cluster network only; TLS planned |
 | 6081 | OVN (Geneve) | UDP | Encap | Tenant traffic overlay between chassis. A kernel UDP-tunnel socket, so packets are delivered locally and traverse the host's netfilter input hook before OVS sees them — a host firewall must accept them explicitly. The socket is opened by the kernel tunnel driver and takes no bind address, so it binds the wildcard by design and reach must be restricted by firewall. | None — see 500/4500 |
-| 500, 4500 | strongSwan `charon` | IKEv2 / UDP | Encap | IKE and NAT-T for the IPsec protecting Geneve, managed entirely by `ovs-monitor-ipsec`. Binds the wildcard by design (the upstream strongSwan default, accepted rather than overridden), so reach must be restricted by firewall. | Certificate-based, against the cluster CA |
-| — | ESP | IP proto 50 | Encap | The IPsec payload itself, once IKE has negotiated an SA | Cluster CA |
+| 500, 4500 | strongSwan `charon`, or libreswan `pluto` on Oracle Linux | IKEv2 / UDP | Encap | IKE and NAT-T for the IPsec protecting Geneve, managed entirely by `ovs-monitor-ipsec`; 4500 carries the ESP payload only where IKE detects NAT between two nodes. `charon` binds the wildcard by design (the upstream strongSwan default, accepted rather than overridden); `pluto` binds each host address separately, the WAN address included, which is no narrower. Either way reach must be restricted by firewall. | Certificate-based, against the cluster CA |
+| — | ESP | IP proto 50 | Encap | The IPsec payload itself, once IKE has negotiated an SA, on Oracle Linux as on Debian. | Cluster CA |
 | 8222 | spinifex-nats (monitoring) | HTTP | Localhost | `varz`/`subsz` metrics consumed by the daemon | Loopback only |
 | 323 | chronyd | NTP | Localhost | Time sync client control socket | Loopback only |
 | 169.254.169.254:80 | spinifex-vpcd (IMDS) | HTTP | Guest | Instance metadata service. One socket per instance, bound to that instance's `ime-*` interface. Terminates on the host, so it traverses the netfilter input hook. | Instance identity by interface; IMDSv2 tokens |
@@ -177,7 +177,7 @@ Control-plane and data-plane traffic between Spinifex nodes, for completeness an
 | OVN NB/SB (client) | 6641 / 6642 | Cluster network only (TLS planned) | Network control plane; vpcd and ovn-controller dial the quorum |
 | OVN NB/SB (RAFT) | 6643 / 6644 | Cluster network only (TLS planned) | NB/SB database replication across the 3 quorum nodes |
 | OVN tunnels (Geneve) | UDP 6081 | Encapsulated by IPsec when `network.ipsec_enabled` is true, which is the default on multi-node clusters | Tenant traffic overlay between chassis, on the `vpc` plane |
-| OVN IPsec (IKE / NAT-T / ESP) | UDP 500, UDP 4500, IP proto 50 | Certificate-based against the cluster CA, negotiated by `ovs-monitor-ipsec` | Protects the Geneve tunnels above. OVN-native only — no layer manages strongSwan directly. |
+| OVN IPsec (IKE / NAT-T / ESP) | UDP 500, UDP 4500, IP proto 50 | Certificate-based against the cluster CA, negotiated by `ovs-monitor-ipsec` | Protects the Geneve tunnels above. OVN-native only: no layer configures strongSwan or libreswan directly. On Oracle Linux the daemon also blocks outbound Geneve that no SA covers, so a peer failing IKE gets no plaintext. |
 | Instance DNS forwarding | UDP/TCP 5300 | None | Each node's per-instance DNS shim forwards guest queries to peer nodes' northstar `:5300` |
 
 Nodes **must** sit on a network segment that is not routed to tenant/guest VMs or to the internet. The Predastore blob and meta transports and the OVN DBs are cluster-internal and must not be reachable from anywhere else.
@@ -251,7 +251,7 @@ Every listener and outbound destination is controlled by one of these files. Cha
 
 | File | Keys | Controls |
 |------|------|----------|
-| `/etc/spinifex/spinifex.toml` | `nodes.<node>.{awsgw,nats,predastore,daemon}.host`, `nodes.<node>.vpcd.ovn_{nb,sb}_addr`, `nodes.<node>.daemon.dev_networking`, `network.ipsec_enabled` | Per-service bind addresses/ports; dev-mode QEMU port forwarding. `network.ipsec_enabled` (default `true`) decides whether OVN-native IPsec protects the Geneve tunnels, and therefore whether `charon` listens on 500/4500 — single-node clusters never enable it. |
+| `/etc/spinifex/spinifex.toml` | `nodes.<node>.{awsgw,nats,predastore,daemon}.host`, `nodes.<node>.vpcd.ovn_{nb,sb}_addr`, `nodes.<node>.daemon.dev_networking`, `network.ipsec_enabled` | Per-service bind addresses/ports; dev-mode QEMU port forwarding. `network.ipsec_enabled` (default `true`) decides whether OVN-native IPsec protects the Geneve tunnels, and therefore whether the IKE daemon (`charon`, or `pluto` on Oracle Linux) listens on 500/4500 — single-node clusters never enable it. |
 | `/etc/spinifex/nats.conf` | `listen`, `cluster.listen`, `cluster.routes`, `http`, `tls`, `cluster.authorization` | NATS client/cluster/monitoring listeners, peer routes, TLS, cluster token. |
 | `/etc/spinifex/predastore/predastore.toml` | `[[host]].bind_addr`, `[[host]].addr`, `[[host]].tls_cert`, `[[host]].tls_key`, `[[host.node]].role`, `[[host.node]].port` | Predastore host and node layout: `bind_addr` is the address the host's sockets bind, `addr` is the address peer hosts dial it on, and both carry no port — the nodes pinned to the host supply their own. The service's `--host`/`--port` override the bind address and the gate's S3 port. |
 | `/etc/spinifex/northstar/northstar.toml` | `listen`, `forwarders` | DNS listen addresses (`:53` on the advertise address, `:5300` wildcard) and upstream resolvers. |

docs/testing/e2e-coverage.md

@@ -40,7 +40,7 @@ Runs **on** a node, shelling out to `ip`/`iptables`/`ovn-nbctl` locally. Skips u
 
 Phases 7 and 8 are gated on a cluster by what the datapath can actually do: the transit segment is a localnet, so host-to-guest Tier 1 ingress only exists where the VPC's gateway router port and the guest are both on this host. The VPC ingress route itself is installed on every node, so phase 8 checks it unconditionally.
 
-Three nightly cells run it: `nat-single` (cell 19), its Oracle Linux twin (cell 36) and `nat-multi` (cell 30). Cell 20 is the bare-metal job and is not in the permutation matrix at all, which is why the multi-node NAT cell is numbered past the end rather than filling a gap.
+Four nightly cells run it: `nat-single` (cell 19) and `nat-multi` (cell 30), and their Oracle Linux twins (cells 36 and 38). Cell 20 is the bare-metal job and is not in the permutation matrix at all, which is why the multi-node NAT cell is numbered past the end rather than filling a gap.
 
 ### `multinode` — behaviour that only exists on more than one node
 
@@ -50,6 +50,9 @@ Three nightly cells run it: `nat-single` (cell 19), its Oracle Linux twin (cell
 
 **`TestMultinodeDNS` convergence budgets are env-tunable.** `SPINIFEX_MULTINODE_DNS_TOPOLOGY_TIMEOUT` (default `90s`) bounds every node agreeing on the NS topology, `SPINIFEX_MULTINODE_DNS_TIMEOUT` (default `2m`) bounds a record becoming answerable from a guest or a node, and `SPINIFEX_MULTINODE_DNS_POLL` (default `3s`) is the poll interval for both. Values are Go durations; a malformed or non-positive value panics rather than falling back to the default.
 
+**`TestMultinodeOVNRaft` restarts each node's OVN central in turn before it kills the NB leader.**
+A `setup-ovn.sh` rerun restarts a node's NB and SB members together, on EL through the one `ovn-northd.service`, so the test does the same on every database node in turn, waits for both members to be back as cluster members under a known leader, and requires an NB write to land after each step and the full quorum with one leader at the end.
+
 **`TestMultinodeIPSec` proves guest traffic is encrypted on the wire, not only that IPsec is configured.**
 It first requires every node to carry the OVS certificate, key and CA pointers, `NB_Global.ipsec=true` never asserted over an unconfigured chassis, and an AES-GCM SA in `ip xfrm state`.
 None of those fails when SAs exist but traffic bypasses them, so it then launches two instances on distinct nodes and sends 200 pings of 1000 bytes between them.

scripts/README.md

@@ -7,9 +7,10 @@ Node installation, dev-environment lifecycle, guest image builds, and verificati
 | Script | What it does |
 |---|---|
 | `setup.sh` | The production binary installer, served from `install.mulgadc.com`. Installs deps, users, sudoers, the host firewall policy, files, systemd units, logrotate and udev rules, and loads `openvswitch`, `geneve`, `vhost_net` and `tun`, listing them in `/etc/modules-load.d/spinifex.conf` for later boots. It masks firewalld wherever it is installed, since the Spinifex nftables policy is the only supported node firewall. An unset `--firewall` keeps the mode recorded in `/etc/spinifex/firewall/mode`, and otherwise arms the policy when firewalld was active. Where systemd-resolved is installed but not running, resolvconf is absent and NetworkManager runs, it hands host DNS to resolved: it saves `/etc/resolv.conf` once to `/var/lib/spinifex-reset/resolv.conf.orig`, links `/etc/resolv.conf` to the resolved stub, and stops unless resolved has upstream servers and the package repository host still resolves. It links `/usr/bin/spx` to `/usr/local/bin/spx` where sudo's PATH leaves out `/usr/local/bin`, and takes the nbdkit plugin directory from `nbdkit --dump-config`, stopping when nbdkit cannot say. `SETUP_STAGES` runs a subset; `ISO_BUILD=1` runs it inside the ISO builder's chroot. |
-| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services, writes their `ovn-ctl` flags to `/etc/default/ovn-central` (Debian) or `/etc/sysconfig/ovn-northd` (EL), and records the role in `/etc/spinifex/ovn-role`. A later run without `--management` refuses while that file, or the `OVN_CTL_OPTS` line older nodes carry in `/etc/default/ovn-central`, records the role. It adds `maxsize 100M` to the OVS and OVN logrotate files, plus `rotate 5` where the file sets no count, and rotates as root, matching the daemons. On a host whose IPsec stack is libreswan it stops and masks `openvswitch-ipsec.service` and `ipsec.service`, because OVN IPsec does not run on libreswan yet; elsewhere it enables and starts `openvswitch-ipsec.service`, unless the daemon has masked it to keep IPsec off. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start, starting `ovn-controller` again when an OVS stop took it down; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
+| `setup-ovn.sh` | Bootstraps a compute node for OVN VPC networking: installs OVN/OVS, creates `br-int`, configures the WAN bridge (bridged, DHCP, or routed-NAT uplink), sets the chassis identity, and applies overlay sysctls. It runs OVS and OVN as root, setting `OVS_USER_ID` and `OVN_USER_ID` where `/etc/sysconfig` holds them, and hooks the socket-permission helper onto `ovs-vswitchd`, `ovn-controller`, `openvswitch-ipsec` and, on EL, `ovn-northd`. `--management` also starts the OVN central services, writes their `ovn-ctl` flags to `/etc/default/ovn-central` (Debian) or `/etc/sysconfig/ovn-northd` (EL), and records the role in `/etc/spinifex/ovn-role`. With clustered databases it returns only once this node's NB and SB members are back in their RAFT clusters under a known leader, and fails after `OVN_RAFT_REJOIN_WAIT` seconds (default 60) otherwise, so reruns one node at a time never take two members of a cluster down. A later run without `--management` refuses while that file, or the `OVN_CTL_OPTS` line older nodes carry in `/etc/default/ovn-central`, records the role. It adds `maxsize 100M` to the OVS and OVN logrotate files, plus `rotate 5` where the file sets no count, and rotates as root, matching the daemons. On a host whose IPsec stack is libreswan it enables, stops and masks `openvswitch-ipsec.service` and `ipsec.service` for the daemon to unmask when IPsec is on, unless `openvswitch-ipsec.service` is already running; elsewhere it enables and starts `openvswitch-ipsec.service`, unless the daemon has masked it to keep IPsec off. The links it creates persist through systemd-networkd where that is enabled, and otherwise through `/etc/spinifex/hostnet.json`, which `spinifex-hostnet.service` (`spx admin hostnet apply`) replays at boot and on every OVS start, starting `ovn-controller` again when an OVS stop took it down; the first run records the choice in `/etc/spinifex/hostnet-backend`. Where NetworkManager runs, it leaves the links Spinifex owns unmanaged. |
 | `install-node.sh` | Forms a multi-node cluster across hosts that are already running Spinifex: rebuilds the OVN database in clustered form, runs `spx admin init` on the first host and `spx admin join` on the rest, then verifies. Before touching any host it refuses hosts of different OS families (`distro_os_family`) or `spx` versions, and `--ipsec on` when any host uses libreswan. The scripted form of [install-multi-node](../docs/install/install-multi-node/README.md). Destroys the joining nodes' logical network state and crypto — see its `--help`. |
 | `dev-install.sh` | Full local dev setup through the production installer: builds from source, packs a tarball, runs `setup.sh`, initialises the cluster, and starts services under systemd. |
+| `build-oracle-appliance.sh` | Turns an Oracle Linux 10.2 system reachable over SSH with passwordless sudo into a sealed gold image, then powers it off. It updates OL and reboots into the newest UEK R8, sets SELinux permissive, installs Spinifex without initialising it, and leaves `spinifex.target` and the NATS watchdog disabled. It creates the `spinifex` login with a default password that must be changed at first login, installs `appliance/firstboot.sh` as `spinifex-appliance-firstboot.service`, and wipes the system's identity (machine-id, SSH host keys, OVS chassis, NetworkManager profiles, logs). On each unit's first boot that service puts the first physical NIC with carrier on a DHCP `br-wan`, names the host `spinifex-<mac>`, runs `setup-ovn.sh --management` and `spx admin init --nodes 1`, and shows the IP and login on the console and in `/etc/motd`. `--channel`, `--version` and `--tarball` pick what to install. |
 | `clone-deps.sh` | Clones or updates the viperblock and predastore checkouts alongside spinifex for cross-repo development. |
 
 ### `lib/distro.sh`: host-OS capability probes
@@ -58,7 +59,7 @@ Both ship to `/usr/local/share/spinifex/` on every node, so they are available o
 
 | Script | What it does |
 |---|---|
-| `node-reset.sh` | Clears **state** and leaves the node installed: stops services, destroys guests, wipes `/etc/spinifex`, `/var/lib/spinifex`, `/var/log/spinifex` and the OVN databases, and removes the OVS bridges. It stops OVS and OVN central by the unit names the host's packages use. It leaves the NetworkManager unmanaged-device config, which the next `setup-ovn.sh` run rewrites, so NetworkManager cannot take a NIC back in between. The node is then ready for a fresh `spx admin init`. `--keep-data` preserves `/var/lib/spinifex`; `SPX_RESET_CONFIRM=reset` runs it unattended. |
+| `node-reset.sh` | Clears **state** and leaves the node installed: stops services, destroys guests, wipes `/etc/spinifex`, `/var/lib/spinifex`, `/var/log/spinifex` and the OVN databases, and removes the OVS bridges. On a libreswan host it deletes the `ovs_cert_*` and `ovs_certkey_*` entries `ovs-monitor-ipsec` imported into libreswan's NSS database, leaving other identities and `/etc/ipsec.d/*.conf` alone. It stops OVS and OVN central by the unit names the host's packages use. It leaves the NetworkManager unmanaged-device config, which the next `setup-ovn.sh` run rewrites, so NetworkManager cannot take a NIC back in between. The node is then ready for a fresh `spx admin init`. `--keep-data` preserves `/var/lib/spinifex`; `SPX_RESET_CONFIRM=reset` runs it unattended. |
 | `uninstall-spx.sh` | Removes the **software** — the inverse of `setup.sh`. Delegates the state teardown above to `node-reset.sh` rather than reimplementing it, then removes the binary, the nbdkit plugin, the systemd units and slices, the service users, the sudoers grant, the sysctl/logrotate/chrony/udev drop-ins, the `systemd-networkd` units, the host firewall policy and its loaded nft table, the Spinifex systemd-resolved drop-in, and the Spinifex CA from the host trust store. It unmasks firewalld and leaves it disabled. Where `setup.sh` handed DNS to systemd-resolved, it restores the saved `/etc/resolv.conf` and turns resolved off. Verifies afterwards and exits non-zero naming anything that survived, then lists the host settings `setup.sh` changed that it leaves in place, such as the dnf repositories, `OVS_USER_ID`, masked IPsec units and the SELinux mode. |
 
 `node-reset.sh` finds guests by process name, `qemu-system-*` or `qemu-kvm`, owned by the daemon's service user, and waits for them to exit before it deletes any state.

scripts/appliance/firstboot.sh

@@ -0,0 +1,146 @@
+#!/bin/bash
+# Spinifex appliance first boot, for Oracle Linux units cloned from the gold
+# image that build-oracle-appliance.sh seals. The OL counterpart of the ISO's
+# spinifex-firstboot.sh: it also does the per-unit work the ISO installer does
+# before first boot (hostname, br-wan on the NIC that is present), then sets
+# up a single node. On success it writes $DONE_MARKER and the unit's
+# ConditionPathExists=! stops it running again. A failed run leaves no marker,
+# so the next boot retries from the top; every step below is safe to repeat.
+set -euo pipefail
+
+DONE_MARKER=/var/lib/spinifex/.appliance-firstboot-done
+SHARE=/usr/local/share/spinifex
+
+# shellcheck source=lib.sh
+. "$SHARE/appliance/lib.sh"
+# shellcheck source=../lib/distro.sh
+. "$SHARE/lib/distro.sh"
+
+log() { echo "[appliance-firstboot] $*"; }
+
+if [ -f "$DONE_MARKER" ]; then
+    log "already complete, skipping"
+    exit 0
+fi
+
+# ── Primary NIC ─────────────────────────────────────────────────────────────
+# Links start down when NetworkManager has no profile for them, so bring every
+# physical NIC up to read its carrier, and give a cable up to 60s to come up.
+nics=$(appliance_physical_nics)
+if [ -z "$nics" ]; then
+    log "no physical Ethernet device found"
+    exit 1
+fi
+for nic in $nics; do
+    ip link set "$nic" up || true
+done
+for _i in $(seq 1 60); do
+    for nic in $nics; do
+        [ "$(cat "/sys/class/net/$nic/carrier" 2>/dev/null)" = 1 ] && break 2
+    done
+    sleep 1
+done
+NIC=$(appliance_pick_nic)
+MAC=$(cat "/sys/class/net/$NIC/address")
+log "primary NIC $NIC ($MAC), carrier $(cat "/sys/class/net/$NIC/carrier" 2>/dev/null || echo 0)"
+
+HOSTNAME_NEW=$(appliance_hostname "$MAC")
+hostnamectl set-hostname "$HOSTNAME_NEW"
+log "hostname $HOSTNAME_NEW"
+
+# ── br-wan on the primary NIC, DHCP ─────────────────────────────────────────
+# The bridge takes the NIC's MAC so a DHCP reservation made for the unit's
+# port keeps working. bridge.stp no: with STP on, the port sits in its
+# forwarding delay long enough for DHCP on the bridge to time out.
+for con in $(nmcli -g UUID,DEVICE connection show | awk -F: -v d="$NIC" '$2 == d {print $1}'); do
+    nmcli connection delete "$con" || true
+done
+nmcli connection delete br-wan br-wan-port >/dev/null 2>&1 || true
+nmcli connection add type bridge ifname br-wan con-name br-wan \
+    bridge.stp no bridge.mac-address "$MAC" \
+    ipv4.method auto ipv6.method disabled connection.autoconnect yes
+nmcli connection add type bridge-slave ifname "$NIC" master br-wan con-name br-wan-port \
+    connection.autoconnect yes
+nmcli connection up br-wan || true
+
+log "waiting for a DHCP address on br-wan"
+WAN_IP=""
+for _i in $(seq 1 120); do
+    WAN_IP=$(ip -4 -o addr show br-wan scope global 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | head -1)
+    if [ -n "$WAN_IP" ] && ip -4 route show default dev br-wan | grep -q .; then
+        break
+    fi
+    WAN_IP=""
+    sleep 1
+done
+if [ -z "$WAN_IP" ]; then
+    log "br-wan got no DHCP address and default route in 120s; check the cable on $NIC and the site's DHCP"
+    exit 1
+fi
+log "br-wan $WAN_IP"
+
+# ── Clock ───────────────────────────────────────────────────────────────────
+# Credentials and TLS certificates are minted below, and a certificate issued
+# against a wrong RTC carries a NotBefore that outlives the correction.
+# Bounded and non-fatal, as on the ISO.
+chronyc -a makestep >/dev/null 2>&1 || true
+if chronyc waitsync 12 0.1 0 5 >/dev/null 2>&1; then
+    log "clock synced: $(date -uIs)"
+else
+    log "WARNING: clock not synced, continuing with $(date -uIs)"
+fi
+
+# ── OVS and OVN ─────────────────────────────────────────────────────────────
+# Start them before setup-ovn.sh so their databases exist when it runs: on
+# physical hardware first-boot DB initialisation outlasts its own timeout.
+OVS_UNIT=$(distro_ovs_unit)
+OVN_LAYOUT=$(distro_ovn_central_layout)
+case $OVN_LAYOUT in
+    split) OVN_CENTRAL_UNIT=ovn-central.service ;;
+    lone-northd) OVN_CENTRAL_UNIT=ovn-northd.service ;;
+esac
+systemctl start "$OVS_UNIT" "$OVN_CENTRAL_UNIT"
+for _i in $(seq 1 120); do
+    if ovn-nbctl --timeout=2 get-connection >/dev/null 2>&1; then
+        log "OVN NB DB ready (${_i}s)"
+        break
+    fi
+    sleep 1
+done
+
+"$SHARE/setup-ovn.sh" --management
+
+# ── Single-node cluster ─────────────────────────────────────────────────────
+# Same arguments as the ISO's single-NIC DHCP path. spx writes credentials to
+# disk and never prints them.
+spx admin init --node "$HOSTNAME_NEW" --nodes 1 2>&1
+
+# spx ran as root, so the files it wrote are root:root. setup.sh's fixown
+# stage knows each file's owner and mode.
+SETUP_STAGES=fixown "$SHARE/setup.sh"
+
+# ── spinifex login ──────────────────────────────────────────────────────────
+if [ ! -f /home/spinifex/.ssh/id_ed25519 ]; then
+    install -d -m 700 -o spinifex -g spinifex /home/spinifex/.ssh
+    ssh-keygen -q -t ed25519 -f /home/spinifex/.ssh/id_ed25519 -N "" -C "spinifex@$HOSTNAME_NEW"
+    chown spinifex:spinifex /home/spinifex/.ssh/id_ed25519 /home/spinifex/.ssh/id_ed25519.pub
+fi
+# init wrote the AWS profile under HOME=/root; the operator logs in as spinifex.
+if [ -f /root/.aws/credentials ]; then
+    install -d -m 700 -o spinifex -g spinifex /home/spinifex/.aws
+    install -m 600 -o spinifex -g spinifex /root/.aws/credentials /home/spinifex/.aws/credentials
+    [ -f /root/.aws/config ] && install -m 600 -o spinifex -g spinifex /root/.aws/config /home/spinifex/.aws/config
+fi
+
+# ── Services ────────────────────────────────────────────────────────────────
+# Enabled here, not in the image, so a clone runs nothing of Spinifex before
+# init has written its config.
+systemctl enable spinifex.target spinifex-banner.service spinifex-firewall.service
+systemctl enable --now spinifex-nats-watchdog.timer
+
+mkdir -p "$(dirname "$DONE_MARKER")"
+touch "$DONE_MARKER"
+
+systemctl start spinifex.target
+/usr/local/bin/spx admin banner --boot-check
+log "complete: https://$WAN_IP:3000, login spinifex"

scripts/appliance/getty-wait.conf

@@ -0,0 +1,4 @@
+# Holds the console login prompt until first boot and the banner have written
+# /etc/issue, so the unit's IP shows before anyone logs in.
+[Unit]
+After=spinifex-appliance-firstboot.service spinifex-banner.service

scripts/appliance/lib.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Helpers shared by the appliance first-boot script and its test. Sourced, not run.
+
+# appliance_physical_nics — physical Ethernet devices, one per line, in PCI
+# order. Skips anything without a backing device (bridges, veths, OVS ports,
+# loopback), non-Ethernet link types and wireless. SYSFS overrides /sys.
+appliance_physical_nics() {
+    local sys="${SYSFS:-/sys}/class/net" dev
+    for dev in "$sys"/*; do
+        [ -e "$dev/device" ] || continue
+        [ "$(cat "$dev/type" 2>/dev/null)" = 1 ] || continue
+        [ -d "$dev/wireless" ] && continue
+        printf '%s %s\n' "$(readlink -f "$dev/device")" "$(basename "$dev")"
+    done | sort | awk '{print $2}'
+}
+
+# appliance_pick_nic — the first physical NIC with carrier, else the first
+# physical NIC. Prints nothing when the host has none.
+appliance_pick_nic() {
+    local sys="${SYSFS:-/sys}/class/net" nic first=""
+    for nic in $(appliance_physical_nics); do
+        [ -n "$first" ] || first=$nic
+        if [ "$(cat "$sys/$nic/carrier" 2>/dev/null)" = 1 ]; then
+            echo "$nic"
+            return 0
+        fi
+    done
+    [ -n "$first" ] && echo "$first"
+    return 0
+}
+
+# appliance_hostname <mac> — spinifex-<last 6 hex digits of the MAC>.
+appliance_hostname() {
+    local hex
+    hex=$(echo "$1" | tr -d ':' | tr '[:upper:]' '[:lower:]')
+    echo "spinifex-${hex: -6}"
+}

scripts/appliance/lib_test.sh

@@ -0,0 +1,63 @@
+#!/bin/sh
+# lib_test.sh — tests for the appliance first-boot NIC and hostname helpers,
+# run against a fixture sysfs tree (SYSFS) so the host's own NICs never count.
+set -eu
+
+DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- appliance/lib.sh"
+
+# nic <name> <pci-slot> <carrier> [type] — a physical NIC behind a PCI device.
+nic() {
+    mkdir -p "$SYS/devices/pci0000:00/$2" "$SYS/class/net/$1"
+    ln -s "../../../devices/pci0000:00/$2" "$SYS/class/net/$1/device"
+    echo "${4:-1}" > "$SYS/class/net/$1/type"
+    echo "$3" > "$SYS/class/net/$1/carrier"
+}
+# virt <name> — a link with no backing device (bridge, veth, OVS port).
+virt() {
+    mkdir -p "$SYS/class/net/$1"
+    echo 1 > "$SYS/class/net/$1/type"
+    echo 1 > "$SYS/class/net/$1/carrier"
+}
+fixture() { SYS="$TMP/$1"; mkdir -p "$SYS/class/net"; }
+run() { SYSFS="$SYS" bash -c ". '$DIR/lib.sh'; $1"; }
+
+check() {
+    if [ "$2" = "$3" ]; then pass "$1"; else fail "$1: want '$2', got '$3'"; fi
+}
+
+fixture order
+nic enp3s0 0000:00:03.0 0
+nic enp1s0 0000:00:01.0 0
+virt br-wan
+virt lo
+check "physical NICs in PCI order, virtual links skipped" "enp1s0 enp3s0" "$(run appliance_physical_nics | tr '\n' ' ' | sed 's/ $//')"
+check "no carrier anywhere picks the first by PCI order" enp1s0 "$(run appliance_pick_nic)"
+
+fixture carrier
+nic eno1 0000:00:01.0 0
+nic eno2 0000:00:02.0 1
+nic eno3 0000:00:03.0 1
+check "the first NIC with carrier wins over PCI order" eno2 "$(run appliance_pick_nic)"
+
+fixture wireless
+nic wlp1s0 0000:00:01.0 1
+mkdir -p "$SYS/class/net/wlp1s0/wireless"
+nic ib0 0000:00:02.0 1 32
+nic enp4s0 0000:00:04.0 0
+check "wireless and non-Ethernet links are skipped" enp4s0 "$(run appliance_pick_nic)"
+
+fixture none
+virt br-wan
+check "no physical NIC prints nothing" "" "$(run appliance_pick_nic)"
+
+check "hostname from the MAC's last 6 hex digits" spinifex-c0ffee "$(run 'appliance_hostname 52:54:00:C0:FF:EE')"
+
+[ "$fails" -eq 0 ] || { echo "  $fails failed"; exit 1; }

scripts/appliance/prepare.sh

@@ -0,0 +1,178 @@
+#!/bin/bash
+# Runs as root on the Oracle Linux build system, one stage per call, driven by
+# build-oracle-appliance.sh from the payload directory it copied over.
+#   prepare.sh check      refuse anything but OL 10.2+ on UEK R8, x86_64
+#   prepare.sh update     dnf update, SELinux permissive; prints PREPARE_REBOOT=1 if
+#                         the newest kernel is not the running one
+#   prepare.sh install    install Spinifex, uninitialised and not enabled
+#   prepare.sh appliance  spinifex login, first-boot service, console placeholder
+#   prepare.sh seal       wipe this system's identity, then power off
+# INSTALL_SPINIFEX_CHANNEL, INSTALL_SPINIFEX_VERSION and INSTALL_SPINIFEX_TARBALL
+# pass through to the installer. BUILD_USER is the SSH user to lock at seal.
+set -euo pipefail
+
+# Shared by every unit built from this image; first login forces a change.
+DEFAULT_PASSWORD=spinifex
+
+HERE=$(cd "$(dirname "$0")" && pwd)
+SHARE=/usr/local/share/spinifex
+
+log() { echo "[prepare] $*"; }
+die() { echo "[prepare] ERROR: $*" >&2; exit 1; }
+
+[ "$(id -u)" = 0 ] || die "run as root"
+
+stage_check() {
+    # shellcheck source=/dev/null
+    . /etc/os-release
+    [ "${ID:-}" = ol ] || die "not Oracle Linux (ID=${ID:-unset})"
+    local major=${VERSION_ID%%.*} minor=${VERSION_ID#*.}
+    [ "$minor" = "$VERSION_ID" ] && minor=0
+    if [ "$major" -lt 10 ] || { [ "$major" -eq 10 ] && [ "$minor" -lt 2 ]; }; then
+        # A 10.0 or 10.1 system reaches 10.2 through the update stage, which
+        # calls this again with "strict" once it has run.
+        if [ "$major" -ne 10 ] || [ "${1:-}" = strict ]; then
+            die "Oracle Linux $VERSION_ID; 10.2 or later is required"
+        fi
+        log "Oracle Linux $VERSION_ID; the update stage brings it to the current minor"
+    fi
+    [ "$(uname -m)" = x86_64 ] || die "$(uname -m); only x86_64 is supported"
+    case $(uname -r) in
+        6.12.0-*uek*) ;;
+        *) die "running kernel $(uname -r) is not UEK R8; install kernel-uek and boot it first" ;;
+    esac
+    command -v nmcli >/dev/null || die "NetworkManager (nmcli) is required"
+    log "OK: $PRETTY_NAME, kernel $(uname -r)"
+}
+
+stage_update() {
+    setenforce 0 2>/dev/null || true
+    sed -i 's/^SELINUX=.*/SELINUX=permissive/' /etc/selinux/config
+    [ "$(getenforce)" != Enforcing ] || die "SELinux is still enforcing"
+    dnf update -y
+    stage_check strict >/dev/null
+    local newest
+    newest=$(grubby --default-kernel | sed 's|^/boot/vmlinuz-||')
+    case $newest in
+        6.12.0-*uek*) ;;
+        *) die "the default boot kernel is $newest, not UEK R8; set a kernel-uek default with grubby" ;;
+    esac
+    if [ "$newest" != "$(uname -r)" ]; then
+        echo PREPARE_REBOOT=1
+    fi
+}
+
+stage_install() {
+    [ ! -e /etc/spinifex/spinifex.toml ] || die "/etc/spinifex/spinifex.toml exists; this system has been initialised, start from a clean OL install"
+    local args=()
+    [ -n "${INSTALL_SPINIFEX_CHANNEL:-}" ] && args+=(--channel "$INSTALL_SPINIFEX_CHANNEL")
+    [ -n "${INSTALL_SPINIFEX_VERSION:-}" ] && args+=(--version "$INSTALL_SPINIFEX_VERSION")
+    if [ -n "${INSTALL_SPINIFEX_TARBALL:-}" ]; then
+        # A local build: the payload carries the setup.sh of the checkout the
+        # tarball was built from, and setup.sh reads the tarball from the env.
+        [ -f "$HERE/setup.sh" ] || die "no setup.sh in the payload for INSTALL_SPINIFEX_TARBALL"
+        INSTALL_SPINIFEX_SKIP_NEWGRP=1 bash "$HERE/setup.sh" "${args[@]}"
+    else
+        curl -fsSL -o /tmp/spinifex-setup.sh https://install.mulgadc.com
+        INSTALL_SPINIFEX_SKIP_NEWGRP=1 bash /tmp/spinifex-setup.sh "${args[@]}"
+        rm -f /tmp/spinifex-setup.sh
+    fi
+    # The installer enables these on a live host. The ISO image ships them off
+    # and first boot enables them after init; a clone that started them first
+    # would run every service with no config.
+    systemctl disable --now spinifex.target spinifex-nats-watchdog.timer
+    log "Spinifex $(spx version 2>/dev/null | head -1) installed, not initialised"
+}
+
+stage_appliance() {
+    # As on the ISO: the existing spinifex system group as primary, so the
+    # login can reach the sockets setup.sh group-owns, and passwordless sudo.
+    if ! id spinifex >/dev/null 2>&1; then
+        useradd --create-home --shell /bin/bash --gid spinifex --groups wheel spinifex
+    fi
+    echo "spinifex:$DEFAULT_PASSWORD" | chpasswd
+    chage -d 0 spinifex
+    echo 'spinifex ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/spinifex-admin
+    chmod 0440 /etc/sudoers.d/spinifex-admin
+    visudo -cf /etc/sudoers.d/spinifex-admin >/dev/null
+
+    # sshd takes the first value it reads, so this sorts ahead of the
+    # 50-cloud-init.conf that templates ship with passwords off.
+    printf 'PasswordAuthentication yes\n' > /etc/ssh/sshd_config.d/10-spinifex-appliance.conf
+
+    # No DHCP on every port: first boot puts the primary NIC on br-wan and
+    # leaves the rest alone, so no second port competes for the default route.
+    printf '[main]\nno-auto-default=*\n' > /etc/NetworkManager/conf.d/90-spinifex-appliance.conf
+
+    install -D -m 0644 "$HERE/lib.sh" "$SHARE/appliance/lib.sh"
+    install -D -m 0755 "$HERE/firstboot.sh" "$SHARE/appliance/firstboot.sh"
+    install -m 0644 "$HERE/spinifex-appliance-firstboot.service" "$HERE/spinifex-banner.service" /etc/systemd/system/
+    local getty
+    for getty in getty@tty1 serial-getty@ttyS0; do
+        install -D -m 0644 "$HERE/getty-wait.conf" "/etc/systemd/system/$getty.service.d/spinifex-wait.conf"
+    done
+    systemctl daemon-reload
+    systemctl enable spinifex-appliance-firstboot.service
+
+    # Cloud-init would re-run on each unit with no datasource and rewrite the
+    # network and users first boot owns.
+    if [ -d /etc/cloud ]; then
+        touch /etc/cloud/cloud-init.disabled
+    fi
+    log "spinifex login (default password '$DEFAULT_PASSWORD', change forced) and first-boot service installed"
+}
+
+stage_seal() {
+    dnf clean all
+
+    systemctl stop "$(. "$SHARE/lib/distro.sh" && distro_ovs_unit)" ovn-northd.service ovn-controller.service 2>/dev/null || true
+    # OVS writes the chassis UUID on first start; a shared one makes every unit
+    # the same chassis in OVN.
+    rm -f /etc/openvswitch/system-id.conf /etc/openvswitch/conf.db /etc/openvswitch/.conf.db.~lock~
+    rm -f /var/lib/ovn/*.db /etc/ovn/*.db
+    rm -f /etc/spinifex/node.conf
+
+    # Profiles for the build system's NICs, its DHCP leases and NM's
+    # per-machine secret. First boot writes br-wan for the unit's own NIC.
+    rm -f /etc/NetworkManager/system-connections/*
+    rm -f /var/lib/NetworkManager/*.lease /var/lib/NetworkManager/secret_key \
+        /var/lib/NetworkManager/timestamps /var/lib/NetworkManager/seen-bssids
+    if command -v cloud-init >/dev/null; then
+        cloud-init clean --logs || true
+    fi
+
+    # sshd-keygen regenerates host keys before sshd on the next boot; systemd
+    # generates a machine-id when the file is empty.
+    rm -f /etc/ssh/ssh_host_*
+    truncate -s 0 /etc/machine-id
+    rm -f /var/lib/dbus/machine-id /var/lib/systemd/random-seed
+    echo localhost > /etc/hostname
+
+    # The build user's access goes; the spinifex login is the way in.
+    if [ -n "${BUILD_USER:-}" ] && [ "$BUILD_USER" != spinifex ] && [ "$BUILD_USER" != root ]; then
+        local home
+        home=$(getent passwd "$BUILD_USER" | cut -d: -f6)
+        rm -f "$home/.ssh/authorized_keys" "$home/.bash_history"
+        passwd -l "$BUILD_USER" >/dev/null
+        rm -f "/etc/sudoers.d/90-cloud-init-users"
+    fi
+    rm -f /root/.ssh/authorized_keys /root/.bash_history /home/spinifex/.bash_history
+
+    printf '\nSpinifex first boot is in progress. This screen updates when it finishes.\n\n' > /etc/issue
+    sed -i '/^# --- Spinifex ---$/,$d' /etc/motd 2>/dev/null || true
+
+    find /var/log -type f -exec truncate -s 0 {} + 2>/dev/null || true
+    rm -rf /var/log/journal/* /tmp/* /var/tmp/*
+    sync
+    log "sealed; powering off"
+    systemctl poweroff --no-block
+}
+
+case "${1:-}" in
+    check) stage_check ;;
+    update) stage_update ;;
+    install) stage_install ;;
+    appliance) stage_appliance ;;
+    seal) stage_seal ;;
+    *) die "usage: $0 check|update|install|appliance|seal" ;;
+esac

scripts/appliance/spinifex-appliance-firstboot.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=Spinifex appliance first-boot provisioning
+After=NetworkManager.service network-online.target chronyd.service
+Wants=NetworkManager.service network-online.target
+ConditionPathExists=!/var/lib/spinifex/.appliance-firstboot-done
+
+[Service]
+Type=oneshot
+Environment=HOME=/root
+ExecStart=/usr/local/share/spinifex/appliance/firstboot.sh
+RemainAfterExit=yes
+# The banner is never enabled when first boot fails, so without this the
+# console would keep the "first boot in progress" placeholder.
+ExecStopPost=/bin/sh -c 'if [ "$EXIT_STATUS" != "0" ]; then printf "\n*** Spinifex first boot FAILED ***\nLog in as spinifex, then run:\n  sudo journalctl -u spinifex-appliance-firstboot.service\nCheck the network cable and DHCP, then reboot to retry.\n\n" > /etc/issue; fi'
+# setup-ovn.sh and spx admin init must not wedge multi-user.target and keep
+# getty from ever showing a login prompt.
+TimeoutStartSec=600s
+StandardOutput=journal+console
+StandardError=journal+console
+
+[Install]
+WantedBy=multi-user.target

scripts/appliance/spinifex-banner.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Spinifex console banner and boot health check
+After=network-online.target
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/bin/spx admin banner --boot-check
+RemainAfterExit=yes
+# Cap it so a stuck boot-check cannot hold getty behind the console drop-in.
+TimeoutStartSec=30s
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

scripts/build-oracle-appliance.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+# build-oracle-appliance.sh — turn an Oracle Linux 10.2 system reachable over
+# SSH with sudo into a sealed Spinifex gold image: installed, not initialised,
+# with a first-boot service that sets each cloned unit up as a single node.
+# The system powers off at the end; capture its disk as the image.
+#
+# Usage: build-oracle-appliance.sh [options] <user>@<host>
+#   --channel latest|dev   install channel (default: latest)
+#   --version <tag>        install a specific release
+#   --tarball <path>       install a local release tarball instead
+#
+# Ports, keys and jump hosts come from ~/.ssh/config. <user> needs
+# passwordless sudo.
+set -euo pipefail
+
+HERE=$(cd "$(dirname "$0")" && pwd)
+PAYLOAD_SRC="$HERE/appliance"
+REMOTE_DIR=/tmp/spinifex-appliance
+
+CHANNEL="" VERSION="" TARBALL="" TARGET=""
+
+usage() { sed -n '2,13p' "$0" | sed 's/^# \{0,1\}//'; exit "${1:-0}"; }
+log() { echo "==> $*"; }
+die() { echo "ERROR: $*" >&2; exit 1; }
+
+while [ $# -gt 0 ]; do
+    case "$1" in
+        --channel) CHANNEL="${2:-}"; shift ;;
+        --version) VERSION="${2:-}"; shift ;;
+        --tarball) TARBALL="${2:-}"; shift ;;
+        -h | --help) usage ;;
+        -*) die "unknown option: $1" ;;
+        *) [ -z "$TARGET" ] || die "one target only"; TARGET=$1 ;;
+    esac
+    shift
+done
+[ -n "$TARGET" ] || usage 1
+case $TARGET in *@*) ;; *) die "target must be <user>@<host>" ;; esac
+BUILD_USER=${TARGET%@*}
+[ -z "$TARBALL" ] || [ -f "$TARBALL" ] || die "no such tarball: $TARBALL"
+
+# One multiplexed connection; host key checking is relaxed because the build
+# system is usually a fresh install whose key nobody has recorded yet.
+CTL_DIR=$(mktemp -d)
+trap 'ssh -o ControlPath="$CTL_DIR/c" -O exit "$TARGET" >/dev/null 2>&1 || true; rm -rf "$CTL_DIR"' EXIT
+SSH_OPTS=(-o ControlMaster=auto -o ControlPath="$CTL_DIR/c" -o ControlPersist=60
+    -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 -o ConnectTimeout=10)
+
+rsh() { ssh "${SSH_OPTS[@]}" "$TARGET" "$@"; }
+
+# prepare <stage> — run one prepare.sh stage as root on the build system.
+prepare() {
+    rsh "sudo env INSTALL_SPINIFEX_CHANNEL='$CHANNEL' INSTALL_SPINIFEX_VERSION='$VERSION' \
+        INSTALL_SPINIFEX_TARBALL='${TARBALL:+$REMOTE_DIR/$(basename "$TARBALL")}' \
+        BUILD_USER='$BUILD_USER' bash $REMOTE_DIR/prepare.sh $1"
+}
+
+push_payload() {
+    rsh "rm -rf $REMOTE_DIR && mkdir -p $REMOTE_DIR"
+    tar -C "$PAYLOAD_SRC" -cf - . | rsh "tar -C $REMOTE_DIR -xf -"
+    if [ -n "$TARBALL" ]; then
+        # setup.sh from this checkout installs the tarball built from it.
+        scp "${SSH_OPTS[@]}" -q "$HERE/setup.sh" "$TARBALL" "$TARGET:$REMOTE_DIR/"
+    fi
+}
+
+wait_for_ssh() {
+    local _i
+    ssh "${SSH_OPTS[@]}" -O exit "$TARGET" >/dev/null 2>&1 || true
+    sleep 10
+    for _i in $(seq 1 60); do
+        if rsh true 2>/dev/null; then return 0; fi
+        sleep 5
+    done
+    die "$TARGET did not come back over SSH after the reboot"
+}
+
+rsh "sudo -n true" || die "$BUILD_USER needs passwordless sudo on the build system"
+
+log "Copying the payload to $TARGET"
+push_payload
+prepare check
+
+log "Updating Oracle Linux"
+prepare update | tee "$CTL_DIR/update.log"
+if grep -qx 'PREPARE_REBOOT=1' "$CTL_DIR/update.log"; then
+    log "Rebooting into the updated kernel"
+    rsh "sudo systemctl reboot" || true
+    wait_for_ssh
+    # /tmp is cleared on boot.
+    push_payload
+    prepare check
+fi
+
+log "Installing Spinifex (not initialising it)"
+prepare install
+
+log "Installing the spinifex login and the first-boot service"
+prepare appliance
+
+log "Sealing; the system powers off when done"
+prepare seal || true
+
+log "Done. Capture $TARGET's disk once it has powered off; each unit set up from it initialises itself on first boot."

scripts/install-node.sh

@@ -298,7 +298,6 @@ log "preflight on $N host(s)"
 
 versions=""
 families=""
-libreswan_hosts=""
 for i in $(seq 0 $((N - 1))); do
     host="${HOSTS[$i]}"
 
@@ -322,12 +321,6 @@ for i in $(seq 0 $((N - 1))); do
         fail "$host: cannot tell its OS from /etc/os-release"
     families+="${host}: ${family}"$'\n'
 
-    # libreswan's pluto: Spinifex cannot run OVN IPsec on it yet, and spx admin
-    # init refuses --ipsec=true there, which would stop formation part way.
-    if [ "$(out "$host" "test -x /usr/libexec/ipsec/pluto && echo yes")" = "yes" ]; then
-        libreswan_hosts+=" $host"
-    fi
-
     # A node that is already in a multi-node cluster is one whose keys and
     # logical network state are load-bearing. Refusing here is deliberate:
     # forming a cluster must never be the command that destroys a live one.
@@ -386,12 +379,6 @@ $(sed '/^$/d; s/^/         /' <<<"$families")
        A cluster must run one OS on every host."
 fi
 
-if $IPSEC_ENABLED && [ -n "$libreswan_hosts" ]; then
-    fail "--ipsec on, but these hosts use libreswan for IPsec:$libreswan_hosts
-       Spinifex cannot run OVN IPsec on libreswan yet. Re-run with --ipsec off
-       only if the nodes share a trusted private link."
-fi
-
 # --- Confirm ---------------------------------------------------------------
 
 echo ""

scripts/ipsec_state_helper_test.sh

@@ -0,0 +1,158 @@
+#!/bin/sh
+# ipsec_state_helper_test.sh — tests for the plaintext Geneve guard in the
+# IPsec state helper that setup.sh installs.
+#
+# The helper is cut out of setup.sh's heredoc and run against stub nft,
+# ovs-vsctl and systemctl commands, with pluto's path pointed at a fixture.
+set -eu
+
+SCRIPT_DIR=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)
+SETUP="$SCRIPT_DIR/setup.sh"
+TMP=$(mktemp -d)
+trap 'rm -rf "$TMP"' EXIT
+
+fails=0
+pass() { echo "  ok   - $1"; }
+fail() { echo "  FAIL - $1"; fails=$((fails + 1)); }
+
+echo "-- IPsec state helper: plaintext Geneve guard"
+
+HELPER="$TMP/helper"
+awk '/tee "\$\{IPSEC_STATE_HELPER\}"/ { on = 1; next } on && /^HELPER$/ { exit } on { print }' "$SETUP" |
+    sed "s#/usr/libexec/ipsec/pluto#$TMP/pluto#g" > "$HELPER"
+chmod +x "$HELPER"
+grep -q '^    guard)' "$HELPER" || { echo "could not extract the helper from setup.sh" >&2; exit 1; }
+
+mkdir -p "$TMP/bin"
+cat > "$TMP/bin/nft" <<EOF
+#!/bin/sh
+[ "\$*" = "-f -" ] || { echo "nft \$*" >> "$TMP/nft.log"; exit 0; }
+cat >> "$TMP/nft.log"
+echo "--" >> "$TMP/nft.log"
+EOF
+cat > "$TMP/bin/ovs-vsctl" <<EOF
+#!/bin/sh
+[ -e "$TMP/ovs-down" ] && { echo "ovs-vsctl: database connection failed" >&2; exit 1; }
+case "\$*" in
+    *" find Interface type=geneve") cat "$TMP/ifaces" ;;
+    *" get Interface "*)
+        for last; do :; done
+        f="$TMP/opt.\$5.\${last#options:}"
+        if [ -e "\$f" ]; then cat "\$f"; else echo; fi
+        ;;
+    *" get Open_vSwitch . external_ids:ovn-encap-ip")
+        if [ -e "$TMP/encap" ]; then cat "$TMP/encap"; else echo; fi
+        ;;
+    *) exit 1 ;;
+esac
+EOF
+printf '#!/bin/sh\nexit 0\n' > "$TMP/bin/systemctl"
+chmod +x "$TMP/bin/nft" "$TMP/bin/ovs-vsctl" "$TMP/bin/systemctl"
+
+# reset — no tunnels, OVS up, no pluto.
+reset() {
+    rm -f "$TMP"/opt.* "$TMP/ovs-down" "$TMP/nft.log" "$TMP/pluto" "$TMP/encap"
+    : > "$TMP/ifaces"
+    touch "$TMP/nft.log"
+}
+# tunnel <name> <local_ip|-> [remote_name] — a geneve interface as ovn-controller writes it.
+tunnel() {
+    echo "$1" >> "$TMP/ifaces"
+    [ "$2" = - ] || printf '"%s"\n' "$2" > "$TMP/opt.$1.local_ip"
+    [ -z "${3:-}" ] || echo "$3" > "$TMP/opt.$1.remote_name"
+}
+run() {
+    RC=0
+    OUT=$(PATH="$TMP/bin:$PATH" "$HELPER" "$1" 2>&1) || RC=$?
+    NFTLOG=$(cat "$TMP/nft.log")
+}
+LIFTED="table inet spinifex_ipsec_guard
+delete table inet spinifex_ipsec_guard
+--"
+# expect_guarded <desc> <sources> — one transaction that swaps in a table
+# dropping Geneve without an SA from exactly those sources.
+expect_guarded() {
+    if [ "$RC" = 0 ] &&
+        [ "$(grep -c '^--$' "$TMP/nft.log")" = 1 ] &&
+        [ "$(sed -n 2p "$TMP/nft.log")" = "delete table inet spinifex_ipsec_guard" ] &&
+        grep -q "^        elements = { $2 }$" "$TMP/nft.log" &&
+        grep -q '^        ip saddr @srcs udp dport 6081 rt ipsec missing drop$' "$TMP/nft.log"; then
+        pass "$1"
+    else
+        fail "$1 (rc=$RC): got [$NFTLOG] want sources [$2] $OUT"
+    fi
+}
+expect_lifted() {
+    if [ "$RC" = 0 ] && [ "$NFTLOG" = "$LIFTED" ]; then pass "$1"; else fail "$1 (rc=$RC): got [$NFTLOG] $OUT"; fi
+}
+
+reset
+tunnel ovn-node2-0 10.26.7.1 node2
+tunnel ovn-node3-0 10.26.7.1 node3
+run guard
+expect_guarded "guards the encap address once OVN wants the tunnels encrypted" "10.26.7.1/32"
+
+reset
+tunnel ovn-node2-0 10.26.7.1
+tunnel ovn-node3-0 10.26.7.1
+run guard
+expect_lifted "lifts the guard while OVN runs the tunnels in plaintext"
+
+reset
+tunnel ovn-node2-0 10.26.7.1 node2
+tunnel ovn-node3-0 10.26.8.1
+run guard
+expect_guarded "guards only the addresses of tunnels OVN encrypts" "10.26.7.1/32"
+
+reset
+tunnel ovn-node2-0 - node2
+run guard
+expect_guarded "guards a tunnel without a local_ip from every source" "0.0.0.0/0"
+
+reset
+echo '"10.26.7.1"' > "$TMP/encap"
+run guard
+expect_guarded "guards the encap address before ovn-controller has built the tunnels" "10.26.7.1/32"
+
+reset
+echo '"10.26.7.1,10.26.8.1"' > "$TMP/encap"
+run guard
+expect_guarded "guards each encap address of a multi-address host before the tunnels exist" "10.26.7.1/32,10.26.8.1/32"
+
+reset
+run guard
+expect_guarded "guards every source before the tunnels exist when no encap address is set" "0.0.0.0/0"
+
+reset
+tunnel ovn-node2-0 10.26.7.1 node2
+touch "$TMP/ovs-down"
+run guard
+if [ "$RC" != 0 ] && [ -z "$NFTLOG" ]; then
+    pass "keeps the guard and fails when OVS cannot be read"
+else
+    fail "with OVS down: rc=$RC nft calls [$NFTLOG]"
+fi
+
+reset
+touch "$TMP/pluto"
+chmod +x "$TMP/pluto"
+run off
+expect_lifted "off lifts the guard on a libreswan host"
+
+reset
+run off
+if [ "$RC" = 0 ] && [ -z "$NFTLOG" ]; then pass "off leaves nft alone without libreswan"; else fail "off without pluto: rc=$RC [$NFTLOG]"; fi
+
+reset
+run unguard
+expect_lifted "unguard lifts the guard"
+
+reset
+run bogus
+if [ "$RC" = 2 ] && [ -z "$NFTLOG" ]; then pass "refuses an unknown verb"; else fail "unknown verb: rc=$RC [$NFTLOG]"; fi
+
+if [ "$fails" -gt 0 ]; then
+    echo "$fails failure(s)"
+    exit 1
+fi
+echo "all passed"

scripts/node-reset.sh

@@ -273,6 +273,33 @@ if [ -n "$OVS_UNIT" ]; then
 fi
 run sudo rm -f /etc/openvswitch/system-id.conf
 
+# --- BEGIN ovs nss nicknames ---
+# ovs_nss_nicknames — the entries ovs-monitor-ipsec imported, read from
+# `certutil -L` on stdin: ovs_certkey_* is this node's certificate and key,
+# ovs_cert_* the CA and any peer certificates. Everything else is someone else's.
+ovs_nss_nicknames() {
+    awk '$1 ~ /^ovs_cert(key)?_/ {print $1}' | LC_ALL=C sort -u
+}
+# --- END ovs nss nicknames ---
+
+# A node rejoining another cluster must not keep trusting this one's CA. Only
+# the NSS entries go: other identities and /etc/ipsec.d/*.conf are left alone.
+if [ -x /usr/libexec/ipsec/pluto ] && command -v certutil >/dev/null 2>&1; then
+    log "removing Spinifex IPsec identities from libreswan's NSS database"
+    # A running ovs-monitor-ipsec imports them again from other_config.
+    run sudo systemctl stop openvswitch-ipsec.service 2>/dev/null || true
+    for nss in /var/lib/ipsec/nss /etc/ipsec.d; do
+        sudo test -e "$nss/cert9.db" || continue
+        for nick in $(sudo certutil -L -d "sql:$nss" 2>/dev/null | ovs_nss_nicknames); do
+            log "  deleting $nick from $nss"
+            delete=-D
+            case "$nick" in ovs_certkey_*) delete=-F ;; esac
+            run sudo certutil "$delete" -d "sql:$nss" -n "$nick" ||
+                log "  NOTE: could not delete $nick from $nss; remove it with certutil by hand"
+        done
+    done
+fi
+
 # Delete the OVN DBs outright — a caller re-running setup-ovn.sh gets fresh
 # empty ones. This is what clears stale chassis rows and port bindings, which
 # otherwise accumulate across resets and wedge ovn-controller in a commit loop.

scripts/node-reset_test.sh

@@ -1,7 +1,8 @@
 #!/bin/sh
 # node-reset_test.sh — tests the QEMU process match node-reset.sh uses to stop
 # guests before it deletes Viperblock state, against real processes, and the
-# instance and mounted-volume counts it and uninstall-spx.sh report before deleting.
+# instance and mounted-volume counts it and uninstall-spx.sh report before deleting,
+# and the libreswan NSS entries it deletes.
 #
 # Each fake guest is a copy of sleep named like a QEMU binary, so the kernel
 # gives it the same comm, including the 15-character truncation.
@@ -129,6 +130,32 @@ mkdir -p "$empty"
 expect "counts no instances without a state file" "$(count count_instances "$empty")" 0
 expect "counts no mounted volumes without a viperblock directory" "$(count count_mounted_volumes "$empty")" 0
 
+echo "-- node-reset.sh libreswan NSS nicknames"
+
+sed -n '/^# --- BEGIN ovs nss nicknames ---$/,/^# --- END ovs nss nicknames ---$/p' \
+    "$SCRIPT_DIR/node-reset.sh" > "$TMP/nss.sh"
+# `certutil -L` as it prints a database holding an operator's own identity and
+# CA beside what ovs-monitor-ipsec imported, one peer listed twice.
+cat > "$TMP/certutil-L" <<'EOF'
+
+Certificate Nickname                                         Trust Attributes
+                                                             SSL,S/MIME,JAR/XPI
+
+ovs_certkey_node1                                            u,u,u
+ovs_cert_cacert                                              CT,,
+ovs_cert_node2                                               P,P,P
+ovs_cert_node2                                               P,P,P
+site-vpn                                                     u,u,u
+corp-ca                                                      CT,C,C
+ovs_certificate_store                                        ,,
+EOF
+if [ ! -s "$TMP/nss.sh" ]; then
+    fail "no ovs nss nicknames block in node-reset.sh"
+else
+    got=$(bash -c '. "$1"; ovs_nss_nicknames < "$2"' _ "$TMP/nss.sh" "$TMP/certutil-L" | tr '\n' ' ')
+    expect "selects only ovs-monitor-ipsec's entries, once each" "$got" "ovs_cert_cacert ovs_cert_node2 ovs_certkey_node1 "
+fi
+
 if [ "$fails" -gt 0 ]; then
     echo "$fails failure(s)"
     exit 1

scripts/setup-ovn.sh

@@ -172,6 +172,8 @@ OVN_DBDIR="${OVN_DBDIR:-/var/lib/ovn}"
 # Seconds to wait for the database servers to exit, then for a SIGKILL to land.
 OVN_DB_STOP_WAIT="${OVN_DB_STOP_WAIT:-30}"
 OVN_DB_KILL_WAIT="${OVN_DB_KILL_WAIT:-10}"
+# Seconds to wait for a restarted clustered NB or SB member to rejoin its cluster.
+OVN_RAFT_REJOIN_WAIT="${OVN_RAFT_REJOIN_WAIT:-60}"
 # Records that --management set this node up. Nodes set up before it existed
 # carry only the legacy marker, OVN_CTL_OPTS in /etc/default/ovn-central.
 OVN_ROLE_FILE=/etc/spinifex/ovn-role
@@ -757,6 +759,42 @@ stop_ovn_databases() {
     exit 1
 }
 
+# db_clients — sets NBCTL/SBCTL. A clustered DB has no unix socket to fall back
+# on, and a follower refuses to answer at all unless leader-only is turned off;
+# writes through a follower are forwarded to the leader.
+db_clients() {
+    NBCTL=(ovn-nbctl)
+    SBCTL=(ovn-sbctl)
+    if [ -n "$DB_CLUSTER_LOCAL_ADDR" ]; then
+        NBCTL=(ovn-nbctl --db="tcp:$DB_CLUSTER_LOCAL_ADDR:6641" --no-leader-only)
+        SBCTL=(ovn-sbctl --db="tcp:$DB_CLUSTER_LOCAL_ADDR:6642" --no-leader-only)
+    fi
+}
+
+# set_client_connections — the loopback client listeners. A rerun's restart can
+# hand the leadership away, so this goes through NBCTL/SBCTL, not the socket.
+set_client_connections() {
+    sudo "${NBCTL[@]}" set-connection ptcp:6641:127.0.0.1
+    sudo "${SBCTL[@]}" set-connection ptcp:6642:127.0.0.1
+}
+
+# wait_for_raft_member <ctl> <schema> — waits until this server is a cluster
+# member that knows its leader, which a restarted member is once it rejoins.
+# Returns 1 on timeout.
+wait_for_raft_member() {
+    local ctl=$1 schema=$2 status i
+    for i in $(seq 1 "$OVN_RAFT_REJOIN_WAIT"); do
+        status=$(sudo ovs-appctl -t "$ctl" cluster/status "$schema" 2>/dev/null || true)
+        if grep -qx 'Status: cluster member' <<<"$status" &&
+            grep -q '^Leader: ' <<<"$status" && ! grep -qx 'Leader: unknown' <<<"$status"; then
+            return 0
+        fi
+        echo "  Waiting for $schema to rejoin its RAFT cluster... ($i/$OVN_RAFT_REJOIN_WAIT)"
+        sleep 1
+    done
+    return 1
+}
+
 # --- Step 2: Enable services ---
 # ovn-ctl consults the RAFT flags only when it creates a database, so any
 # existing DB file is served as it stands and the cluster flags are ignored.
@@ -996,16 +1034,9 @@ EOF
         fi
     fi
 
-    # A clustered DB has no unix socket to fall back on, and a follower refuses
-    # to answer at all unless leader-only is turned off. Without both of these
-    # the waits below can never succeed on a clustered node: they burn their
-    # full timeout, gate nothing, and the run still continues.
-    NBCTL=(ovn-nbctl)
-    SBCTL=(ovn-sbctl)
-    if [ -n "$DB_CLUSTER_LOCAL_ADDR" ]; then
-        NBCTL=(ovn-nbctl --db="tcp:$DB_CLUSTER_LOCAL_ADDR:6641" --no-leader-only)
-        SBCTL=(ovn-sbctl --db="tcp:$DB_CLUSTER_LOCAL_ADDR:6642" --no-leader-only)
-    fi
+    # Without these the waits below can never succeed on a clustered node:
+    # they burn their full timeout, gate nothing, and the run still continues.
+    db_clients
 
     # Wait for OVN NB DB socket to become available
     for i in $(seq 1 15); do
@@ -1067,29 +1098,34 @@ EOF
     # with no join step to absorb the window — attaching before the leader is
     # elected (and before northd writes the gateway logical flows) leaves it with
     # a partial datapath that never reconverges. Gate on SB reachable and, when
-    # clustered, an elected leader.
+    # clustered, both members back in their clusters under an elected leader.
     for i in $(seq 1 30); do
-        if sudo "${SBCTL[@]}" --timeout=2 show >/dev/null 2>&1; then
-            if [ -z "$DB_CLUSTER_LOCAL_ADDR" ]; then
-                break
-            fi
-            SB_LEADER=$(sudo ovs-appctl -t /var/run/ovn/ovnsb_db.ctl \
-                cluster/status OVN_Southbound 2>/dev/null \
-                | awk '/^Leader:/{print $2; exit}')
-            if [ -n "$SB_LEADER" ] && [ "$SB_LEADER" != "unknown" ]; then
-                break
-            fi
-        fi
-        echo "  Waiting for OVN SB DB leader... ($i/30)"
+        sudo "${SBCTL[@]}" --timeout=2 show >/dev/null 2>&1 && break
+        echo "  Waiting for OVN SB DB... ($i/30)"
         sleep 1
     done
 
+    # The restart above took both of this node's members down at once. Returning
+    # before they rejoin lets a rerun on the next database node take a second
+    # member of each cluster down, and one of three left is below quorum.
+    if [ -n "$DB_CLUSTER_LOCAL_ADDR" ]; then
+        for member in "/var/run/ovn/ovnnb_db.ctl OVN_Northbound" "/var/run/ovn/ovnsb_db.ctl OVN_Southbound"; do
+            # shellcheck disable=SC2086 # two words: the ctl socket and the schema
+            if ! wait_for_raft_member $member; then
+                echo "ERROR: this node's ${member#* } server is not back in its RAFT cluster after ${OVN_RAFT_REJOIN_WAIT}s." >&2
+                echo "       Restarting OVN central on another database node now would leave the cluster" >&2
+                echo "       without quorum. Check: sudo ovs-appctl -t ${member% *} cluster/status ${member#* }" >&2
+                exit 1
+            fi
+        done
+        echo "  RAFT: NB and SB members rejoined, leader elected"
+    fi
+
     # set-connection replicates through RAFT, so it only runs on the create
     # node. Scoped to loopback, not a wildcard ptcp:6641, because 127.0.0.1
     # is valid on every node; the differing LAN address is set above instead.
     if [ -z "$DB_CLUSTER_REMOTE_ADDR" ]; then
-        sudo ovn-nbctl set-connection ptcp:6641:127.0.0.1
-        sudo ovn-sbctl set-connection ptcp:6642:127.0.0.1
+        set_client_connections
         if [ -n "$LAN_ADDR" ]; then
             echo "  OVN NB DB listening on tcp:127.0.0.1:6641, tcp:$LAN_ADDR:6641"
             echo "  OVN SB DB listening on tcp:127.0.0.1:6642, tcp:$LAN_ADDR:6642"
@@ -1958,16 +1994,21 @@ echo "Step 10: Enabling OVN auto-start on boot..."
 sudo systemctl enable "$OVS_UNIT" 2>/dev/null || true
 sudo systemctl enable ovn-controller 2>/dev/null || true
 printf '  %-21s enabled on boot\n' "${OVS_UNIT%.service}:" ovn-controller:
-if distro_has_libreswan; then
-    # OVN IPsec does not run on libreswan yet, so it stays off here. Starting
-    # openvswitch-ipsec would also start pluto, which listens on UDP 500/4500.
+if distro_has_libreswan && [ "$(systemctl is-active openvswitch-ipsec.service 2>/dev/null)" = active ]; then
+    # The daemon turned IPsec on; stopping it here would drop live tunnels.
+    printf '  %-21s %s\n' openvswitch-ipsec: "active: IPsec is on, left to the daemon" ipsec: "active: IPsec is on, left to the daemon"
+elif distro_has_libreswan; then
+    # Off until the daemon asks: pluto listens on UDP 500/4500 once started.
+    # Enabled under the mask, so the daemon's unmask also brings both back on boot.
     IPSEC_UNITS=(ipsec.service)
     distro_unit_exists openvswitch-ipsec.service && IPSEC_UNITS+=(openvswitch-ipsec.service)
-    if ! sudo systemctl mask --now "${IPSEC_UNITS[@]}" >/dev/null; then
-        echo "ERROR: could not stop and mask ${IPSEC_UNITS[*]}; IPsec must stay off on libreswan" >&2
+    if ! sudo systemctl unmask "${IPSEC_UNITS[@]}" >/dev/null 2>&1 ||
+        ! sudo systemctl enable "${IPSEC_UNITS[@]}" >/dev/null 2>&1 ||
+        ! sudo systemctl mask --now "${IPSEC_UNITS[@]}" >/dev/null; then
+        echo "ERROR: could not enable, stop and mask ${IPSEC_UNITS[*]}; IPsec must stay off until the daemon turns it on" >&2
         exit 1
     fi
-    printf '  %-21s %s\n' openvswitch-ipsec: "masked: IPsec stays off on libreswan" ipsec: "masked: IPsec stays off on libreswan"
+    printf '  %-21s %s\n' openvswitch-ipsec: "masked: IPsec is off, left to the daemon" ipsec: "masked: IPsec is off, left to the daemon"
 elif [ "$(systemctl is-enabled openvswitch-ipsec.service 2>/dev/null)" = masked ]; then
     # The daemon masks it while IPsec is off and unmasks it to turn IPsec on.
     printf '  %-21s %s\n' openvswitch-ipsec: "masked: IPsec is off, left to the daemon"

scripts/setup-ovn_test.sh

@@ -454,6 +454,47 @@ case "$RC:$OUT" in
     *) fail "no recreate-db: standalone DB rc=$RC out=$OUT" ;;
 esac
 
+# --------------------------------------------------------------------------
+# A rerun returns only once both of this node's RAFT members have rejoined.
+# --------------------------------------------------------------------------
+# A rerun restarts the node's NB and SB members together, so returning early
+# lets the next database node's rerun take a second member of each down.
+awk '$0 == "wait_for_raft_member() {" { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >"$TMP/raft_fn.sh"
+mkdir -p "$TMP/raftbin"
+printf '#!/bin/sh\nexit 0\n' >"$TMP/raftbin/sleep"
+printf '#!/bin/sh\n[ -f "%s" ] || exit 1\ncat "%s"\n' "$TMP/cluster-status" "$TMP/cluster-status" >"$TMP/raftbin/ovs-appctl"
+chmod +x "$TMP/raftbin/sleep" "$TMP/raftbin/ovs-appctl"
+
+# raft_member <Status:> <Leader:> — runs wait_for_raft_member against a
+# cluster/status in that state, or none when Status is "absent"; sets RC.
+raft_member() {
+    rm -f "$TMP/cluster-status"
+    if [ "$1" != absent ]; then
+        printf 'a1b2\nName: OVN_Northbound\nStatus: %s\nRole: follower\nTerm: 4\nLeader: %s\nVote: c3d4\n\nServers:\n    a1b2 (a1b2 at tcp:10.0.0.1:6643) (self)\n    c3d4 (c3d4 at tcp:10.0.0.2:6643)\n' \
+            "$1" "$2" >"$TMP/cluster-status"
+    fi
+    RC=0
+    PATH="$TMP/raftbin:$PATH" bash -c 'sudo() { "$@"; }; OVN_RAFT_REJOIN_WAIT=2; . "$1"
+        wait_for_raft_member /var/run/ovn/ovnnb_db.ctl OVN_Northbound' _ "$TMP/raft_fn.sh" >/dev/null 2>&1 || RC=$?
+}
+
+if [ ! -s "$TMP/raft_fn.sh" ]; then
+    fail "raft: no wait_for_raft_member in setup-ovn.sh"
+else
+    raft_member "cluster member" c3d4
+    if [ "$RC" = 0 ]; then pass "raft: a member following a known leader has rejoined"; else fail "raft: follower with leader rc=$RC"; fi
+    raft_member "cluster member" self
+    if [ "$RC" = 0 ]; then pass "raft: a member that is the leader has rejoined"; else fail "raft: leader rc=$RC"; fi
+    raft_member "cluster member" unknown
+    if [ "$RC" != 0 ]; then pass "raft: a member that has not found its leader has not rejoined"; else fail "raft: unknown leader accepted"; fi
+    raft_member "disconnected from the cluster (election timeout)" c3d4
+    if [ "$RC" != 0 ]; then pass "raft: a disconnected member that remembers a leader has not rejoined"; else fail "raft: disconnected member accepted"; fi
+    raft_member "joining cluster" c3d4
+    if [ "$RC" != 0 ]; then pass "raft: a member still joining has not rejoined"; else fail "raft: joining member accepted"; fi
+    raft_member absent ""
+    if [ "$RC" != 0 ]; then pass "raft: a server that does not answer has not rejoined"; else fail "raft: unanswered cluster/status accepted"; fi
+fi
+
 rm -rf "$TMP/dbdir"
 mkdir -p "$TMP/dbdir"
 ensure_db true
@@ -478,6 +519,33 @@ else
 fi
 printf '#!/bin/sh\nexit 1\n' >"$TMP/bin/pgrep"
 
+# A rerun restarts this node's members, which can hand the leadership to a
+# peer, so the create node's set-connection must still land as a follower.
+for fn in db_clients set_client_connections; do
+    awk -v f="$fn() {" '$0 == f { on = 1 } on { print } on && $0 == "}" { exit }' "$SETUP" >>"$TMP/conn_fn.sh"
+done
+mkdir -p "$TMP/followerbin"
+for ctl in ovn-nbctl ovn-sbctl; do
+    cat >"$TMP/followerbin/$ctl" <<EOF
+#!/bin/sh
+case " \$* " in
+    *" --no-leader-only "*) echo "$ctl \$*" >>"$TMP/conn.log" ;;
+    *) echo "$ctl: unix:/var/run/ovn/db.sock: database connection failed ()" >&2; exit 1 ;;
+esac
+EOF
+    chmod +x "$TMP/followerbin/$ctl"
+done
+: >"$TMP/conn.log"
+RC=0
+PATH="$TMP/followerbin:$PATH" bash -c 'set -e; sudo() { "$@"; }; DB_CLUSTER_LOCAL_ADDR=10.0.0.1; . "$1"
+    db_clients; set_client_connections' _ "$TMP/conn_fn.sh" >/dev/null 2>&1 || RC=$?
+if [ "$RC" = 0 ] && [ "$(cat "$TMP/conn.log")" = "ovn-nbctl --db=tcp:10.0.0.1:6641 --no-leader-only set-connection ptcp:6641:127.0.0.1
+ovn-sbctl --db=tcp:10.0.0.1:6642 --no-leader-only set-connection ptcp:6642:127.0.0.1" ]; then
+    pass "connections: the create node sets the client listeners while a follower"
+else
+    fail "connections: as a follower rc=$RC calls [$(cat "$TMP/conn.log")]"
+fi
+
 if [ "$fails" -ne 0 ]; then
     echo "  setup-ovn.sh: $fails failure(s)"
     exit 1

scripts/setup.sh

@@ -390,10 +390,68 @@ install_ipsec_state_helper() {
 set -eu
 
 if [ "$#" -ne 1 ]; then
-    echo "usage: $0 <on|off>" >&2
+    echo "usage: $0 <on|off|guard|unguard>" >&2
     exit 2
 fi
 
+# libreswan drops a tunnel's xfrm policies while IKE fails, and pluto flushes
+# the whole xfrm table when it starts or stops, so Geneve from the encap address
+# is dropped in nft unless the route carries an SA.
+GUARD_TABLE="inet spinifex_ipsec_guard"
+
+unguard() {
+    printf 'table %s\ndelete table %s\n' "$GUARD_TABLE" "$GUARD_TABLE" | nft -f -
+}
+
+want=""
+want_src() {
+    case "$1" in
+        *.*.*.*) case "$1" in *[!0-9.]*) src=0.0.0.0/0 ;; *) src="$1/32" ;; esac ;;
+        *) src=0.0.0.0/0 ;;
+    esac
+    case " $want " in *" $src "*) ;; *) want="$want $src" ;; esac
+}
+
+# One source per local address of a tunnel with a remote_name, which
+# ovn-controller sets only while NB_Global.ipsec is on. A tunnel without a
+# usable local_ip is guarded from every source rather than left open.
+guard() {
+    ifaces=$(ovs-vsctl --timeout=5 --bare --columns=name find Interface type=geneve)
+    for iface in $ifaces; do
+        peer=$(ovs-vsctl --timeout=5 --if-exists get Interface "$iface" options:remote_name)
+        [ -n "$peer" ] || continue
+        want_src "$(ovs-vsctl --timeout=5 --if-exists get Interface "$iface" options:local_ip | tr -d '"')"
+    done
+    # At boot pluto is up before ovn-controller builds the tunnels, and its
+    # callers guard only while encryption is wanted, so guard the encap address
+    # until the tunnels say otherwise.
+    if [ -z "$ifaces" ]; then
+        encap=$(ovs-vsctl --timeout=5 --if-exists get Open_vSwitch . external_ids:ovn-encap-ip | tr -d '"')
+        for ip in $(echo "$encap" | tr ',' ' '); do want_src "$ip"; done
+        [ -n "$want" ] || want_src ""
+    fi
+    if [ -z "$want" ]; then
+        unguard
+        return
+    fi
+    # One transaction, so the old set is never gone before the new one is in.
+    nft -f - << NFT
+table $GUARD_TABLE
+delete table $GUARD_TABLE
+table $GUARD_TABLE {
+    set srcs {
+        type ipv4_addr
+        flags interval
+        elements = { $(echo $want | tr ' ' ',') }
+    }
+    chain out {
+        type filter hook output priority 0; policy accept;
+        ip saddr @srcs udp dport 6081 rt ipsec missing drop
+    }
+}
+NFT
+}
+
 # mask/unmask rather than disable/enable. The daemon's unit sets
 # ProtectSystem=full, so /etc is read-only in the namespace its children inherit,
 # and disabling a unit that also ships a SysV script shells out to update-rc.d
@@ -401,12 +459,24 @@ fi
 
 # ovs-monitor-ipsec execs the strongSwan starter itself rather than going through
 # this unit, so leaving it enabled only contends for UDP 500/4500. Off in both
-# states. Absent on a host without strongswan, which is not an error.
-systemctl mask --now strongswan-starter.service >/dev/null 2>&1 || true
+# states. Masking a unit no package ships leaves a stray /dev/null link, so a
+# host without strongSwan gets none, and loses one an older helper left.
+if [ -e /usr/lib/systemd/system/strongswan-starter.service ] || [ -e /lib/systemd/system/strongswan-starter.service ]; then
+    systemctl mask --now strongswan-starter.service >/dev/null 2>&1 || true
+elif [ "$(systemctl is-enabled strongswan-starter.service 2>/dev/null)" = masked ]; then
+    systemctl unmask strongswan-starter.service >/dev/null 2>&1 || true
+fi
 
 case "$1" in
     on)
-        systemctl unmask openvswitch-ipsec.service
+        if [ -x /usr/libexec/ipsec/pluto ]; then
+            # ovs-monitor-ipsec drives pluto over its control socket but does
+            # not start it, so pluto comes up first under its own unit.
+            systemctl unmask ipsec.service openvswitch-ipsec.service
+            systemctl start ipsec.service
+        else
+            systemctl unmask openvswitch-ipsec.service
+        fi
         exec systemctl start openvswitch-ipsec.service
         ;;
     off)
@@ -415,8 +485,15 @@ case "$1" in
         # On Debian that name is strongswan-starter's alias, masked above.
         if [ -x /usr/libexec/ipsec/pluto ]; then
             systemctl mask --now ipsec.service
+            unguard
         fi
         ;;
+    guard)
+        guard
+        ;;
+    unguard)
+        unguard
+        ;;
     *)
         echo "state not permitted: $1" >&2
         exit 2
@@ -1769,6 +1846,18 @@ CPUWeight=300
 EOF
     info "  /etc/systemd/system/system.slice.d/spinifex-reserve.conf"
 
+    # The plaintext Geneve guard is not persisted, so it goes in when pluto
+    # starts at boot, before ovn-controller builds the tunnels. `-`: the guard
+    # must never fail the unit, and the daemon re-asserts it each pass.
+    if [ -x /usr/libexec/ipsec/pluto ]; then
+        $SUDO mkdir -p /etc/systemd/system/ipsec.service.d
+        $SUDO tee /etc/systemd/system/ipsec.service.d/spinifex-guard.conf > /dev/null << EOF
+[Service]
+ExecStartPost=-${IPSEC_STATE_HELPER} guard
+EOF
+        info "  /etc/systemd/system/ipsec.service.d/spinifex-guard.conf"
+    fi
+
     # daemon-reload / enable require a running systemd — skip inside the ISO
     # chroot. Unit files are still dropped into place; firstboot enables them.
     if [ "${ISO_BUILD:-0}" = "1" ]; then

scripts/uninstall-spx.sh

@@ -362,9 +362,10 @@ rm_path /etc/systemd/system/openvswitch-switch.service.d/spinifex-perms.conf \
         /etc/systemd/system/ovn-controller.service.d/log-level.conf \
         /etc/systemd/system/openvswitch-ipsec.service.d/spinifex-perms.conf \
         /etc/systemd/system/ovn-northd.service.d/spinifex-perms.conf \
-        /etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf
+        /etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf \
+        /etc/systemd/system/ipsec.service.d/spinifex-guard.conf
 # Only if we emptied them — another package may own the same override dir.
-for d in openvswitch-switch ovs-vswitchd ovn-controller openvswitch-ipsec ovn-northd; do
+for d in openvswitch-switch ovs-vswitchd ovn-controller openvswitch-ipsec ovn-northd ipsec; do
     dir="/etc/systemd/system/${d}.service.d"
     if sudo test -d "$dir" && [ -z "$(sudo ls -A "$dir" 2>/dev/null)" ]; then
         rm_path "$dir"
@@ -431,6 +432,11 @@ if [ -z "$PLUGINDIR" ]; then
 fi
 [ -n "$PLUGINDIR" ] && rm_path "$PLUGINDIR/nbdkit-viperblock-plugin.so"
 
+# A leftover guard table drops all Geneve from this host's encap address until
+# reboot, and the helper is the one place that names it.
+if [ -x /usr/local/lib/spinifex/spinifex-set-ipsec-state ]; then
+    run sudo /usr/local/lib/spinifex/spinifex-set-ipsec-state unguard || true
+fi
 rm_path /usr/local/bin/spx \
         /usr/local/share/spinifex \
         /usr/local/lib/spinifex \
@@ -619,6 +625,7 @@ if ! $DRY_RUN; then
              /etc/NetworkManager/conf.d/*spinifex* \
              /etc/systemd/system/*.service.d/spinifex-perms.conf \
              /etc/systemd/system/ovn-northd.service.d/no-manage-ovsdb.conf \
+             /etc/systemd/system/ipsec.service.d/spinifex-guard.conf \
              /etc/systemd/system/ovn-controller.service.d/log-level.conf; do
         exists "$p" && LEFT+=("$p")
     done

spinifex/domains/network/host/ipsec_cert.go

@@ -48,7 +48,7 @@ func IPSecCertPaths(configDir string) (certPath, keyPath string) {
 // GenerateIPSecPeerCert issues an IKEv2 peer certificate for this node signed
 // by the existing cluster CA. The cert carries the IPSec-IKE EKU so strongSwan
 // accepts it. The dnsName SAN must equal the OVS chassis-id (system-id).
-func GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, hostname, nodeIP string) error {
+func GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, hostname, nodeIP string, libreswan bool) error {
 	if hostname == "" {
 		return fmt.Errorf("ipsec peer cert: hostname required")
 	}
@@ -115,8 +115,11 @@ func GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, hostname, nodeIP st
 		return fmt.Errorf("ipsec peer cert: write key: %w", err)
 	}
 
-	// strongSwan 6.0 only loads CAs placed in /etc/ipsec.d/cacerts at startup;
-	// per-conn cacert= references are insufficient. Install the cluster CA there.
+	// strongSwan 6.0 only loads CAs placed in /etc/ipsec.d/cacerts at startup.
+	// On libreswan ovs-monitor-ipsec imports the CA into NSS from other_config.
+	if libreswan {
+		return nil
+	}
 	if err := installCAIntoCharonTrustStore(caCertPath); err != nil {
 		return fmt.Errorf("ipsec peer cert: install CA into charon trust store: %w", err)
 	}

spinifex/domains/network/host/ipsec_cert_test.go

@@ -68,7 +68,7 @@ func TestGenerateIPSecPeerCert(t *testing.T) {
 	t.Run("happy path", func(t *testing.T) {
 		t.Parallel()
 		dir := t.TempDir()
-		require.NoError(t, GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-1", "10.0.0.5"))
+		require.NoError(t, GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-1", "10.0.0.5", false))
 
 		certPath, keyPath := IPSecCertPaths(dir)
 		certPEM, err := os.ReadFile(certPath)
@@ -109,8 +109,8 @@ func TestGenerateIPSecPeerCert(t *testing.T) {
 	t.Run("idempotent overwrite", func(t *testing.T) {
 		t.Parallel()
 		dir := t.TempDir()
-		require.NoError(t, GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-2", "10.0.0.6"))
-		require.NoError(t, GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-2", "10.0.0.7"))
+		require.NoError(t, GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-2", "10.0.0.6", false))
+		require.NoError(t, GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-2", "10.0.0.7", false))
 
 		certPath, _ := IPSecCertPaths(dir)
 		certPEM, _ := os.ReadFile(certPath)
@@ -122,7 +122,7 @@ func TestGenerateIPSecPeerCert(t *testing.T) {
 	t.Run("rejects missing hostname", func(t *testing.T) {
 		t.Parallel()
 		dir := t.TempDir()
-		err := GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "", "10.0.0.5")
+		err := GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "", "10.0.0.5", false)
 		require.Error(t, err)
 		assert.Contains(t, err.Error(), "hostname required")
 	})
@@ -130,7 +130,7 @@ func TestGenerateIPSecPeerCert(t *testing.T) {
 	t.Run("rejects missing nodeIP", func(t *testing.T) {
 		t.Parallel()
 		dir := t.TempDir()
-		err := GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-1", "")
+		err := GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-1", "", false)
 		require.Error(t, err)
 		assert.Contains(t, err.Error(), "nodeIP required")
 	})
@@ -138,7 +138,7 @@ func TestGenerateIPSecPeerCert(t *testing.T) {
 	t.Run("rejects invalid nodeIP", func(t *testing.T) {
 		t.Parallel()
 		dir := t.TempDir()
-		err := GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-1", "not-an-ip")
+		err := GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-1", "not-an-ip", false)
 		require.Error(t, err)
 		assert.Contains(t, err.Error(), "invalid nodeIP")
 	})
@@ -146,7 +146,7 @@ func TestGenerateIPSecPeerCert(t *testing.T) {
 	t.Run("missing CA file", func(t *testing.T) {
 		t.Parallel()
 		dir := t.TempDir()
-		err := GenerateIPSecPeerCert(dir, filepath.Join(dir, "ghost-ca.pem"), caKeyPath, "node-1", "10.0.0.5")
+		err := GenerateIPSecPeerCert(dir, filepath.Join(dir, "ghost-ca.pem"), caKeyPath, "node-1", "10.0.0.5", false)
 		require.Error(t, err)
 		assert.Contains(t, err.Error(), "read CA cert")
 	})
@@ -213,3 +213,31 @@ func TestInstallCAIntoCharonTrustStore_SwallowsRereadError(t *testing.T) {
 type stubError string
 
 func (e stubError) Error() string { return string(e) }
+
+// ovs-monitor-ipsec imports the CA into libreswan's NSS database itself, so
+// strongSwan's cacerts directory and rereadcacerts stay untouched there.
+func TestGenerateIPSecPeerCert_LibreswanLeavesCharonTrustStoreAlone(t *testing.T) {
+	origTrustDir := charonCATrustDir
+	charonCATrustDir = filepath.Join(t.TempDir(), "cacerts")
+	t.Cleanup(func() { charonCATrustDir = origTrustDir })
+
+	origReread := charonRereadCAs
+	charonRereadCAs = func() error {
+		t.Error("ipsec rereadcacerts run on a libreswan host")
+		return nil
+	}
+	t.Cleanup(func() { charonRereadCAs = origReread })
+
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	writeTestCA(t, caCertPath, caKeyPath)
+
+	dir := t.TempDir()
+	require.NoError(t, GenerateIPSecPeerCert(dir, caCertPath, caKeyPath, "node-1", "10.0.0.5", true))
+
+	certPath, keyPath := IPSecCertPaths(dir)
+	assert.FileExists(t, certPath)
+	assert.FileExists(t, keyPath)
+	assert.NoDirExists(t, charonCATrustDir, "strongSwan trust store created on a libreswan host")
+}

spinifex/domains/network/host/ipsec_enable.go

@@ -68,16 +68,17 @@ type IPSecBarrier interface {
 }
 
 const (
-	// ovsIPSecUnit owns charon: ovs-monitor-ipsec execs the strongSwan starter
-	// itself rather than going through the starter's own unit.
+	// ovsIPSecUnit owns charon on strongSwan: ovs-monitor-ipsec execs the
+	// starter itself rather than going through the starter's own unit.
 	ovsIPSecUnit = "openvswitch-ipsec.service"
 
 	// strongswanStarterUnit would start a second charon competing for UDP
 	// 500/4500. Nothing here needs it, so it stays off in both states.
 	strongswanStarterUnit = "strongswan-starter.service"
 
-	// libreswanUnit runs pluto, which listens on UDP 500/4500 once started. On
-	// Debian the same name is an alias of strongswanStarterUnit.
+	// libreswanUnit runs pluto, which ovs-monitor-ipsec drives but does not
+	// start, so it is on exactly while IPsec is. On Debian the same name is an
+	// alias of strongswanStarterUnit.
 	libreswanUnit = "ipsec.service"
 )
 
@@ -94,17 +95,15 @@ var ipsecStateHelper = "/usr/local/lib/spinifex/spinifex-set-ipsec-state"
 // unconfigured until something restarts the daemon — while another node may
 // already have made encryption mandatory cluster-wide.
 //
-// libreswan reports that the host's IPsec stack is libreswan, which OVN IPsec
-// does not run on yet, so such a host keeps IPsec off whatever the cluster asks.
+// libreswan reports that the host's IPsec stack is libreswan, whose pluto runs
+// under its own unit rather than being execed by ovs-monitor-ipsec.
 func MaintainIPSec(ctx context.Context, configPath string, clusterConfig *config.ClusterConfig, barrier IPSecBarrier, libreswan bool) {
 	// Said once, because a pass that reconciles to nothing logs nothing, which in
 	// a journal is indistinguishable from a node that armed.
 	switch {
 	case clusterConfig == nil:
 		slog.Info("ipsec: cluster membership unknown, leaving the host's IPsec services alone")
-	case ipsecRequested(clusterConfig) && libreswan:
-		slog.Error("ipsec: the cluster requires IPsec, but this host's IPsec stack is libreswan, which Spinifex cannot run OVN IPsec on yet; IPsec stays off here")
-	case !ipsecWanted(clusterConfig, libreswan):
+	case !ipsecRequested(clusterConfig):
 		slog.Info("ipsec: not in use on this cluster, IKE and NAT-T listeners stay stopped")
 	}
 
@@ -119,7 +118,7 @@ func MaintainIPSec(ctx context.Context, configPath string, clusterConfig *config
 			// Only the first success is news. The steady state re-runs every
 			// interval for the lifetime of the daemon, and saying so each time
 			// buries the passes that changed something.
-			if !announced && ipsecWanted(clusterConfig, libreswan) {
+			if !announced && ipsecRequested(clusterConfig) {
 				slog.Info("ipsec: local IPsec configuration complete on this node")
 				announced = true
 			}
@@ -134,12 +133,6 @@ func MaintainIPSec(ctx context.Context, configPath string, clusterConfig *config
 	}
 }
 
-// ipsecWanted reports whether this host turns IPsec on: the cluster asks for
-// it and the host's IPsec stack can run it.
-func ipsecWanted(clusterConfig *config.ClusterConfig, libreswan bool) bool {
-	return ipsecRequested(clusterConfig) && !libreswan
-}
-
 // ipsecRequested reports the cluster's intent. Single-node clusters have no Geneve
 // tunnels to protect, so charon must not be left listening on 500/4500 there
 // even though ipsec_enabled defaults to true.
@@ -157,7 +150,7 @@ func ReconcileOVNIPSec(ctx context.Context, configPath string, clusterConfig *co
 		return nil
 	}
 
-	want := ipsecWanted(clusterConfig, libreswan)
+	want := ipsecRequested(clusterConfig)
 
 	if err := ensureIPSecServices(want, libreswan); err != nil {
 		return err
@@ -165,7 +158,22 @@ func ReconcileOVNIPSec(ctx context.Context, configPath string, clusterConfig *co
 	if !want {
 		return DisableOVNIPSec(ctx, clusterConfig, barrier)
 	}
-	return EnableOVNIPSec(ctx, configPath, clusterConfig, barrier)
+	err := EnableOVNIPSec(ctx, configPath, clusterConfig, barrier)
+	if libreswan {
+		err = errors.Join(err, guardPlaintextGeneve())
+	}
+	return err
+}
+
+// guardPlaintextGeneve re-syncs the helper's nft drop on Geneve, which keeps a
+// tunnel whose SA is down from falling back to plaintext. Every pass, because
+// the tunnel set follows cluster membership and NB_Global.ipsec.
+func guardPlaintextGeneve() error {
+	out, err := hostcommand.SudoCommand(ipsecStateHelper, "guard").CombinedOutput()
+	if err != nil {
+		return fmt.Errorf("guard plaintext geneve: %w: %s", err, strings.TrimSpace(string(out)))
+	}
+	return nil
 }
 
 // DisableOVNIPSec is the off direction, and it has to reach NB_Global. Charon is
@@ -221,14 +229,21 @@ func ensureIPSecServices(want, libreswan bool) error {
 	return nil
 }
 
+// ipsecServicesMatch reads only the units the host's stack has: on libreswan
+// pluto runs under its own unit and follows the wanted state with
+// openvswitch-ipsec, while on Debian ipsec.service is strongSwan's alias.
 func ipsecServicesMatch(want, libreswan bool) bool {
 	if unitIsEnabled(strongswanStarterUnit) || unitIsActive(strongswanStarterUnit) {
 		return false
 	}
-	if !want && libreswan && (unitIsEnabled(libreswanUnit) || unitIsActive(libreswanUnit)) {
+	if libreswan && !unitInState(libreswanUnit, want) {
 		return false
 	}
-	return unitIsEnabled(ovsIPSecUnit) == want && unitIsActive(ovsIPSecUnit) == want
+	return unitInState(ovsIPSecUnit, want)
+}
+
+func unitInState(unit string, want bool) bool {
+	return unitIsEnabled(unit) == want && unitIsActive(unit) == want
 }
 
 // unitIsEnabled treats every state other than enabled as not enabled, which

spinifex/domains/network/host/ipsec_enable_test.go

@@ -43,6 +43,8 @@ type recordingSudo struct {
 	// change on the socket, a missing binary, a timed-out transaction. Each of
 	// those is a real fault.
 	nbError string
+	// guardError makes the helper's guard verb fail with this message.
+	guardError string
 }
 
 func (r *recordingSudo) stub(name string, args ...string) *exec.Cmd {
@@ -60,6 +62,9 @@ func (r *recordingSudo) stub(name string, args ...string) *exec.Cmd {
 		}
 		return exec.Command("true")
 	}
+	if name == ipsecStateHelper && slices.Equal(args, []string{"guard"}) && r.guardError != "" {
+		return exec.Command("sh", "-c", `echo "`+r.guardError+`" >&2; exit 1`)
+	}
 	if name != "systemctl" || len(args) < 2 {
 		return exec.Command("true")
 	}
@@ -497,22 +502,92 @@ func TestReconcileOVNIPSec_LibreswanAlreadyOffIsNoOp(t *testing.T) {
 	assert.Empty(t, recorder.helperRuns())
 }
 
-// OVN IPsec does not run on libreswan yet, so a cluster that asks for it gets
-// the off state on such a host rather than a half-armed one.
-func TestReconcileOVNIPSec_LibreswanStaysOffWhenTheClusterAsksForIPsec(t *testing.T) {
+// ipsecOnLibreswanHost is a libreswan host already in the on state, with
+// strongSwan absent.
+func ipsecOnLibreswanHost() *recordingSudo {
+	return &recordingSudo{
+		enabledOutput: map[string]string{
+			strongswanStarterUnit: "Failed to get unit file state for strongswan-starter.service: No such file or directory\n",
+			ovsIPSecUnit:          "enabled\n",
+			libreswanUnit:         "enabled\n",
+		},
+		activePerUnit: map[string]string{
+			strongswanStarterUnit: "inactive\n",
+			ovsIPSecUnit:          "active\n",
+			libreswanUnit:         "active\n",
+		},
+		nbUnreachable: true,
+	}
+}
+
+func TestReconcileOVNIPSec_LibreswanTurnsIPsecOnWhenTheClusterAsks(t *testing.T) {
 	recorder := ipsecOffHost()
-	recorder.activePerUnit[libreswanUnit] = "active\n"
+	recorder.nbUnreachable = true
+	// The probe after the helper ran; the static stub cannot change state mid-pass.
+	recorder.activePerUnit[ovsIPSecUnit] = "active\n"
 	t.Cleanup(hostcommand.SetSudoCommandForTest(recorder.stub))
 
 	configPath := filepath.Join(ipsecTestConfigDir(t), "spinifex.toml")
 	require.NoError(t, ReconcileOVNIPSec(t.Context(), configPath, multiNodeIPSecConfig(true), allReady(), true))
 
-	assert.Equal(t, [][]string{{ipsecStateHelper, "off"}}, recorder.helperRuns())
-	for _, run := range recorder.runs {
-		assert.NotEqual(t, "ovs-vsctl", run[0], "must not configure OVS IPsec on libreswan: %v", run)
+	assert.Equal(t, [][]string{{ipsecStateHelper, "on"}, {ipsecStateHelper, "guard"}}, recorder.helperRuns())
+	assert.Contains(t, strings.Join(recorder.ovsSets(), "\n"), "ipsec_encapsulation=true")
+}
+
+// ovs-monitor-ipsec does not start pluto on EL, so openvswitch-ipsec running
+// alone is not the on state.
+func TestReconcileOVNIPSec_LibreswanPlutoStoppedIsTurnedOn(t *testing.T) {
+	for _, tc := range []struct{ name, enabled, active string }{
+		{name: "inactive", enabled: "enabled\n", active: "inactive\n"},
+		{name: "masked", enabled: "masked\n", active: "inactive\n"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			recorder := ipsecOnLibreswanHost()
+			recorder.enabledOutput[libreswanUnit] = tc.enabled
+			recorder.activePerUnit[libreswanUnit] = tc.active
+			t.Cleanup(hostcommand.SetSudoCommandForTest(recorder.stub))
+
+			configPath := filepath.Join(ipsecTestConfigDir(t), "spinifex.toml")
+			require.NoError(t, ReconcileOVNIPSec(t.Context(), configPath, multiNodeIPSecConfig(true), allReady(), true))
+			assert.Equal(t, [][]string{{ipsecStateHelper, "on"}, {ipsecStateHelper, "guard"}}, recorder.helperRuns())
+		})
 	}
 }
 
+// The units are left alone in the steady state. The guard is not: a peer that
+// joined since the last pass has a tunnel the block does not cover yet.
+func TestReconcileOVNIPSec_LibreswanAlreadyOnOnlyResyncsTheGuard(t *testing.T) {
+	recorder := ipsecOnLibreswanHost()
+	t.Cleanup(hostcommand.SetSudoCommandForTest(recorder.stub))
+
+	configPath := filepath.Join(ipsecTestConfigDir(t), "spinifex.toml")
+	require.NoError(t, ReconcileOVNIPSec(t.Context(), configPath, multiNodeIPSecConfig(true), allReady(), true))
+	assert.Equal(t, [][]string{{ipsecStateHelper, "guard"}}, recorder.helperRuns())
+}
+
+// A guard that did not go in leaves plaintext Geneve possible, so the pass must
+// fail and retry rather than report IPsec configured.
+func TestReconcileOVNIPSec_LibreswanGuardFailureFailsThePass(t *testing.T) {
+	recorder := ipsecOnLibreswanHost()
+	recorder.guardError = "ovs-vsctl: unix:/var/run/openvswitch/db.sock: database connection failed"
+	t.Cleanup(hostcommand.SetSudoCommandForTest(recorder.stub))
+
+	configPath := filepath.Join(ipsecTestConfigDir(t), "spinifex.toml")
+	err := ReconcileOVNIPSec(t.Context(), configPath, multiNodeIPSecConfig(true), allReady(), true)
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "database connection failed")
+}
+
+// The block outlives pluto, so a host that turned IPsec off must not keep
+// re-adding it; the helper's off verb is what removes it.
+func TestReconcileOVNIPSec_LibreswanOffNeverGuards(t *testing.T) {
+	recorder := ipsecOnLibreswanHost()
+	t.Cleanup(hostcommand.SetSudoCommandForTest(recorder.stub))
+
+	require.NoError(t, ReconcileOVNIPSec(t.Context(), "/etc/spinifex/spinifex.toml", multiNodeIPSecConfig(false), allReady(), true))
+	assert.Equal(t, [][]string{{ipsecStateHelper, "off"}}, recorder.helperRuns())
+}
+
 // On Debian ipsec.service is an alias of strongswan-starter, which is already
 // checked under its own name; reading the alias again must not change the answer.
 func TestReconcileOVNIPSec_WithoutLibreswanIPsecServiceIsNotRead(t *testing.T) {

spinifex/operator/cli/admin.go

@@ -411,7 +411,7 @@ func init() {
 	adminInitCmd.Flags().StringSlice("external-dns", nil, "Comma-separated IPv4 DNS servers handed to guests on the external pool, at most 3 (auto-detected from the host resolver when omitted)")
 	adminInitCmd.Flags().Int("external-prefix-len", 24, "External pool subnet prefix length (auto-detected)")
 	adminInitCmd.Flags().Bool("gpu-passthrough", false, "Enable VFIO GPU passthrough (sets gpu_passthrough = true in daemon config)")
-	adminInitCmd.Flags().Bool("ipsec", true, "Encrypt intra-AZ Geneve via OVN native IPsec (cluster-wide); disable only for trusted single-rack lab. Defaults to false, and true is refused, on a host whose IPsec stack is libreswan")
+	adminInitCmd.Flags().Bool("ipsec", true, "Encrypt intra-AZ Geneve via OVN native IPsec (cluster-wide); disable only for trusted single-rack lab")
 	adminInitCmd.Flags().Bool("skip-host-dns", false, "Do not point this node's host resolver at its local northstar (LB/EKS names then won't resolve from the node)")
 
 	// Flags for admin join
@@ -1274,20 +1274,6 @@ func validateNodeCount(nodes int) error {
 	return nil
 }
 
-// resolveIPSecForHost applies the host's IPsec stack to the --ipsec choice. On
-// libreswan IPsec defaults to off and an explicit request for it is refused, so
-// nobody builds a cluster believing its underlay is encrypted.
-func resolveIPSecForHost(requested, explicit, libreswan bool) (bool, error) {
-	if !libreswan {
-		return requested, nil
-	}
-	if explicit && requested {
-		return false, fmt.Errorf("true is refused: this host's IPsec stack is libreswan (%s), and Spinifex cannot run OVN IPsec on libreswan yet; run without --ipsec, which defaults to false here",
-			hostos.LibreswanPlutoPath)
-	}
-	return false, nil
-}
-
 // TODO: Move all logic to a module, use minimal application logic in viper commands.
 func runAdminInit(cmd *cobra.Command, args []string) {
 	if os.Getuid() != 0 {
@@ -1355,15 +1341,6 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	gatewayIP, _ := cmd.Flags().GetString("gateway-ip")
 	gpuPassthrough, _ := cmd.Flags().GetBool("gpu-passthrough")
 	ipsecEnabled, _ := cmd.Flags().GetBool("ipsec")
-	libreswan := hostos.HasLibreswan()
-	ipsecEnabled, err := resolveIPSecForHost(ipsecEnabled, cmd.Flags().Changed("ipsec"), libreswan)
-	if err != nil {
-		fmt.Fprintf(os.Stderr, "--ipsec: %v\n", err)
-		os.Exit(1)
-	}
-	if libreswan && !cmd.Flags().Changed("ipsec") {
-		fmt.Println("IPsec: off, because this host's IPsec stack is libreswan, which Spinifex cannot run OVN IPsec on yet")
-	}
 
 	// Fire telemetry in background (completes during init work, waited at end)
 	noTelemetry, _ := cmd.Flags().GetBool("no-telemetry")
@@ -1746,7 +1723,7 @@ func runAdminInit(cmd *cobra.Command, args []string) {
 	if ipsecEnabled {
 		caCertPath := filepath.Join(configDir, "ca.pem")
 		caKeyPath := filepath.Join(configDir, "ca.key")
-		if err := host.GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, node, bindIP); err != nil {
+		if err := host.GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, node, bindIP, hostos.HasLibreswan()); err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating IPsec peer certificate: %v\n", err)
 			os.Exit(1)
 		}
@@ -2757,7 +2734,7 @@ func runAdminJoin(cmd *cobra.Command, args []string) {
 	if ipsecEnabled {
 		caCertPath := filepath.Join(configDir, "ca.pem")
 		caKeyPath := filepath.Join(configDir, "ca.key")
-		if err := host.GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, node, bindIP); err != nil {
+		if err := host.GenerateIPSecPeerCert(configDir, caCertPath, caKeyPath, node, bindIP, hostos.HasLibreswan()); err != nil {
 			fmt.Fprintf(os.Stderr, "Error generating IPsec peer certificate: %v\n", err)
 			os.Exit(1)
 		}

spinifex/operator/cli/admin_ipsec_host_test.go

@@ -1,48 +0,0 @@
-//test:in-package — resolveIPSecForHost is an unexported guard on this command's
-// flags, and exporting it would widen the package's API for a test's convenience.
-
-package cli
-
-import (
-	"strings"
-	"testing"
-)
-
-// A libreswan host cannot run OVN IPsec, so it defaults to off and refuses an
-// explicit request; every other host keeps whatever --ipsec says.
-func TestResolveIPSecForHost(t *testing.T) {
-	for _, tc := range []struct {
-		name                           string
-		requested, explicit, libreswan bool
-		want                           bool
-		wantErr                        bool
-	}{
-		{name: "strongSwan default", requested: true, want: true},
-		{name: "strongSwan explicit off", explicit: true},
-		{name: "libreswan default", requested: true, libreswan: true},
-		{name: "libreswan explicit off", explicit: true, libreswan: true},
-		{name: "libreswan explicit on", requested: true, explicit: true, libreswan: true, wantErr: true},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			got, err := resolveIPSecForHost(tc.requested, tc.explicit, tc.libreswan)
-			if tc.wantErr {
-				if err == nil {
-					t.Fatal("want an error, got nil")
-				}
-				// The operator needs to know why, and what to run instead.
-				for _, word := range []string{"libreswan", "/usr/libexec/ipsec/pluto", "without --ipsec"} {
-					if !strings.Contains(err.Error(), word) {
-						t.Errorf("error does not name %q: %v", word, err)
-					}
-				}
-				return
-			}
-			if err != nil {
-				t.Fatal(err)
-			}
-			if got != tc.want {
-				t.Fatalf("got %v, want %v", got, tc.want)
-			}
-		})
-	}
-}

tests/e2e/lb/lb_test.go

@@ -1708,7 +1708,9 @@ func remoteTCPRoundRobin(t *testing.T, ctx context.Context, ssh *harness.PeerSSH
 	t.Helper()
 	var lines []string
 	for i := 0; i < n; i++ {
-		out, err := ssh.Run(ctx, peer, fmt.Sprintf("echo '' | nc -w5 '%s' %d", host, port))
+		// Read without writing, as TCPRoundRobin does: nmap-ncat (Oracle Linux) half-closes
+		// on stdin EOF where netcat-openbsd does not, and the NLB then drops the reply.
+		out, err := ssh.Run(ctx, peer, fmt.Sprintf("timeout 5 bash -c 'exec 3<>/dev/tcp/%s/%d && head -n1 <&3'", host, port))
 		if err != nil {
 			continue
 		}

tests/e2e/multinode/ovn_raft_test.go

@@ -26,9 +26,10 @@ func ovnCentralUnits(debianUnits string) string {
 }
 
 // runOVNRaft asserts the clustered OVN control plane: every DB node's NB and SB
-// RAFT cluster reports the full quorum with exactly one leader, then kills the
-// NB leader's ovn-central and proves NB writes still land via a surviving
-// endpoint (libovsdb-style multi-endpoint failover, here through ovn-nbctl).
+// RAFT cluster reports the full quorum with exactly one leader, survives a
+// rolling restart of each node's OVN central, then kills the NB leader's
+// ovn-central and proves NB writes still land via a surviving endpoint
+// (libovsdb-style multi-endpoint failover, here through ovn-nbctl).
 func runOVNRaft(t *testing.T, fix *Fixture) {
 	harness.Phase(t, "Multinode — OVN OVSDB RAFT")
 
@@ -40,25 +41,39 @@ func runOVNRaft(t *testing.T, fix *Fixture) {
 	ssh := harness.NewPeerSSH()
 
 	// 1. Quorum health: 3 servers + exactly one leader, per schema, per node.
-	for _, db := range []struct{ name, sock, schema string }{
-		{"NB", ovnNBSock, host.OVNNBSchema},
-		{"SB", ovnSBSock, host.OVNSBSchema},
-	} {
-		leaders := 0
-		for _, n := range dbNodes {
-			harness.Step(t, "%s cluster/status on %s (%s)", db.name, n.Name, n.Addr)
-			servers, role := ovnClusterStatus(t, ssh, n, db.sock, db.schema)
-			harness.Detail(t, fmt.Sprintf("%s_%s_servers", db.name, n.Name), servers)
-			harness.Detail(t, fmt.Sprintf("%s_%s_role", db.name, n.Name), role)
-			require.Equalf(t, 3, servers, "%s on %s: want 3 servers in quorum", db.name, n.Name)
-			if role == "leader" {
-				leaders++
-			}
-		}
-		require.Equalf(t, 1, leaders, "%s: want exactly one leader across the quorum", db.name)
-	}
+	assertOVNQuorum(t, ssh, dbNodes)
 
-	// 2. Failover: kill the NB leader and confirm an NB write still succeeds.
+	// 2. Rolling restart, as setup-ovn.sh reruns do it: each node restarts both
+	// of its members at once, on EL through the one ovn-northd unit, and the
+	// next node waits for them to rejoin. NB writes must land after every step.
+	dbList := nbEndpointList(dbNodes)
+	for i, n := range dbNodes {
+		harness.Step(t, "restart OVN central on %s (%s)", n.Name, n.Addr)
+		restartCtx, restartCancel := context.WithTimeout(context.Background(), 60*time.Second)
+		_, err := ssh.Run(restartCtx, n.Addr, "sudo systemctl restart "+ovnCentralUnits("ovn-ovsdb-server-nb ovn-ovsdb-server-sb ovn-northd"))
+		restartCancel()
+		require.NoErrorf(t, err, "restart OVN central on %s", n.Name)
+
+		for _, db := range ovnRaftDBs {
+			waitOVNRaftMember(t, ssh, n, db.sock, db.schema, 60*time.Second)
+		}
+
+		sw := fmt.Sprintf("e2e-raft-rolling-%d-%d", time.Now().Unix(), i)
+		t.Cleanup(func() {
+			ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+			defer cancel()
+			_, _ = ssh.Run(ctx, n.Addr,
+				fmt.Sprintf("sudo ovn-nbctl --db=%s --timeout=10 --if-exists ls-del %s", dbList, sw))
+		})
+		harness.Step(t, "NB write (ls-add %s) after %s rejoined", sw, n.Name)
+		writeCtx, writeCancel := context.WithTimeout(context.Background(), 25*time.Second)
+		_, err = ssh.Run(writeCtx, n.Addr, fmt.Sprintf("sudo ovn-nbctl --db=%s --timeout=20 ls-add %s", dbList, sw))
+		writeCancel()
+		require.NoErrorf(t, err, "NB ls-add after restarting OVN central on %s", n.Name)
+	}
+	assertOVNQuorum(t, ssh, dbNodes)
+
+	// 3. Failover: kill the NB leader and confirm an NB write still succeeds.
 	var leader harness.Node
 	var survivors []harness.Node
 	for _, n := range dbNodes {
@@ -85,7 +100,6 @@ func runOVNRaft(t *testing.T, fix *Fixture) {
 	})
 
 	// The --db list includes the dead leader so the write must fail over past it.
-	dbList := nbEndpointList(dbNodes)
 	sw := fmt.Sprintf("e2e-raft-failover-%d", time.Now().Unix())
 	t.Cleanup(func() {
 		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
@@ -120,6 +134,67 @@ func runOVNRaft(t *testing.T, fix *Fixture) {
 	require.Containsf(t, string(out), sw, "switch %s not visible after failover write", sw)
 }
 
+var ovnRaftDBs = []struct{ name, sock, schema string }{
+	{"NB", ovnNBSock, host.OVNNBSchema},
+	{"SB", ovnSBSock, host.OVNSBSchema},
+}
+
+// assertOVNQuorum checks every DB node lists 3 servers per schema and that
+// exactly one of them leads each.
+func assertOVNQuorum(t *testing.T, ssh *harness.PeerSSH, dbNodes []harness.Node) {
+	t.Helper()
+	for _, db := range ovnRaftDBs {
+		leaders := 0
+		for _, n := range dbNodes {
+			harness.Step(t, "%s cluster/status on %s (%s)", db.name, n.Name, n.Addr)
+			servers, role := ovnClusterStatus(t, ssh, n, db.sock, db.schema)
+			harness.Detail(t, fmt.Sprintf("%s_%s_servers", db.name, n.Name), servers)
+			harness.Detail(t, fmt.Sprintf("%s_%s_role", db.name, n.Name), role)
+			require.Equalf(t, 3, servers, "%s on %s: want 3 servers in quorum", db.name, n.Name)
+			if role == "leader" {
+				leaders++
+			}
+		}
+		require.Equalf(t, 1, leaders, "%s: want exactly one leader across the quorum", db.name)
+	}
+}
+
+// waitOVNRaftMember polls until the node's server for schema is a cluster
+// member that knows its leader, the condition setup-ovn.sh waits on.
+func waitOVNRaftMember(t *testing.T, ssh *harness.PeerSSH, n harness.Node, sock, schema string, timeout time.Duration) {
+	t.Helper()
+	var last string
+	deadline := time.Now().Add(timeout)
+	for time.Now().Before(deadline) {
+		ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+		out, err := ssh.Run(ctx, n.Addr, fmt.Sprintf("sudo ovn-appctl -t %s cluster/status %s", sock, schema))
+		cancel()
+		last = string(out)
+		if err == nil && ovnRaftMemberRejoined(last) {
+			return
+		}
+		time.Sleep(2 * time.Second)
+	}
+	t.Fatalf("%s on %s did not rejoin its RAFT cluster within %s; last cluster/status:\n%s", schema, n.Name, timeout, last)
+}
+
+// ovnRaftMemberRejoined reports whether cluster/status shows a cluster member
+// following a known leader, or leading itself.
+func ovnRaftMemberRejoined(out string) bool {
+	var member, leader bool
+	for line := range strings.SplitSeq(out, "\n") {
+		line = strings.TrimSpace(line)
+		if line == "Status: cluster member" {
+			member = true
+		}
+		if rest, ok := strings.CutPrefix(line, "Leader:"); ok {
+			v := strings.TrimSpace(rest)
+			leader = v != "" && v != "unknown"
+		}
+	}
+	return member && leader
+}
+
 // ovnClusterStatus runs cluster/status for one schema on a node and returns the
 // reported server count and the queried server's role.
 func ovnClusterStatus(t *testing.T, ssh *harness.PeerSSH, n harness.Node, sock, schema string) (servers int, role string) {

tests/e2e/multinode/tests_test.go

@@ -83,8 +83,9 @@ func TestMultinodeClusterRestartResumesGuests(t *testing.T) {
 	runClusterRestartResumesGuests(t, requireMultiNodeFixture(t))
 }
 
-// TestMultinodeOVNRaft is sequential: stops ovn-central on the NB leader to
-// prove DB failover, restoring it in cleanup before later tests run.
+// TestMultinodeOVNRaft is sequential: restarts each node's OVN central in turn,
+// then stops ovn-central on the NB leader to prove DB failover, restoring it
+// in cleanup before later tests run.
 func TestMultinodeOVNRaft(t *testing.T) {
 	runOVNRaft(t, requireMultiNodeFixture(t))
 }