fix(ipsec): guard ipsec_encapsulation on ovs-monitor-ipsec liveness

enableOVNIPSec set OVS cert pointers + ipsec_encapsulation=true without
ensuring ovs-monitor-ipsec.service was running. Without that daemon, OVS
holds the cert pointers unused, strongSwan never gets IKE config, and
tunnel traffic ovn-controller marks for encryption gets silently
dropped. TestMultinodeIPSec fails on `node1 xfrm has no AEAD SAs`.

- systemd: spinifex-daemon Wants=/After= ovs-monitor-ipsec.service
- daemon: ensureOVSMonitorIPSecActive — systemctl enable --now +
  is-active poll; refuses to flip ipsec_encapsulation if daemon is dead
- daemon: single-node short-circuit — no peers means no tunnels means
  the flag is at best inert and at worst load-bearing-silent-drop
- ci: capture-net-diag.sh + workflow steps to snapshot OVS/OVN/xfrm
  state on failure, fan out across all 3 multinode nodes

Refs: mulga-siv-136

.github/scripts/capture-net-diag.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# capture-net-diag.sh — write a snapshot of network / OVN / OVS / IPsec
+# state into $1 (a directory the caller already created on the local host).
+# Designed to run on a spinifex node after an E2E suite, before tofu destroy
+# pulls the rug out. Every command is independent — `|| true` so a missing
+# binary on one path doesn't abort the rest of the capture.
+#
+# Output layout (under $1/net-diag/):
+#   ovs-monitor-ipsec.status         systemctl status (output-only)
+#   ovs-monitor-ipsec.journal        last 200 journal lines
+#   ipsec.openvswitch_conf           ovs-vsctl get Open_vSwitch . other_config
+#   ipsec.xfrm_state                 ip xfrm state list
+#   ipsec.xfrm_policy                ip xfrm policy list
+#   ovn.sb_show                      ovn-sbctl show
+#   ovn.nb_show                      ovn-nbctl show
+#   ovs.show                         ovs-vsctl show
+#   ovs.flows_br_int                 ovs-ofctl dump-flows br-int
+#   ovs.tunnels                      ovs-appctl ofproto/list-tunnels br-int
+#   net.links                        ip -s link show
+#   net.routes                       ip route show table all
+#   net.addrs                        ip -br addr show
+#   systemd.failed_units             systemctl list-units --state=failed
+set +e
+OUT="${1:?usage: capture-net-diag.sh <output-dir>}"
+mkdir -p "$OUT"
+
+run() {
+    local label="$1"; shift
+    {
+        echo "# $*"
+        "$@" 2>&1
+    } > "$OUT/$label" || true
+}
+
+run ovs-monitor-ipsec.status            sudo systemctl status ovs-monitor-ipsec.service --no-pager --full --lines=50
+run ovs-monitor-ipsec.journal           sudo journalctl -u ovs-monitor-ipsec.service --no-pager --output=short-iso -n 200
+run ipsec.openvswitch_conf              sudo ovs-vsctl get Open_vSwitch . other_config
+run ipsec.xfrm_state                    sudo ip xfrm state list
+run ipsec.xfrm_policy                   sudo ip xfrm policy list
+
+run ovn.sb_show                         sudo ovn-sbctl show
+run ovn.nb_show                         sudo ovn-nbctl show
+run ovn.sb_chassis                      sudo ovn-sbctl list chassis
+
+run ovs.show                            sudo ovs-vsctl show
+run ovs.flows_br_int                    sudo ovs-ofctl dump-flows br-int
+run ovs.tunnels                         sudo ovs-appctl ofproto/list-tunnels br-int
+
+run net.links                           ip -s link show
+run net.routes                          ip route show table all
+run net.addrs                           ip -br addr show
+
+run systemd.failed_units                sudo systemctl list-units --state=failed --no-pager
+
+exit 0

.github/workflows/e2e-go.yml

@@ -277,6 +277,23 @@ jobs:
           echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
           exit 0  # don't fail the step — artifact upload + final gate happen below
 
+      - name: Capture network/IPsec diagnostics on VM
+        if: always()
+        run: |
+          set +e
+          # shellcheck source=/dev/null
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
+          VM=tf-user@${{ steps.vm.outputs.wan_ip }}
+          # Ship the diag script, run it into the VM's ARTIFACT_DIR, leave the
+          # output for the next step's tar to pull down. Captures OVN/OVS/IPsec
+          # state at failure time — before teardown destroys the VM.
+          scp "${SSH_OPTS[@]}" \
+            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
+            "$VM":/tmp/capture-net-diag.sh
+          ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'"
+          ok "net diagnostics captured"
+
       - name: Collect artifacts from VM
         if: always()
         run: |
@@ -653,6 +670,42 @@ jobs:
           echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
           exit 0
 
+      - name: Capture network/IPsec diagnostics on all nodes
+        if: always()
+        run: |
+          set +e
+          # shellcheck source=/dev/null
+          source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
+          SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
+          # Capture per-node into the node1 ARTIFACT_DIR (where Collect pulls
+          # from). One subdir per node so xfrm SA state is comparable across
+          # peers — critical for diagnosing the IPsec wiring (mulga-siv-136).
+          IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
+          NODE1_IP="${NODE_LIST[0]}"
+          scp "${SSH_OPTS[@]}" \
+            "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
+            tf-user@"$NODE1_IP":/tmp/capture-net-diag.sh
+          for i in "${!NODE_LIST[@]}"; do
+            idx=$((i + 1))
+            ip="${NODE_LIST[$i]}"
+            if [ "$idx" -ne 1 ]; then
+              scp "${SSH_OPTS[@]}" \
+                "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
+                tf-user@"$ip":/tmp/capture-net-diag.sh
+            fi
+            ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
+              "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag-node${idx}'"
+            # node1 keeps its diag locally; pull node2..N back into node1's
+            # ARTIFACT_DIR so the existing tar-pull below catches everything.
+            if [ "$idx" -ne 1 ]; then
+              ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
+                "tar -C '${ARTIFACT_DIR}' -cf - 'net-diag-node${idx}' 2>/dev/null || true" \
+                | ssh "${SSH_OPTS[@]}" tf-user@"$NODE1_IP" \
+                  "tar -C '${ARTIFACT_DIR}' -xf - 2>/dev/null"
+            fi
+            ok "net diagnostics node${idx} captured"
+          done
+
       - name: Collect artifacts from node1
         if: always()
         run: |
@@ -675,6 +728,13 @@ jobs:
               > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
             ok "journal node${idx}: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null || echo 0) lines"
           done
+          # ovs-monitor-ipsec journal per node (separate file — bead 136).
+          for i in "${!NODE_LIST[@]}"; do
+            idx=$((i + 1))
+            ip="${NODE_LIST[$i]}"
+            ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u ovs-monitor-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
+              > "${ARTIFACT_DIR}/ovs-monitor-ipsec-node${idx}.log" 2>/dev/null
+          done
 
       - name: Convert test logs to JUnit XML
         if: always()

build/systemd/spinifex-daemon.service

@@ -1,7 +1,14 @@
 [Unit]
 Description=Spinifex Daemon (VM Orchestration)
 PartOf=spinifex.target
-After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service
+After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service ovs-monitor-ipsec.service
+# Wants=ovs-monitor-ipsec.service so systemd brings the IPsec userspace
+# daemon up before spinifex flips other_config:ipsec_encapsulation=true.
+# Without that daemon, OVS holds the cert pointers unused and strongSwan
+# never gets IKE config — tunnel traffic is encrypted with no SAs and
+# silently dropped. Wants= (not Requires=) keeps daemon bootable on
+# single-node / lab hosts where the package is intentionally absent.
+Wants=ovs-monitor-ipsec.service
 # No Wants=spinifex-nats.service: Wants= reactivates a stopped unit during
 # `systemctl restart spinifex-daemon`, which would mask the standalone-startup
 # path the Tier 1 DDIL Scenario B exercises. spinifex.target Wants= covers

spinifex/daemon/ipsec.go

@@ -6,10 +6,16 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/mulgadc/spinifex/spinifex/admin"
 )
 
+// systemctlActiveTimeout bounds how long enableOVNIPSec waits for
+// ovs-monitor-ipsec.service to report active after a `systemctl enable
+// --now`. Overridden in tests.
+var systemctlActiveTimeout = 5 * time.Second
+
 // enableOVNIPSec wires the per-node IPsec peer cert into the local OVS DB and
 // flips ipsec_encapsulation=true. ovs-monitor-ipsec (shipped by the
 // openvswitch-ipsec package) reads the cert/key/CA pointers and materialises
@@ -24,10 +30,19 @@ import (
 // function returns an error so the daemon can log it and continue; the
 // cluster keeps running with plaintext Geneve until an operator reissues
 // certs.
+//
+// Single-node clusters short-circuit: with no peers, there are no Geneve
+// tunnels to encrypt, and flipping ipsec_encapsulation=true on a host where
+// ovs-monitor-ipsec is absent or dead would create the silent-drop trap that
+// triggers mulga-siv-136.
 func (d *Daemon) enableOVNIPSec() error {
 	if d.configPath == "" {
 		return fmt.Errorf("config path unset")
 	}
+	if d.clusterConfig != nil && len(d.clusterConfig.Nodes) <= 1 {
+		slog.Info("ipsec: single-node cluster, skipping enable (no peers)")
+		return nil
+	}
 	configDir := filepath.Dir(d.configPath)
 	certPath, keyPath := admin.IPSecCertPaths(configDir)
 	caCertPath := filepath.Join(configDir, "ca.pem")
@@ -38,6 +53,10 @@ func (d *Daemon) enableOVNIPSec() error {
 		}
 	}
 
+	if err := ensureOVSMonitorIPSecActive(); err != nil {
+		return fmt.Errorf("ovs-monitor-ipsec: %w", err)
+	}
+
 	if out, err := sudoCommand("ovs-vsctl", "set", "Open_vSwitch", ".",
 		fmt.Sprintf("other_config:certificate=%s", certPath),
 		fmt.Sprintf("other_config:private_key=%s", keyPath),
@@ -59,3 +78,24 @@ func (d *Daemon) enableOVNIPSec() error {
 	)
 	return nil
 }
+
+// ensureOVSMonitorIPSecActive enables + starts ovs-monitor-ipsec.service and
+// waits for it to report active. Without this guarantee, ipsec_encapsulation
+// silently drops all tunnel traffic because no userspace daemon is reading
+// the OVS DB cert pointers to drive strongSwan IKE.
+func ensureOVSMonitorIPSecActive() error {
+	if out, err := sudoCommand("systemctl", "enable", "--now", "ovs-monitor-ipsec.service").CombinedOutput(); err != nil {
+		return fmt.Errorf("enable --now: %s: %w", strings.TrimSpace(string(out)), err)
+	}
+	deadline := time.Now().Add(systemctlActiveTimeout)
+	var lastOut string
+	for time.Now().Before(deadline) {
+		out, _ := sudoCommand("systemctl", "is-active", "ovs-monitor-ipsec.service").CombinedOutput()
+		lastOut = strings.TrimSpace(string(out))
+		if lastOut == "active" {
+			return nil
+		}
+		time.Sleep(250 * time.Millisecond)
+	}
+	return fmt.Errorf("not active after %s: %s", systemctlActiveTimeout, lastOut)
+}

spinifex/daemon/ipsec_test.go

@@ -6,20 +6,88 @@ import (
 	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
+	"github.com/mulgadc/spinifex/spinifex/config"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func TestEnableOVNIPSec(t *testing.T) {
-	// Swap sudoCommand for a stub that records args and returns nil.
-	type recorderture struct {
-		argsRuns [][]string
+// multiNodeClusterConfig returns a ClusterConfig with two nodes so
+// enableOVNIPSec runs its full path (single-node would short-circuit).
+func multiNodeClusterConfig() *config.ClusterConfig {
+	return &config.ClusterConfig{
+		Node: "node1",
+		Nodes: map[string]config.Config{
+			"node1": {},
+			"node2": {},
+		},
 	}
-	recorder := &recorderture{}
+}
+
+// recordingSudo is a sudoCommand stub that records every invocation and
+// returns canned stdout for `systemctl is-active`. Tests that need a dead
+// daemon override the activeOutput.
+type recordingSudo struct {
+	runs         [][]string
+	activeOutput string
+}
+
+func (r *recordingSudo) stub(name string, args ...string) *exec.Cmd {
+	r.runs = append(r.runs, append([]string{name}, args...))
+	if name == "systemctl" && len(args) >= 1 && args[0] == "is-active" {
+		out := r.activeOutput
+		if out == "" {
+			out = "active\n"
+		}
+		return exec.Command("printf", "%s", out)
+	}
+	return exec.Command("true")
+}
+
+func TestEnableOVNIPSec(t *testing.T) {
+	recorder := &recordingSudo{}
+	orig := sudoCommand
+	sudoCommand = recorder.stub
+	t.Cleanup(func() { sudoCommand = orig })
+
+	configDir := t.TempDir()
+	configPath := filepath.Join(configDir, "spinifex.toml")
+	require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600))
+
+	for _, rel := range []string{"ca.pem", "ipsec/peer.pem", "ipsec/peer.key"} {
+		full := filepath.Join(configDir, rel)
+		require.NoError(t, os.MkdirAll(filepath.Dir(full), 0750))
+		require.NoError(t, os.WriteFile(full, []byte("x"), 0600))
+	}
+
+	d := &Daemon{configPath: configPath, clusterConfig: multiNodeClusterConfig()}
+	require.NoError(t, d.enableOVNIPSec())
+
+	// Expected sudo invocations in order:
+	//   systemctl enable --now ovs-monitor-ipsec.service
+	//   systemctl is-active ovs-monitor-ipsec.service
+	//   ovs-vsctl set Open_vSwitch . other_config:certificate=... private_key=... ca_cert=...
+	//   ovs-vsctl set Open_vSwitch . other_config:ipsec_encapsulation=true
+	require.Len(t, recorder.runs, 4)
+	assert.Equal(t, []string{"systemctl", "enable", "--now", "ovs-monitor-ipsec.service"}, recorder.runs[0])
+	assert.Equal(t, []string{"systemctl", "is-active", "ovs-monitor-ipsec.service"}, recorder.runs[1])
+	for _, run := range recorder.runs[2:] {
+		assert.Equal(t, "ovs-vsctl", run[0])
+		assert.Equal(t, "set", run[1])
+		assert.Equal(t, "Open_vSwitch", run[2])
+	}
+	joined := strings.Join(recorder.runs[2], " ")
+	assert.Contains(t, joined, "other_config:certificate="+filepath.Join(configDir, "ipsec", "peer.pem"))
+	assert.Contains(t, joined, "other_config:private_key="+filepath.Join(configDir, "ipsec", "peer.key"))
+	assert.Contains(t, joined, "other_config:ca_cert="+filepath.Join(configDir, "ca.pem"))
+	assert.Contains(t, strings.Join(recorder.runs[3], " "), "other_config:ipsec_encapsulation=true")
+}
+
+func TestEnableOVNIPSec_SingleNodeSkip(t *testing.T) {
 	orig := sudoCommand
 	sudoCommand = func(name string, args ...string) *exec.Cmd {
-		recorder.argsRuns = append(recorder.argsRuns, append([]string{name}, args...))
+		t.Fatalf("sudoCommand must not run on single-node short-circuit; got %s %v", name, args)
 		return exec.Command("true")
 	}
 	t.Cleanup(func() { sudoCommand = orig })
@@ -28,29 +96,46 @@ func TestEnableOVNIPSec(t *testing.T) {
 	configPath := filepath.Join(configDir, "spinifex.toml")
 	require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600))
 
-	// Seed the three credential files enableOVNIPSec checks for. Bytes
-	// don't matter — only the presence stat.
+	d := &Daemon{
+		configPath: configPath,
+		clusterConfig: &config.ClusterConfig{
+			Node:  "node1",
+			Nodes: map[string]config.Config{"node1": {}},
+		},
+	}
+	require.NoError(t, d.enableOVNIPSec())
+}
+
+func TestEnableOVNIPSec_MonitorIPSecInactive(t *testing.T) {
+	recorder := &recordingSudo{activeOutput: "inactive\n"}
+	orig := sudoCommand
+	sudoCommand = recorder.stub
+	t.Cleanup(func() { sudoCommand = orig })
+
+	origTimeout := systemctlActiveTimeout
+	systemctlActiveTimeout = 100 * time.Millisecond
+	t.Cleanup(func() { systemctlActiveTimeout = origTimeout })
+
+	configDir := t.TempDir()
+	configPath := filepath.Join(configDir, "spinifex.toml")
+	require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600))
 	for _, rel := range []string{"ca.pem", "ipsec/peer.pem", "ipsec/peer.key"} {
 		full := filepath.Join(configDir, rel)
 		require.NoError(t, os.MkdirAll(filepath.Dir(full), 0750))
 		require.NoError(t, os.WriteFile(full, []byte("x"), 0600))
 	}
 
-	d := &Daemon{configPath: configPath}
-	require.NoError(t, d.enableOVNIPSec())
+	d := &Daemon{configPath: configPath, clusterConfig: multiNodeClusterConfig()}
+	err := d.enableOVNIPSec()
+	require.Error(t, err)
+	assert.Contains(t, err.Error(), "ovs-monitor-ipsec")
+	assert.Contains(t, err.Error(), "not active")
 
-	// Two ovs-vsctl invocations: cert pointers + ipsec_enrecordersulation toggle.
-	require.Len(t, recorder.argsRuns, 2)
-	for _, run := range recorder.argsRuns {
-		assert.Equal(t, "ovs-vsctl", run[0])
-		assert.Equal(t, "set", run[1])
-		assert.Equal(t, "Open_vSwitch", run[2])
+	// ovs-vsctl must NOT have been called — flipping ipsec_encapsulation=true
+	// with no daemon is the silent-drop trap this guard exists to prevent.
+	for _, run := range recorder.runs {
+		assert.NotEqual(t, "ovs-vsctl", run[0], "ovs-vsctl invoked despite dead daemon: %v", run)
 	}
-	joined := strings.Join(recorder.argsRuns[0], " ")
-	assert.Contains(t, joined, "other_config:certificate="+filepath.Join(configDir, "ipsec", "peer.pem"))
-	assert.Contains(t, joined, "other_config:private_key="+filepath.Join(configDir, "ipsec", "peer.key"))
-	assert.Contains(t, joined, "other_config:ca_cert="+filepath.Join(configDir, "ca.pem"))
-	assert.Contains(t, strings.Join(recorder.argsRuns[1], " "), "other_config:ipsec_encapsulation=true")
 }
 
 func TestEnableOVNIPSec_MissingCert(t *testing.T) {
@@ -64,10 +149,9 @@ func TestEnableOVNIPSec_MissingCert(t *testing.T) {
 	configDir := t.TempDir()
 	configPath := filepath.Join(configDir, "spinifex.toml")
 	require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600))
-	// CA exists; peer cert missing.
 	require.NoError(t, os.WriteFile(filepath.Join(configDir, "ca.pem"), []byte("x"), 0600))
 
-	d := &Daemon{configPath: configPath}
+	d := &Daemon{configPath: configPath, clusterConfig: multiNodeClusterConfig()}
 	err := d.enableOVNIPSec()
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "missing IPsec credential")