diff --git a/.github/scripts/capture-net-diag.sh b/.github/scripts/capture-net-diag.sh new file mode 100755 index 000000000..0efba1f45 --- /dev/null +++ b/.github/scripts/capture-net-diag.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# capture-net-diag.sh — write a snapshot of network / OVN / OVS / IPsec +# state into $1 (a directory the caller already created on the local host). +# Designed to run on a spinifex node after an E2E suite, before tofu destroy +# pulls the rug out. Every command is independent — `|| true` so a missing +# binary on one path doesn't abort the rest of the capture. +# +# Output layout (under $1/net-diag/): +# ovs-monitor-ipsec.status systemctl status (output-only) +# ovs-monitor-ipsec.journal last 200 journal lines +# ipsec.openvswitch_conf ovs-vsctl get Open_vSwitch . other_config +# ipsec.xfrm_state ip xfrm state list +# ipsec.xfrm_policy ip xfrm policy list +# ovn.sb_show ovn-sbctl show +# ovn.nb_show ovn-nbctl show +# ovs.show ovs-vsctl show +# ovs.flows_br_int ovs-ofctl dump-flows br-int +# ovs.tunnels ovs-appctl ofproto/list-tunnels br-int +# net.links ip -s link show +# net.routes ip route show table all +# net.addrs ip -br addr show +# systemd.failed_units systemctl list-units --state=failed +set +e +OUT="${1:?usage: capture-net-diag.sh }" +mkdir -p "$OUT" + +run() { + local label="$1"; shift + { + echo "# $*" + "$@" 2>&1 + } > "$OUT/$label" || true +} + +run ovs-monitor-ipsec.status sudo systemctl status ovs-monitor-ipsec.service --no-pager --full --lines=50 +run ovs-monitor-ipsec.journal sudo journalctl -u ovs-monitor-ipsec.service --no-pager --output=short-iso -n 200 +run ipsec.openvswitch_conf sudo ovs-vsctl get Open_vSwitch . other_config +run ipsec.xfrm_state sudo ip xfrm state list +run ipsec.xfrm_policy sudo ip xfrm policy list + +run ovn.sb_show sudo ovn-sbctl show +run ovn.nb_show sudo ovn-nbctl show +run ovn.sb_chassis sudo ovn-sbctl list chassis + +run ovs.show sudo ovs-vsctl show +run ovs.flows_br_int sudo ovs-ofctl dump-flows br-int +run ovs.tunnels sudo ovs-appctl ofproto/list-tunnels br-int + +run net.links ip -s link show +run net.routes ip route show table all +run net.addrs ip -br addr show + +run systemd.failed_units sudo systemctl list-units --state=failed --no-pager + +exit 0 diff --git a/.github/workflows/e2e-go.yml b/.github/workflows/e2e-go.yml index 49141b187..97a983815 100644 --- a/.github/workflows/e2e-go.yml +++ b/.github/workflows/e2e-go.yml @@ -277,6 +277,23 @@ jobs: echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT" exit 0 # don't fail the step — artifact upload + final gate happen below + - name: Capture network/IPsec diagnostics on VM + if: always() + run: | + set +e + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}") + VM=tf-user@${{ steps.vm.outputs.wan_ip }} + # Ship the diag script, run it into the VM's ARTIFACT_DIR, leave the + # output for the next step's tar to pull down. Captures OVN/OVS/IPsec + # state at failure time — before teardown destroys the VM. + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ + "$VM":/tmp/capture-net-diag.sh + ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'" + ok "net diagnostics captured" + - name: Collect artifacts from VM if: always() run: | @@ -653,6 +670,42 @@ jobs: echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT" exit 0 + - name: Capture network/IPsec diagnostics on all nodes + if: always() + run: | + set +e + # shellcheck source=/dev/null + source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh" + SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}") + # Capture per-node into the node1 ARTIFACT_DIR (where Collect pulls + # from). One subdir per node so xfrm SA state is comparable across + # peers — critical for diagnosing the IPsec wiring (mulga-siv-136). + IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}" + NODE1_IP="${NODE_LIST[0]}" + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ + tf-user@"$NODE1_IP":/tmp/capture-net-diag.sh + for i in "${!NODE_LIST[@]}"; do + idx=$((i + 1)) + ip="${NODE_LIST[$i]}" + if [ "$idx" -ne 1 ]; then + scp "${SSH_OPTS[@]}" \ + "${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \ + tf-user@"$ip":/tmp/capture-net-diag.sh + fi + ssh "${SSH_OPTS[@]}" tf-user@"$ip" \ + "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag-node${idx}'" + # node1 keeps its diag locally; pull node2..N back into node1's + # ARTIFACT_DIR so the existing tar-pull below catches everything. + if [ "$idx" -ne 1 ]; then + ssh "${SSH_OPTS[@]}" tf-user@"$ip" \ + "tar -C '${ARTIFACT_DIR}' -cf - 'net-diag-node${idx}' 2>/dev/null || true" \ + | ssh "${SSH_OPTS[@]}" tf-user@"$NODE1_IP" \ + "tar -C '${ARTIFACT_DIR}' -xf - 2>/dev/null" + fi + ok "net diagnostics node${idx} captured" + done + - name: Collect artifacts from node1 if: always() run: | @@ -675,6 +728,13 @@ jobs: > "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null ok "journal node${idx}: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null || echo 0) lines" done + # ovs-monitor-ipsec journal per node (separate file — bead 136). + for i in "${!NODE_LIST[@]}"; do + idx=$((i + 1)) + ip="${NODE_LIST[$i]}" + ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u ovs-monitor-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \ + > "${ARTIFACT_DIR}/ovs-monitor-ipsec-node${idx}.log" 2>/dev/null + done - name: Convert test logs to JUnit XML if: always() diff --git a/build/systemd/spinifex-daemon.service b/build/systemd/spinifex-daemon.service index 6d9edb95c..1db8bfb90 100644 --- a/build/systemd/spinifex-daemon.service +++ b/build/systemd/spinifex-daemon.service @@ -1,7 +1,14 @@ [Unit] Description=Spinifex Daemon (VM Orchestration) PartOf=spinifex.target -After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service +After=spinifex-nats.service spinifex-viperblock.service openvswitch-switch.service ovn-controller.service ovs-monitor-ipsec.service +# Wants=ovs-monitor-ipsec.service so systemd brings the IPsec userspace +# daemon up before spinifex flips other_config:ipsec_encapsulation=true. +# Without that daemon, OVS holds the cert pointers unused and strongSwan +# never gets IKE config — tunnel traffic is encrypted with no SAs and +# silently dropped. Wants= (not Requires=) keeps daemon bootable on +# single-node / lab hosts where the package is intentionally absent. +Wants=ovs-monitor-ipsec.service # No Wants=spinifex-nats.service: Wants= reactivates a stopped unit during # `systemctl restart spinifex-daemon`, which would mask the standalone-startup # path the Tier 1 DDIL Scenario B exercises. spinifex.target Wants= covers diff --git a/spinifex/daemon/ipsec.go b/spinifex/daemon/ipsec.go index 81f1d98e0..6d0a2ab55 100644 --- a/spinifex/daemon/ipsec.go +++ b/spinifex/daemon/ipsec.go @@ -6,10 +6,16 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/mulgadc/spinifex/spinifex/admin" ) +// systemctlActiveTimeout bounds how long enableOVNIPSec waits for +// ovs-monitor-ipsec.service to report active after a `systemctl enable +// --now`. Overridden in tests. +var systemctlActiveTimeout = 5 * time.Second + // enableOVNIPSec wires the per-node IPsec peer cert into the local OVS DB and // flips ipsec_encapsulation=true. ovs-monitor-ipsec (shipped by the // openvswitch-ipsec package) reads the cert/key/CA pointers and materialises @@ -24,10 +30,19 @@ import ( // function returns an error so the daemon can log it and continue; the // cluster keeps running with plaintext Geneve until an operator reissues // certs. +// +// Single-node clusters short-circuit: with no peers, there are no Geneve +// tunnels to encrypt, and flipping ipsec_encapsulation=true on a host where +// ovs-monitor-ipsec is absent or dead would create the silent-drop trap that +// triggers mulga-siv-136. func (d *Daemon) enableOVNIPSec() error { if d.configPath == "" { return fmt.Errorf("config path unset") } + if d.clusterConfig != nil && len(d.clusterConfig.Nodes) <= 1 { + slog.Info("ipsec: single-node cluster, skipping enable (no peers)") + return nil + } configDir := filepath.Dir(d.configPath) certPath, keyPath := admin.IPSecCertPaths(configDir) caCertPath := filepath.Join(configDir, "ca.pem") @@ -38,6 +53,10 @@ func (d *Daemon) enableOVNIPSec() error { } } + if err := ensureOVSMonitorIPSecActive(); err != nil { + return fmt.Errorf("ovs-monitor-ipsec: %w", err) + } + if out, err := sudoCommand("ovs-vsctl", "set", "Open_vSwitch", ".", fmt.Sprintf("other_config:certificate=%s", certPath), fmt.Sprintf("other_config:private_key=%s", keyPath), @@ -59,3 +78,24 @@ func (d *Daemon) enableOVNIPSec() error { ) return nil } + +// ensureOVSMonitorIPSecActive enables + starts ovs-monitor-ipsec.service and +// waits for it to report active. Without this guarantee, ipsec_encapsulation +// silently drops all tunnel traffic because no userspace daemon is reading +// the OVS DB cert pointers to drive strongSwan IKE. +func ensureOVSMonitorIPSecActive() error { + if out, err := sudoCommand("systemctl", "enable", "--now", "ovs-monitor-ipsec.service").CombinedOutput(); err != nil { + return fmt.Errorf("enable --now: %s: %w", strings.TrimSpace(string(out)), err) + } + deadline := time.Now().Add(systemctlActiveTimeout) + var lastOut string + for time.Now().Before(deadline) { + out, _ := sudoCommand("systemctl", "is-active", "ovs-monitor-ipsec.service").CombinedOutput() + lastOut = strings.TrimSpace(string(out)) + if lastOut == "active" { + return nil + } + time.Sleep(250 * time.Millisecond) + } + return fmt.Errorf("not active after %s: %s", systemctlActiveTimeout, lastOut) +} diff --git a/spinifex/daemon/ipsec_test.go b/spinifex/daemon/ipsec_test.go index 998057d5c..ba5fab310 100644 --- a/spinifex/daemon/ipsec_test.go +++ b/spinifex/daemon/ipsec_test.go @@ -6,20 +6,88 @@ import ( "path/filepath" "strings" "testing" + "time" + "github.com/mulgadc/spinifex/spinifex/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func TestEnableOVNIPSec(t *testing.T) { - // Swap sudoCommand for a stub that records args and returns nil. - type recorderture struct { - argsRuns [][]string +// multiNodeClusterConfig returns a ClusterConfig with two nodes so +// enableOVNIPSec runs its full path (single-node would short-circuit). +func multiNodeClusterConfig() *config.ClusterConfig { + return &config.ClusterConfig{ + Node: "node1", + Nodes: map[string]config.Config{ + "node1": {}, + "node2": {}, + }, } - recorder := &recorderture{} +} + +// recordingSudo is a sudoCommand stub that records every invocation and +// returns canned stdout for `systemctl is-active`. Tests that need a dead +// daemon override the activeOutput. +type recordingSudo struct { + runs [][]string + activeOutput string +} + +func (r *recordingSudo) stub(name string, args ...string) *exec.Cmd { + r.runs = append(r.runs, append([]string{name}, args...)) + if name == "systemctl" && len(args) >= 1 && args[0] == "is-active" { + out := r.activeOutput + if out == "" { + out = "active\n" + } + return exec.Command("printf", "%s", out) + } + return exec.Command("true") +} + +func TestEnableOVNIPSec(t *testing.T) { + recorder := &recordingSudo{} + orig := sudoCommand + sudoCommand = recorder.stub + t.Cleanup(func() { sudoCommand = orig }) + + configDir := t.TempDir() + configPath := filepath.Join(configDir, "spinifex.toml") + require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600)) + + for _, rel := range []string{"ca.pem", "ipsec/peer.pem", "ipsec/peer.key"} { + full := filepath.Join(configDir, rel) + require.NoError(t, os.MkdirAll(filepath.Dir(full), 0750)) + require.NoError(t, os.WriteFile(full, []byte("x"), 0600)) + } + + d := &Daemon{configPath: configPath, clusterConfig: multiNodeClusterConfig()} + require.NoError(t, d.enableOVNIPSec()) + + // Expected sudo invocations in order: + // systemctl enable --now ovs-monitor-ipsec.service + // systemctl is-active ovs-monitor-ipsec.service + // ovs-vsctl set Open_vSwitch . other_config:certificate=... private_key=... ca_cert=... + // ovs-vsctl set Open_vSwitch . other_config:ipsec_encapsulation=true + require.Len(t, recorder.runs, 4) + assert.Equal(t, []string{"systemctl", "enable", "--now", "ovs-monitor-ipsec.service"}, recorder.runs[0]) + assert.Equal(t, []string{"systemctl", "is-active", "ovs-monitor-ipsec.service"}, recorder.runs[1]) + for _, run := range recorder.runs[2:] { + assert.Equal(t, "ovs-vsctl", run[0]) + assert.Equal(t, "set", run[1]) + assert.Equal(t, "Open_vSwitch", run[2]) + } + joined := strings.Join(recorder.runs[2], " ") + assert.Contains(t, joined, "other_config:certificate="+filepath.Join(configDir, "ipsec", "peer.pem")) + assert.Contains(t, joined, "other_config:private_key="+filepath.Join(configDir, "ipsec", "peer.key")) + assert.Contains(t, joined, "other_config:ca_cert="+filepath.Join(configDir, "ca.pem")) + assert.Contains(t, strings.Join(recorder.runs[3], " "), "other_config:ipsec_encapsulation=true") +} + +func TestEnableOVNIPSec_SingleNodeSkip(t *testing.T) { orig := sudoCommand sudoCommand = func(name string, args ...string) *exec.Cmd { - recorder.argsRuns = append(recorder.argsRuns, append([]string{name}, args...)) + t.Fatalf("sudoCommand must not run on single-node short-circuit; got %s %v", name, args) return exec.Command("true") } t.Cleanup(func() { sudoCommand = orig }) @@ -28,29 +96,46 @@ func TestEnableOVNIPSec(t *testing.T) { configPath := filepath.Join(configDir, "spinifex.toml") require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600)) - // Seed the three credential files enableOVNIPSec checks for. Bytes - // don't matter — only the presence stat. + d := &Daemon{ + configPath: configPath, + clusterConfig: &config.ClusterConfig{ + Node: "node1", + Nodes: map[string]config.Config{"node1": {}}, + }, + } + require.NoError(t, d.enableOVNIPSec()) +} + +func TestEnableOVNIPSec_MonitorIPSecInactive(t *testing.T) { + recorder := &recordingSudo{activeOutput: "inactive\n"} + orig := sudoCommand + sudoCommand = recorder.stub + t.Cleanup(func() { sudoCommand = orig }) + + origTimeout := systemctlActiveTimeout + systemctlActiveTimeout = 100 * time.Millisecond + t.Cleanup(func() { systemctlActiveTimeout = origTimeout }) + + configDir := t.TempDir() + configPath := filepath.Join(configDir, "spinifex.toml") + require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600)) for _, rel := range []string{"ca.pem", "ipsec/peer.pem", "ipsec/peer.key"} { full := filepath.Join(configDir, rel) require.NoError(t, os.MkdirAll(filepath.Dir(full), 0750)) require.NoError(t, os.WriteFile(full, []byte("x"), 0600)) } - d := &Daemon{configPath: configPath} - require.NoError(t, d.enableOVNIPSec()) + d := &Daemon{configPath: configPath, clusterConfig: multiNodeClusterConfig()} + err := d.enableOVNIPSec() + require.Error(t, err) + assert.Contains(t, err.Error(), "ovs-monitor-ipsec") + assert.Contains(t, err.Error(), "not active") - // Two ovs-vsctl invocations: cert pointers + ipsec_enrecordersulation toggle. - require.Len(t, recorder.argsRuns, 2) - for _, run := range recorder.argsRuns { - assert.Equal(t, "ovs-vsctl", run[0]) - assert.Equal(t, "set", run[1]) - assert.Equal(t, "Open_vSwitch", run[2]) + // ovs-vsctl must NOT have been called — flipping ipsec_encapsulation=true + // with no daemon is the silent-drop trap this guard exists to prevent. + for _, run := range recorder.runs { + assert.NotEqual(t, "ovs-vsctl", run[0], "ovs-vsctl invoked despite dead daemon: %v", run) } - joined := strings.Join(recorder.argsRuns[0], " ") - assert.Contains(t, joined, "other_config:certificate="+filepath.Join(configDir, "ipsec", "peer.pem")) - assert.Contains(t, joined, "other_config:private_key="+filepath.Join(configDir, "ipsec", "peer.key")) - assert.Contains(t, joined, "other_config:ca_cert="+filepath.Join(configDir, "ca.pem")) - assert.Contains(t, strings.Join(recorder.argsRuns[1], " "), "other_config:ipsec_encapsulation=true") } func TestEnableOVNIPSec_MissingCert(t *testing.T) { @@ -64,10 +149,9 @@ func TestEnableOVNIPSec_MissingCert(t *testing.T) { configDir := t.TempDir() configPath := filepath.Join(configDir, "spinifex.toml") require.NoError(t, os.WriteFile(configPath, []byte("placeholder"), 0600)) - // CA exists; peer cert missing. require.NoError(t, os.WriteFile(filepath.Join(configDir, "ca.pem"), []byte("x"), 0600)) - d := &Daemon{configPath: configPath} + d := &Daemon{configPath: configPath, clusterConfig: multiNodeClusterConfig()} err := d.enableOVNIPSec() require.Error(t, err) assert.Contains(t, err.Error(), "missing IPsec credential")