mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 19:01:42 +00:00
fix(ipsec): guard ipsec_encapsulation on ovs-monitor-ipsec liveness
enableOVNIPSec set OVS cert pointers + ipsec_encapsulation=true without ensuring ovs-monitor-ipsec.service was running. Without that daemon, OVS holds the cert pointers unused, strongSwan never gets IKE config, and tunnel traffic ovn-controller marks for encryption gets silently dropped. TestMultinodeIPSec fails on `node1 xfrm has no AEAD SAs`. - systemd: spinifex-daemon Wants=/After= ovs-monitor-ipsec.service - daemon: ensureOVSMonitorIPSecActive — systemctl enable --now + is-active poll; refuses to flip ipsec_encapsulation if daemon is dead - daemon: single-node short-circuit — no peers means no tunnels means the flag is at best inert and at worst load-bearing-silent-drop - ci: capture-net-diag.sh + workflow steps to snapshot OVS/OVN/xfrm state on failure, fan out across all 3 multinode nodes Refs: mulga-siv-136
This commit is contained in:
Executable
+55
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
# capture-net-diag.sh — write a snapshot of network / OVN / OVS / IPsec
|
||||
# state into $1 (a directory the caller already created on the local host).
|
||||
# Designed to run on a spinifex node after an E2E suite, before tofu destroy
|
||||
# pulls the rug out. Every command is independent — `|| true` so a missing
|
||||
# binary on one path doesn't abort the rest of the capture.
|
||||
#
|
||||
# Output layout (under $1/net-diag/):
|
||||
# ovs-monitor-ipsec.status systemctl status (output-only)
|
||||
# ovs-monitor-ipsec.journal last 200 journal lines
|
||||
# ipsec.openvswitch_conf ovs-vsctl get Open_vSwitch . other_config
|
||||
# ipsec.xfrm_state ip xfrm state list
|
||||
# ipsec.xfrm_policy ip xfrm policy list
|
||||
# ovn.sb_show ovn-sbctl show
|
||||
# ovn.nb_show ovn-nbctl show
|
||||
# ovs.show ovs-vsctl show
|
||||
# ovs.flows_br_int ovs-ofctl dump-flows br-int
|
||||
# ovs.tunnels ovs-appctl ofproto/list-tunnels br-int
|
||||
# net.links ip -s link show
|
||||
# net.routes ip route show table all
|
||||
# net.addrs ip -br addr show
|
||||
# systemd.failed_units systemctl list-units --state=failed
|
||||
set +e
|
||||
OUT="${1:?usage: capture-net-diag.sh <output-dir>}"
|
||||
mkdir -p "$OUT"
|
||||
|
||||
run() {
|
||||
local label="$1"; shift
|
||||
{
|
||||
echo "# $*"
|
||||
"$@" 2>&1
|
||||
} > "$OUT/$label" || true
|
||||
}
|
||||
|
||||
run ovs-monitor-ipsec.status sudo systemctl status ovs-monitor-ipsec.service --no-pager --full --lines=50
|
||||
run ovs-monitor-ipsec.journal sudo journalctl -u ovs-monitor-ipsec.service --no-pager --output=short-iso -n 200
|
||||
run ipsec.openvswitch_conf sudo ovs-vsctl get Open_vSwitch . other_config
|
||||
run ipsec.xfrm_state sudo ip xfrm state list
|
||||
run ipsec.xfrm_policy sudo ip xfrm policy list
|
||||
|
||||
run ovn.sb_show sudo ovn-sbctl show
|
||||
run ovn.nb_show sudo ovn-nbctl show
|
||||
run ovn.sb_chassis sudo ovn-sbctl list chassis
|
||||
|
||||
run ovs.show sudo ovs-vsctl show
|
||||
run ovs.flows_br_int sudo ovs-ofctl dump-flows br-int
|
||||
run ovs.tunnels sudo ovs-appctl ofproto/list-tunnels br-int
|
||||
|
||||
run net.links ip -s link show
|
||||
run net.routes ip route show table all
|
||||
run net.addrs ip -br addr show
|
||||
|
||||
run systemd.failed_units sudo systemctl list-units --state=failed --no-pager
|
||||
|
||||
exit 0
|
||||
@@ -277,6 +277,23 @@ jobs:
|
||||
echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
|
||||
exit 0 # don't fail the step — artifact upload + final gate happen below
|
||||
|
||||
- name: Capture network/IPsec diagnostics on VM
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
# shellcheck source=/dev/null
|
||||
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
||||
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.vm.outputs.ssh_key }}")
|
||||
VM=tf-user@${{ steps.vm.outputs.wan_ip }}
|
||||
# Ship the diag script, run it into the VM's ARTIFACT_DIR, leave the
|
||||
# output for the next step's tar to pull down. Captures OVN/OVS/IPsec
|
||||
# state at failure time — before teardown destroys the VM.
|
||||
scp "${SSH_OPTS[@]}" \
|
||||
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
|
||||
"$VM":/tmp/capture-net-diag.sh
|
||||
ssh "${SSH_OPTS[@]}" "$VM" "bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag'"
|
||||
ok "net diagnostics captured"
|
||||
|
||||
- name: Collect artifacts from VM
|
||||
if: always()
|
||||
run: |
|
||||
@@ -653,6 +670,42 @@ jobs:
|
||||
echo "rc=${E2E_RC}" >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
|
||||
- name: Capture network/IPsec diagnostics on all nodes
|
||||
if: always()
|
||||
run: |
|
||||
set +e
|
||||
# shellcheck source=/dev/null
|
||||
source "${GITHUB_WORKSPACE}/spinifex/.github/scripts/ci-fmt.sh"
|
||||
SSH_OPTS=(-o StrictHostKeyChecking=no -i "${{ steps.cluster.outputs.ssh_key }}")
|
||||
# Capture per-node into the node1 ARTIFACT_DIR (where Collect pulls
|
||||
# from). One subdir per node so xfrm SA state is comparable across
|
||||
# peers — critical for diagnosing the IPsec wiring (mulga-siv-136).
|
||||
IFS=',' read -ra NODE_LIST <<< "${{ steps.cluster.outputs.nodes_csv }}"
|
||||
NODE1_IP="${NODE_LIST[0]}"
|
||||
scp "${SSH_OPTS[@]}" \
|
||||
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
|
||||
tf-user@"$NODE1_IP":/tmp/capture-net-diag.sh
|
||||
for i in "${!NODE_LIST[@]}"; do
|
||||
idx=$((i + 1))
|
||||
ip="${NODE_LIST[$i]}"
|
||||
if [ "$idx" -ne 1 ]; then
|
||||
scp "${SSH_OPTS[@]}" \
|
||||
"${GITHUB_WORKSPACE}/spinifex/.github/scripts/capture-net-diag.sh" \
|
||||
tf-user@"$ip":/tmp/capture-net-diag.sh
|
||||
fi
|
||||
ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
|
||||
"bash /tmp/capture-net-diag.sh '${ARTIFACT_DIR}/net-diag-node${idx}'"
|
||||
# node1 keeps its diag locally; pull node2..N back into node1's
|
||||
# ARTIFACT_DIR so the existing tar-pull below catches everything.
|
||||
if [ "$idx" -ne 1 ]; then
|
||||
ssh "${SSH_OPTS[@]}" tf-user@"$ip" \
|
||||
"tar -C '${ARTIFACT_DIR}' -cf - 'net-diag-node${idx}' 2>/dev/null || true" \
|
||||
| ssh "${SSH_OPTS[@]}" tf-user@"$NODE1_IP" \
|
||||
"tar -C '${ARTIFACT_DIR}' -xf - 2>/dev/null"
|
||||
fi
|
||||
ok "net diagnostics node${idx} captured"
|
||||
done
|
||||
|
||||
- name: Collect artifacts from node1
|
||||
if: always()
|
||||
run: |
|
||||
@@ -675,6 +728,13 @@ jobs:
|
||||
> "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null
|
||||
ok "journal node${idx}: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal-node${idx}.log" 2>/dev/null || echo 0) lines"
|
||||
done
|
||||
# ovs-monitor-ipsec journal per node (separate file — bead 136).
|
||||
for i in "${!NODE_LIST[@]}"; do
|
||||
idx=$((i + 1))
|
||||
ip="${NODE_LIST[$i]}"
|
||||
ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u ovs-monitor-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
|
||||
> "${ARTIFACT_DIR}/ovs-monitor-ipsec-node${idx}.log" 2>/dev/null
|
||||
done
|
||||
|
||||
- name: Convert test logs to JUnit XML
|
||||
if: always()
|
||||
|
||||
Reference in New Issue
Block a user