ci(e2e): Phase 1 diag — tcpdump br-wan + bridge fdb + hairpin modes (siv-167)

Mulga-siv-167 Phase 1. Adds evidence-capture for the LB
single-host return-path drop class:

e2e_single only:
- New step 'Start Packet Capture' between Bootstrap and Run E2E
  Tests. SSHes to test VM and starts background tcpdump on br-wan
  capturing TCP/22/80/443/9999 + ARP into /tmp/spinifex-diag/br-wan.pcap.
  PID stored at /tmp/spinifex-diag/tcpdump.pid.

All three jobs (single, pseudo-multi, real-multi):
- Collect Diagnostics: kill tcpdump (single only — PID file
  guarded) -INT for graceful flush, falls through to -KILL after 5s.
- Adds journal-spinifex-full.log (no -n truncation) alongside
  the existing -n 10000 capped one.
- Adds bridge fdb show, bridge link, and hairpin_mode/learning/flood
  for every br-wan brif — needed to confirm/disconfirm the
  hairpin-loss hypothesis in the plan doc.
- Adds 'chmod -R a+r /tmp/spinifex-diag' before tar to ensure pcap
  (created by sudo tcpdump as root) is readable by tf-user.

bash -n verified on all four rendered scripts. YAML parses.
Zero new gold-image deps (tcpdump, bridge present in base image).
This commit is contained in:
joshsiv-mulga
2026-06-02 19:28:54 +10:00
parent 2574956f08
commit afd63ed967
+51 -1
View File
@@ -77,6 +77,31 @@ jobs:
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
- name: Start Packet Capture (siv-167)
working-directory: mulga/scripts/tofu-cluster
run: |
INVENTORY=$(tofu output -json inventory)
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
SSH_KEY="${SSH_KEY/#\~/$HOME}"
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' <<'REMOTE'
set -x
sudo mkdir -p /tmp/spinifex-diag
sudo rm -f /tmp/spinifex-diag/br-wan.pcap /tmp/spinifex-diag/tcpdump.pid
sudo nohup tcpdump -i br-wan -nn -s 0 -U \
-w /tmp/spinifex-diag/br-wan.pcap \
'(tcp port 22 or tcp port 80 or tcp port 443 or tcp port 9999) or arp' \
> /tmp/spinifex-diag/tcpdump.log 2>&1 &
echo $! | sudo tee /tmp/spinifex-diag/tcpdump.pid
sleep 1
if sudo kill -0 "$(cat /tmp/spinifex-diag/tcpdump.pid)" 2>/dev/null; then
echo "tcpdump running PID=$(cat /tmp/spinifex-diag/tcpdump.pid)"
else
echo "tcpdump failed to start"
sudo cat /tmp/spinifex-diag/tcpdump.log
fi
REMOTE
- name: Run E2E Tests
working-directory: mulga/scripts/tofu-cluster
run: |
@@ -100,7 +125,14 @@ jobs:
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' > "$DIAG_DIR/collect.log" 2>&1 <<'REMOTE' || true
set -x
DIAG=/tmp/spinifex-diag
sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG"
sudo mkdir -p "$DIAG" && cd "$DIAG"
if [ -f tcpdump.pid ]; then
PID=$(cat tcpdump.pid)
sudo kill -INT "$PID" 2>/dev/null || true
for _ in 1 2 3 4 5; do sudo kill -0 "$PID" 2>/dev/null || break; sleep 1; done
sudo kill -KILL "$PID" 2>/dev/null || true
fi
sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
@@ -120,11 +152,17 @@ jobs:
sudo ip addr > ip-addr.txt 2>&1 || true
sudo ip route > ip-route.txt 2>&1 || true
sudo ip neigh > ip-neigh.txt 2>&1 || true
sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
sudo bridge link > bridge-link.txt 2>&1 || true
for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
[ -e "$f" ] && echo "$f $(cat $f)"
done > hairpin-modes.txt 2>&1 || true
sudo iptables-save > iptables.txt 2>&1 || true
sudo nft list ruleset > nft.txt 2>&1 || true
sudo dmesg --ctime > dmesg.txt 2>&1 || true
sudo virsh list --all > virsh-list.txt 2>&1 || true
ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true
sudo chmod -R a+r /tmp/spinifex-diag
tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag
REMOTE
scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \
@@ -217,6 +255,7 @@ jobs:
set -x
DIAG=/tmp/spinifex-diag
sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG"
sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
@@ -236,6 +275,11 @@ jobs:
sudo ip addr > ip-addr.txt 2>&1 || true
sudo ip route > ip-route.txt 2>&1 || true
sudo ip neigh > ip-neigh.txt 2>&1 || true
sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
sudo bridge link > bridge-link.txt 2>&1 || true
for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
[ -e "$f" ] && echo "$f $(cat $f)"
done > hairpin-modes.txt 2>&1 || true
sudo iptables-save > iptables.txt 2>&1 || true
sudo nft list ruleset > nft.txt 2>&1 || true
sudo dmesg --ctime > dmesg.txt 2>&1 || true
@@ -347,6 +391,7 @@ jobs:
set -x
DIAG=/tmp/spinifex-diag
sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG"
sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
@@ -366,6 +411,11 @@ jobs:
sudo ip addr > ip-addr.txt 2>&1 || true
sudo ip route > ip-route.txt 2>&1 || true
sudo ip neigh > ip-neigh.txt 2>&1 || true
sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
sudo bridge link > bridge-link.txt 2>&1 || true
for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
[ -e "$f" ] && echo "$f $(cat $f)"
done > hairpin-modes.txt 2>&1 || true
sudo iptables-save > iptables.txt 2>&1 || true
sudo nft list ruleset > nft.txt 2>&1 || true
sudo dmesg --ctime > dmesg.txt 2>&1 || true