From afd63ed967ea70728f40eb92f67c3900fa3f8129 Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Tue, 2 Jun 2026 19:28:54 +1000 Subject: [PATCH] =?UTF-8?q?ci(e2e):=20Phase=201=20diag=20=E2=80=94=20tcpdu?= =?UTF-8?q?mp=20br-wan=20+=20bridge=20fdb=20+=20hairpin=20modes=20(siv-167?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mulga-siv-167 Phase 1. Adds evidence-capture for the LB single-host return-path drop class: e2e_single only: - New step 'Start Packet Capture' between Bootstrap and Run E2E Tests. SSHes to test VM and starts background tcpdump on br-wan capturing TCP/22/80/443/9999 + ARP into /tmp/spinifex-diag/br-wan.pcap. PID stored at /tmp/spinifex-diag/tcpdump.pid. All three jobs (single, pseudo-multi, real-multi): - Collect Diagnostics: kill tcpdump (single only — PID file guarded) -INT for graceful flush, falls through to -KILL after 5s. - Adds journal-spinifex-full.log (no -n truncation) alongside the existing -n 10000 capped one. - Adds bridge fdb show, bridge link, and hairpin_mode/learning/flood for every br-wan brif — needed to confirm/disconfirm the hairpin-loss hypothesis in the plan doc. - Adds 'chmod -R a+r /tmp/spinifex-diag' before tar to ensure pcap (created by sudo tcpdump as root) is readable by tf-user. bash -n verified on all four rendered scripts. YAML parses. Zero new gold-image deps (tcpdump, bridge present in base image). --- .github/workflows/e2e.yml | 52 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1bf5565aa..a10825f37 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -77,6 +77,31 @@ jobs: "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" + - name: Start Packet Capture (siv-167) + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory) + WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' <<'REMOTE' + set -x + sudo mkdir -p /tmp/spinifex-diag + sudo rm -f /tmp/spinifex-diag/br-wan.pcap /tmp/spinifex-diag/tcpdump.pid + sudo nohup tcpdump -i br-wan -nn -s 0 -U \ + -w /tmp/spinifex-diag/br-wan.pcap \ + '(tcp port 22 or tcp port 80 or tcp port 443 or tcp port 9999) or arp' \ + > /tmp/spinifex-diag/tcpdump.log 2>&1 & + echo $! | sudo tee /tmp/spinifex-diag/tcpdump.pid + sleep 1 + if sudo kill -0 "$(cat /tmp/spinifex-diag/tcpdump.pid)" 2>/dev/null; then + echo "tcpdump running PID=$(cat /tmp/spinifex-diag/tcpdump.pid)" + else + echo "tcpdump failed to start" + sudo cat /tmp/spinifex-diag/tcpdump.log + fi + REMOTE + - name: Run E2E Tests working-directory: mulga/scripts/tofu-cluster run: | @@ -100,7 +125,14 @@ jobs: ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' > "$DIAG_DIR/collect.log" 2>&1 <<'REMOTE' || true set -x DIAG=/tmp/spinifex-diag - sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG" + sudo mkdir -p "$DIAG" && cd "$DIAG" + if [ -f tcpdump.pid ]; then + PID=$(cat tcpdump.pid) + sudo kill -INT "$PID" 2>/dev/null || true + for _ in 1 2 3 4 5; do sudo kill -0 "$PID" 2>/dev/null || break; sleep 1; done + sudo kill -KILL "$PID" 2>/dev/null || true + fi + sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true @@ -120,11 +152,17 @@ jobs: sudo ip addr > ip-addr.txt 2>&1 || true sudo ip route > ip-route.txt 2>&1 || true sudo ip neigh > ip-neigh.txt 2>&1 || true + sudo bridge fdb show > bridge-fdb.txt 2>&1 || true + sudo bridge link > bridge-link.txt 2>&1 || true + for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do + [ -e "$f" ] && echo "$f $(cat $f)" + done > hairpin-modes.txt 2>&1 || true sudo iptables-save > iptables.txt 2>&1 || true sudo nft list ruleset > nft.txt 2>&1 || true sudo dmesg --ctime > dmesg.txt 2>&1 || true sudo virsh list --all > virsh-list.txt 2>&1 || true ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true + sudo chmod -R a+r /tmp/spinifex-diag tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag REMOTE scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \ @@ -217,6 +255,7 @@ jobs: set -x DIAG=/tmp/spinifex-diag sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG" + sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true @@ -236,6 +275,11 @@ jobs: sudo ip addr > ip-addr.txt 2>&1 || true sudo ip route > ip-route.txt 2>&1 || true sudo ip neigh > ip-neigh.txt 2>&1 || true + sudo bridge fdb show > bridge-fdb.txt 2>&1 || true + sudo bridge link > bridge-link.txt 2>&1 || true + for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do + [ -e "$f" ] && echo "$f $(cat $f)" + done > hairpin-modes.txt 2>&1 || true sudo iptables-save > iptables.txt 2>&1 || true sudo nft list ruleset > nft.txt 2>&1 || true sudo dmesg --ctime > dmesg.txt 2>&1 || true @@ -347,6 +391,7 @@ jobs: set -x DIAG=/tmp/spinifex-diag sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG" + sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true @@ -366,6 +411,11 @@ jobs: sudo ip addr > ip-addr.txt 2>&1 || true sudo ip route > ip-route.txt 2>&1 || true sudo ip neigh > ip-neigh.txt 2>&1 || true + sudo bridge fdb show > bridge-fdb.txt 2>&1 || true + sudo bridge link > bridge-link.txt 2>&1 || true + for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do + [ -e "$f" ] && echo "$f $(cat $f)" + done > hairpin-modes.txt 2>&1 || true sudo iptables-save > iptables.txt 2>&1 || true sudo nft list ruleset > nft.txt 2>&1 || true sudo dmesg --ctime > dmesg.txt 2>&1 || true