diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1bf5565aa..a10825f37 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -77,6 +77,31 @@ jobs: "${RUNNER_TEMP}/spinifex-distro.tar.gz" \ --test-tar "${RUNNER_TEMP}/spinifex-tests.tar" + - name: Start Packet Capture (siv-167) + working-directory: mulga/scripts/tofu-cluster + run: | + INVENTORY=$(tofu output -json inventory) + WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip') + SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path') + SSH_KEY="${SSH_KEY/#\~/$HOME}" + ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' <<'REMOTE' + set -x + sudo mkdir -p /tmp/spinifex-diag + sudo rm -f /tmp/spinifex-diag/br-wan.pcap /tmp/spinifex-diag/tcpdump.pid + sudo nohup tcpdump -i br-wan -nn -s 0 -U \ + -w /tmp/spinifex-diag/br-wan.pcap \ + '(tcp port 22 or tcp port 80 or tcp port 443 or tcp port 9999) or arp' \ + > /tmp/spinifex-diag/tcpdump.log 2>&1 & + echo $! | sudo tee /tmp/spinifex-diag/tcpdump.pid + sleep 1 + if sudo kill -0 "$(cat /tmp/spinifex-diag/tcpdump.pid)" 2>/dev/null; then + echo "tcpdump running PID=$(cat /tmp/spinifex-diag/tcpdump.pid)" + else + echo "tcpdump failed to start" + sudo cat /tmp/spinifex-diag/tcpdump.log + fi + REMOTE + - name: Run E2E Tests working-directory: mulga/scripts/tofu-cluster run: | @@ -100,7 +125,14 @@ jobs: ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' > "$DIAG_DIR/collect.log" 2>&1 <<'REMOTE' || true set -x DIAG=/tmp/spinifex-diag - sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG" + sudo mkdir -p "$DIAG" && cd "$DIAG" + if [ -f tcpdump.pid ]; then + PID=$(cat tcpdump.pid) + sudo kill -INT "$PID" 2>/dev/null || true + for _ in 1 2 3 4 5; do sudo kill -0 "$PID" 2>/dev/null || break; sleep 1; done + sudo kill -KILL "$PID" 2>/dev/null || true + fi + sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true @@ -120,11 +152,17 @@ jobs: sudo ip addr > ip-addr.txt 2>&1 || true sudo ip route > ip-route.txt 2>&1 || true sudo ip neigh > ip-neigh.txt 2>&1 || true + sudo bridge fdb show > bridge-fdb.txt 2>&1 || true + sudo bridge link > bridge-link.txt 2>&1 || true + for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do + [ -e "$f" ] && echo "$f $(cat $f)" + done > hairpin-modes.txt 2>&1 || true sudo iptables-save > iptables.txt 2>&1 || true sudo nft list ruleset > nft.txt 2>&1 || true sudo dmesg --ctime > dmesg.txt 2>&1 || true sudo virsh list --all > virsh-list.txt 2>&1 || true ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true + sudo chmod -R a+r /tmp/spinifex-diag tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag REMOTE scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \ @@ -217,6 +255,7 @@ jobs: set -x DIAG=/tmp/spinifex-diag sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG" + sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true @@ -236,6 +275,11 @@ jobs: sudo ip addr > ip-addr.txt 2>&1 || true sudo ip route > ip-route.txt 2>&1 || true sudo ip neigh > ip-neigh.txt 2>&1 || true + sudo bridge fdb show > bridge-fdb.txt 2>&1 || true + sudo bridge link > bridge-link.txt 2>&1 || true + for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do + [ -e "$f" ] && echo "$f $(cat $f)" + done > hairpin-modes.txt 2>&1 || true sudo iptables-save > iptables.txt 2>&1 || true sudo nft list ruleset > nft.txt 2>&1 || true sudo dmesg --ctime > dmesg.txt 2>&1 || true @@ -347,6 +391,7 @@ jobs: set -x DIAG=/tmp/spinifex-diag sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG" + sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true @@ -366,6 +411,11 @@ jobs: sudo ip addr > ip-addr.txt 2>&1 || true sudo ip route > ip-route.txt 2>&1 || true sudo ip neigh > ip-neigh.txt 2>&1 || true + sudo bridge fdb show > bridge-fdb.txt 2>&1 || true + sudo bridge link > bridge-link.txt 2>&1 || true + for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do + [ -e "$f" ] && echo "$f $(cat $f)" + done > hairpin-modes.txt 2>&1 || true sudo iptables-save > iptables.txt 2>&1 || true sudo nft list ruleset > nft.txt 2>&1 || true sudo dmesg --ctime > dmesg.txt 2>&1 || true