mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
ci(e2e): Phase 1 diag — tcpdump br-wan + bridge fdb + hairpin modes (siv-167)
Mulga-siv-167 Phase 1. Adds evidence-capture for the LB single-host return-path drop class: e2e_single only: - New step 'Start Packet Capture' between Bootstrap and Run E2E Tests. SSHes to test VM and starts background tcpdump on br-wan capturing TCP/22/80/443/9999 + ARP into /tmp/spinifex-diag/br-wan.pcap. PID stored at /tmp/spinifex-diag/tcpdump.pid. All three jobs (single, pseudo-multi, real-multi): - Collect Diagnostics: kill tcpdump (single only — PID file guarded) -INT for graceful flush, falls through to -KILL after 5s. - Adds journal-spinifex-full.log (no -n truncation) alongside the existing -n 10000 capped one. - Adds bridge fdb show, bridge link, and hairpin_mode/learning/flood for every br-wan brif — needed to confirm/disconfirm the hairpin-loss hypothesis in the plan doc. - Adds 'chmod -R a+r /tmp/spinifex-diag' before tar to ensure pcap (created by sudo tcpdump as root) is readable by tf-user. bash -n verified on all four rendered scripts. YAML parses. Zero new gold-image deps (tcpdump, bridge present in base image).
This commit is contained in:
@@ -77,6 +77,31 @@ jobs:
|
||||
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
||||
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
|
||||
|
||||
- name: Start Packet Capture (siv-167)
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
run: |
|
||||
INVENTORY=$(tofu output -json inventory)
|
||||
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
||||
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
||||
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
||||
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' <<'REMOTE'
|
||||
set -x
|
||||
sudo mkdir -p /tmp/spinifex-diag
|
||||
sudo rm -f /tmp/spinifex-diag/br-wan.pcap /tmp/spinifex-diag/tcpdump.pid
|
||||
sudo nohup tcpdump -i br-wan -nn -s 0 -U \
|
||||
-w /tmp/spinifex-diag/br-wan.pcap \
|
||||
'(tcp port 22 or tcp port 80 or tcp port 443 or tcp port 9999) or arp' \
|
||||
> /tmp/spinifex-diag/tcpdump.log 2>&1 &
|
||||
echo $! | sudo tee /tmp/spinifex-diag/tcpdump.pid
|
||||
sleep 1
|
||||
if sudo kill -0 "$(cat /tmp/spinifex-diag/tcpdump.pid)" 2>/dev/null; then
|
||||
echo "tcpdump running PID=$(cat /tmp/spinifex-diag/tcpdump.pid)"
|
||||
else
|
||||
echo "tcpdump failed to start"
|
||||
sudo cat /tmp/spinifex-diag/tcpdump.log
|
||||
fi
|
||||
REMOTE
|
||||
|
||||
- name: Run E2E Tests
|
||||
working-directory: mulga/scripts/tofu-cluster
|
||||
run: |
|
||||
@@ -100,7 +125,14 @@ jobs:
|
||||
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' > "$DIAG_DIR/collect.log" 2>&1 <<'REMOTE' || true
|
||||
set -x
|
||||
DIAG=/tmp/spinifex-diag
|
||||
sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG"
|
||||
sudo mkdir -p "$DIAG" && cd "$DIAG"
|
||||
if [ -f tcpdump.pid ]; then
|
||||
PID=$(cat tcpdump.pid)
|
||||
sudo kill -INT "$PID" 2>/dev/null || true
|
||||
for _ in 1 2 3 4 5; do sudo kill -0 "$PID" 2>/dev/null || break; sleep 1; done
|
||||
sudo kill -KILL "$PID" 2>/dev/null || true
|
||||
fi
|
||||
sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
|
||||
sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
|
||||
sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
|
||||
sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
|
||||
@@ -120,11 +152,17 @@ jobs:
|
||||
sudo ip addr > ip-addr.txt 2>&1 || true
|
||||
sudo ip route > ip-route.txt 2>&1 || true
|
||||
sudo ip neigh > ip-neigh.txt 2>&1 || true
|
||||
sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
|
||||
sudo bridge link > bridge-link.txt 2>&1 || true
|
||||
for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
|
||||
[ -e "$f" ] && echo "$f $(cat $f)"
|
||||
done > hairpin-modes.txt 2>&1 || true
|
||||
sudo iptables-save > iptables.txt 2>&1 || true
|
||||
sudo nft list ruleset > nft.txt 2>&1 || true
|
||||
sudo dmesg --ctime > dmesg.txt 2>&1 || true
|
||||
sudo virsh list --all > virsh-list.txt 2>&1 || true
|
||||
ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true
|
||||
sudo chmod -R a+r /tmp/spinifex-diag
|
||||
tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag
|
||||
REMOTE
|
||||
scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \
|
||||
@@ -217,6 +255,7 @@ jobs:
|
||||
set -x
|
||||
DIAG=/tmp/spinifex-diag
|
||||
sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG"
|
||||
sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
|
||||
sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
|
||||
sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
|
||||
sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
|
||||
@@ -236,6 +275,11 @@ jobs:
|
||||
sudo ip addr > ip-addr.txt 2>&1 || true
|
||||
sudo ip route > ip-route.txt 2>&1 || true
|
||||
sudo ip neigh > ip-neigh.txt 2>&1 || true
|
||||
sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
|
||||
sudo bridge link > bridge-link.txt 2>&1 || true
|
||||
for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
|
||||
[ -e "$f" ] && echo "$f $(cat $f)"
|
||||
done > hairpin-modes.txt 2>&1 || true
|
||||
sudo iptables-save > iptables.txt 2>&1 || true
|
||||
sudo nft list ruleset > nft.txt 2>&1 || true
|
||||
sudo dmesg --ctime > dmesg.txt 2>&1 || true
|
||||
@@ -347,6 +391,7 @@ jobs:
|
||||
set -x
|
||||
DIAG=/tmp/spinifex-diag
|
||||
sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG"
|
||||
sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
|
||||
sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
|
||||
sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
|
||||
sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
|
||||
@@ -366,6 +411,11 @@ jobs:
|
||||
sudo ip addr > ip-addr.txt 2>&1 || true
|
||||
sudo ip route > ip-route.txt 2>&1 || true
|
||||
sudo ip neigh > ip-neigh.txt 2>&1 || true
|
||||
sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
|
||||
sudo bridge link > bridge-link.txt 2>&1 || true
|
||||
for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
|
||||
[ -e "$f" ] && echo "$f $(cat $f)"
|
||||
done > hairpin-modes.txt 2>&1 || true
|
||||
sudo iptables-save > iptables.txt 2>&1 || true
|
||||
sudo nft list ruleset > nft.txt 2>&1 || true
|
||||
sudo dmesg --ctime > dmesg.txt 2>&1 || true
|
||||
|
||||
Reference in New Issue
Block a user