mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 10:52:04 +00:00
joshsiv-mulga
490 lines
22 KiB
YAML
490 lines
22 KiB
YAML
name: E2E
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
- dev
|
|
paths:
|
|
- "**/*.go"
|
|
- go.mod
|
|
- go.sum
|
|
- Makefile
|
|
- tests/e2e/**
|
|
- .github/workflows/e2e.yml
|
|
workflow_dispatch:
|
|
inputs:
|
|
sha:
|
|
description: "PR head SHA to report status on"
|
|
required: false
|
|
|
|
permissions:
|
|
contents: read
|
|
statuses: write
|
|
|
|
env:
|
|
GIT_BRANCH: ${{ github.ref_name }}
|
|
|
|
jobs:
|
|
e2e_single:
|
|
name: Single-Node E2E
|
|
runs-on: [ self-hosted, ci-single ]
|
|
timeout-minutes: 35
|
|
env:
|
|
TOFU_LOG: /tmp/e2e-tofu-single-${{ github.run_id }}.log
|
|
steps:
|
|
- name: Clean Workspace
|
|
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
|
|
|
|
- uses: actions/create-github-app-token@v3
|
|
id: app-token
|
|
with:
|
|
client-id: ${{ secrets.APP_ID }}
|
|
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
|
repositories: mulga
|
|
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
repository: mulgadc/mulga
|
|
token: ${{ steps.app-token.outputs.token }}
|
|
sparse-checkout: scripts/tofu-cluster
|
|
path: mulga
|
|
|
|
- name: Build Install Artifacts
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
./build-install-artifacts.sh "${GIT_BRANCH}" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Provision VM
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
export TF_VAR_git_branch="${GIT_BRANCH}"
|
|
tofu init -input=false >> "$TOFU_LOG" 2>&1
|
|
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
|
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
|
|
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
|
tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
|
|
|
- name: Bootstrap via Binary Install
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
|
|
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Start Packet Capture (siv-167)
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' <<'REMOTE'
|
|
set -x
|
|
sudo mkdir -p /tmp/spinifex-diag
|
|
sudo chmod 0777 /tmp/spinifex-diag
|
|
sudo rm -f /tmp/spinifex-diag/br-wan.pcap /tmp/spinifex-diag/tcpdump.pid /tmp/spinifex-diag/tcpdump.log
|
|
sudo bash -c '
|
|
nohup tcpdump -i br-wan -nn -s 0 -U \
|
|
-w /tmp/spinifex-diag/br-wan.pcap \
|
|
"(tcp port 22 or tcp port 80 or tcp port 443 or tcp port 9999) or arp" \
|
|
</dev/null >/tmp/spinifex-diag/tcpdump.log 2>&1 &
|
|
echo $! > /tmp/spinifex-diag/tcpdump.pid
|
|
'
|
|
sleep 2
|
|
PID=$(cat /tmp/spinifex-diag/tcpdump.pid 2>/dev/null || echo "")
|
|
if [ -n "$PID" ] && sudo kill -0 "$PID" 2>/dev/null; then
|
|
echo "tcpdump running PID=$PID"
|
|
ls -la /tmp/spinifex-diag/
|
|
else
|
|
echo "tcpdump failed to start"
|
|
cat /tmp/spinifex-diag/tcpdump.log 2>&1 || true
|
|
exit 1
|
|
fi
|
|
REMOTE
|
|
|
|
- name: Run E2E Tests
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
|
|
"cd ~/e2e-tests && ./run-e2e.sh && ./run-cert-e2e.sh && ENDPOINT=https://${WAN_IP}:9999 ./run-lb-e2e.sh"
|
|
|
|
- name: Collect Diagnostics
|
|
if: always()
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
DIAG_DIR="${RUNNER_TEMP}/diag-single"
|
|
mkdir -p "$DIAG_DIR"
|
|
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' > "$DIAG_DIR/collect.log" 2>&1 <<'REMOTE' || true
|
|
set -x
|
|
DIAG=/tmp/spinifex-diag
|
|
sudo mkdir -p "$DIAG" && cd "$DIAG"
|
|
if [ -f tcpdump.pid ]; then
|
|
PID=$(cat tcpdump.pid)
|
|
sudo kill -INT "$PID" 2>/dev/null || true
|
|
for _ in 1 2 3 4 5; do sudo kill -0 "$PID" 2>/dev/null || break; sleep 1; done
|
|
sudo kill -KILL "$PID" 2>/dev/null || true
|
|
fi
|
|
sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
|
|
sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
|
|
sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
|
|
sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
|
|
sudo ovs-vsctl show > ovs-vsctl-show.txt 2>&1 || true
|
|
sudo ovs-ofctl dump-flows br-int > br-int-flows.txt 2>&1 || true
|
|
sudo ovs-ofctl dump-flows br-ex > br-ex-flows.txt 2>&1 || true
|
|
sudo ovs-appctl dpctl/dump-flows > dpctl-flows.txt 2>&1 || true
|
|
sudo ovs-appctl dpctl/dump-conntrack -m > dpctl-conntrack.txt 2>&1 || true
|
|
sudo ovn-nbctl show > ovn-nbctl-show.txt 2>&1 || true
|
|
sudo ovn-sbctl show > ovn-sbctl-show.txt 2>&1 || true
|
|
sudo ovn-sbctl list Port_Binding > sbdb-port-binding.txt 2>&1 || true
|
|
sudo ovn-sbctl list Chassis > sbdb-chassis.txt 2>&1 || true
|
|
sudo ovn-nbctl list NAT > nbdb-nat.txt 2>&1 || true
|
|
sudo ovn-nbctl list Logical_Router > nbdb-lr.txt 2>&1 || true
|
|
sudo ovn-nbctl list Logical_Router_Policy > nbdb-lr-policy.txt 2>&1 || true
|
|
sudo ovn-nbctl list Logical_Router_Static_Route > nbdb-lr-static-route.txt 2>&1 || true
|
|
sudo ovn-appctl -t ovn-controller ct-zone-list > ovn-ctzones.txt 2>&1 || true
|
|
sudo ip -d link > ip-link.txt 2>&1 || true
|
|
sudo ip addr > ip-addr.txt 2>&1 || true
|
|
sudo ip route > ip-route.txt 2>&1 || true
|
|
sudo ip neigh > ip-neigh.txt 2>&1 || true
|
|
sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
|
|
sudo bridge link > bridge-link.txt 2>&1 || true
|
|
for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
|
|
[ -e "$f" ] && echo "$f $(cat $f)"
|
|
done > hairpin-modes.txt 2>&1 || true
|
|
sudo iptables-save > iptables.txt 2>&1 || true
|
|
sudo nft list ruleset > nft.txt 2>&1 || true
|
|
sudo dmesg --ctime > dmesg.txt 2>&1 || true
|
|
sudo virsh list --all > virsh-list.txt 2>&1 || true
|
|
ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true
|
|
sudo chmod -R a+r /tmp/spinifex-diag
|
|
tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag
|
|
REMOTE
|
|
scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \
|
|
tf-user@${WAN_IP}:/tmp/spinifex-diag.tar.gz "$DIAG_DIR/spinifex-diag.tar.gz" || true
|
|
|
|
- name: Upload Diagnostics
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: e2e-diag-single-${{ github.run_id }}
|
|
path: ${{ runner.temp }}/diag-single/
|
|
if-no-files-found: warn
|
|
retention-days: 14
|
|
|
|
- name: Teardown
|
|
if: always()
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
|
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
|
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
|
|
|
e2e_pseudo_multi:
|
|
name: Pseudo Multi-Node E2E
|
|
runs-on: [ self-hosted, ci-pseudo ]
|
|
timeout-minutes: 25
|
|
env:
|
|
TOFU_LOG: /tmp/e2e-tofu-pseudo-${{ github.run_id }}.log
|
|
steps:
|
|
- name: Clean Workspace
|
|
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
|
|
|
|
- uses: actions/create-github-app-token@v3
|
|
id: app-token
|
|
with:
|
|
client-id: ${{ secrets.APP_ID }}
|
|
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
|
repositories: mulga
|
|
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
repository: mulgadc/mulga
|
|
token: ${{ steps.app-token.outputs.token }}
|
|
sparse-checkout: scripts/tofu-cluster
|
|
path: mulga
|
|
|
|
- name: Clone and Build
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: ./build-artifacts.sh "${GIT_BRANCH}"
|
|
"${RUNNER_TEMP}/spinifex-artifacts.tar"
|
|
|
|
- name: Provision VM
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
export TF_VAR_git_branch="${GIT_BRANCH}"
|
|
tofu init -input=false >> "$TOFU_LOG" 2>&1
|
|
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
|
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
|
|
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
|
tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
|
|
|
- name: Bootstrap Cluster
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
SPX_PREBUILT=true \
|
|
SPX_ARTIFACT_TAR="${RUNNER_TEMP}/spinifex-artifacts.tar" \
|
|
./bootstrap.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
|
|
|
- name: Run E2E Tests
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
|
|
"cd ~/Development/mulga/spinifex && SPX_EXTERNAL_POOL=\"${SPINIFEX_CI_POOL}\" ./tests/e2e/run-pseudo-multinode-e2e.sh"
|
|
|
|
- name: Collect Diagnostics
|
|
if: always()
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
INVENTORY=$(tofu output -json inventory)
|
|
WAN_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
DIAG_DIR="${RUNNER_TEMP}/diag-pseudo"
|
|
mkdir -p "$DIAG_DIR"
|
|
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${WAN_IP} 'bash -s' > "$DIAG_DIR/collect.log" 2>&1 <<'REMOTE' || true
|
|
set -x
|
|
DIAG=/tmp/spinifex-diag
|
|
sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG"
|
|
sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
|
|
sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
|
|
sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
|
|
sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
|
|
sudo ovs-vsctl show > ovs-vsctl-show.txt 2>&1 || true
|
|
sudo ovs-ofctl dump-flows br-int > br-int-flows.txt 2>&1 || true
|
|
sudo ovs-ofctl dump-flows br-ex > br-ex-flows.txt 2>&1 || true
|
|
sudo ovs-appctl dpctl/dump-flows > dpctl-flows.txt 2>&1 || true
|
|
sudo ovs-appctl dpctl/dump-conntrack -m > dpctl-conntrack.txt 2>&1 || true
|
|
sudo ovn-nbctl show > ovn-nbctl-show.txt 2>&1 || true
|
|
sudo ovn-sbctl show > ovn-sbctl-show.txt 2>&1 || true
|
|
sudo ovn-sbctl list Port_Binding > sbdb-port-binding.txt 2>&1 || true
|
|
sudo ovn-sbctl list Chassis > sbdb-chassis.txt 2>&1 || true
|
|
sudo ovn-nbctl list NAT > nbdb-nat.txt 2>&1 || true
|
|
sudo ovn-nbctl list Logical_Router > nbdb-lr.txt 2>&1 || true
|
|
sudo ovn-nbctl list Logical_Router_Policy > nbdb-lr-policy.txt 2>&1 || true
|
|
sudo ovn-nbctl list Logical_Router_Static_Route > nbdb-lr-static-route.txt 2>&1 || true
|
|
sudo ovn-appctl -t ovn-controller ct-zone-list > ovn-ctzones.txt 2>&1 || true
|
|
sudo ip -d link > ip-link.txt 2>&1 || true
|
|
sudo ip addr > ip-addr.txt 2>&1 || true
|
|
sudo ip route > ip-route.txt 2>&1 || true
|
|
sudo ip neigh > ip-neigh.txt 2>&1 || true
|
|
sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
|
|
sudo bridge link > bridge-link.txt 2>&1 || true
|
|
for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
|
|
[ -e "$f" ] && echo "$f $(cat $f)"
|
|
done > hairpin-modes.txt 2>&1 || true
|
|
sudo iptables-save > iptables.txt 2>&1 || true
|
|
sudo nft list ruleset > nft.txt 2>&1 || true
|
|
sudo dmesg --ctime > dmesg.txt 2>&1 || true
|
|
sudo virsh list --all > virsh-list.txt 2>&1 || true
|
|
ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true
|
|
tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag
|
|
REMOTE
|
|
scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \
|
|
tf-user@${WAN_IP}:/tmp/spinifex-diag.tar.gz "$DIAG_DIR/spinifex-diag.tar.gz" || true
|
|
|
|
- name: Upload Diagnostics
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: e2e-diag-pseudo-${{ github.run_id }}
|
|
path: ${{ runner.temp }}/diag-pseudo/
|
|
if-no-files-found: warn
|
|
retention-days: 14
|
|
|
|
- name: Teardown
|
|
if: always()
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
|
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
|
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
|
|
|
e2e_real_multi:
|
|
name: Real Multi-Node E2E
|
|
runs-on: [ self-hosted, ci-multi ]
|
|
timeout-minutes: 30
|
|
env:
|
|
TOFU_LOG: /tmp/e2e-tofu-multi-${{ github.run_id }}.log
|
|
steps:
|
|
- name: Clean Workspace
|
|
run: sudo rm -rf "$GITHUB_WORKSPACE"/*
|
|
|
|
- uses: actions/create-github-app-token@v3
|
|
id: app-token
|
|
with:
|
|
client-id: ${{ secrets.APP_ID }}
|
|
private-key: ${{ secrets.APP_PRIVATE_KEY }}
|
|
repositories: mulga
|
|
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
repository: mulgadc/mulga
|
|
token: ${{ steps.app-token.outputs.token }}
|
|
sparse-checkout: scripts/tofu-cluster
|
|
path: mulga
|
|
|
|
- name: Build Install Artifacts
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
./build-install-artifacts.sh "${GIT_BRANCH}" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Provision VMs
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
export TF_VAR_git_branch="${GIT_BRANCH}"
|
|
tofu init -input=false >> "$TOFU_LOG" 2>&1
|
|
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
|
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1 || true
|
|
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
|
tofu apply -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
|
|
|
- name: Bootstrap via Binary Install
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
SPX_EXTERNAL_POOL="${SPINIFEX_CI_POOL}" \
|
|
./bootstrap-install.sh "${SPINIFEX_CI_ENV}" \
|
|
"${RUNNER_TEMP}/setup.sh" \
|
|
"${RUNNER_TEMP}/spinifex-distro.tar.gz" \
|
|
--test-tar "${RUNNER_TEMP}/spinifex-tests.tar"
|
|
|
|
- name: Run E2E Tests
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
INVENTORY=$(tofu output -json inventory)
|
|
NODE1_IP=$(echo "$INVENTORY" | jq -r '.nodes[0].wan_ip')
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
|
|
# Build node IP list
|
|
NODE_IPS=""
|
|
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
|
|
for i in $(seq 0 $((NODE_COUNT - 1))); do
|
|
NODE_IPS="${NODE_IPS} $(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")"
|
|
done
|
|
|
|
NODE2_IP=$(echo "$INVENTORY" | jq -r '.nodes[1].wan_ip')
|
|
|
|
ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
|
|
"cd ~/e2e-tests && ./run-multinode-e2e.sh ${NODE_IPS} && ./run-cert-e2e.sh --multinode ${NODE_IPS} && ENDPOINT=https://${NODE1_IP}:9999 ./run-lb-e2e.sh --nodes ${NODE_IPS}"
|
|
|
|
- name: Collect Diagnostics
|
|
if: always()
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
INVENTORY=$(tofu output -json inventory)
|
|
SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
|
|
SSH_KEY="${SSH_KEY/#\~/$HOME}"
|
|
DIAG_DIR="${RUNNER_TEMP}/diag-multi"
|
|
mkdir -p "$DIAG_DIR"
|
|
cat > "${RUNNER_TEMP}/collect-diag.sh" <<'REMOTE'
|
|
set -x
|
|
DIAG=/tmp/spinifex-diag
|
|
sudo rm -rf "$DIAG" && mkdir -p "$DIAG" && cd "$DIAG"
|
|
sudo journalctl -u 'spinifex-*' --no-pager > journal-spinifex-full.log 2>&1 || true
|
|
sudo journalctl -u 'spinifex-*' --no-pager -n 10000 > journal-spinifex.log 2>&1 || true
|
|
sudo journalctl -u 'ovn-*' --no-pager -n 5000 > journal-ovn.log 2>&1 || true
|
|
sudo journalctl -u 'openvswitch-*' --no-pager -n 2000 > journal-ovs.log 2>&1 || true
|
|
sudo ovs-vsctl show > ovs-vsctl-show.txt 2>&1 || true
|
|
sudo ovs-ofctl dump-flows br-int > br-int-flows.txt 2>&1 || true
|
|
sudo ovs-ofctl dump-flows br-ex > br-ex-flows.txt 2>&1 || true
|
|
sudo ovs-appctl dpctl/dump-flows > dpctl-flows.txt 2>&1 || true
|
|
sudo ovs-appctl dpctl/dump-conntrack -m > dpctl-conntrack.txt 2>&1 || true
|
|
sudo ovn-nbctl show > ovn-nbctl-show.txt 2>&1 || true
|
|
sudo ovn-sbctl show > ovn-sbctl-show.txt 2>&1 || true
|
|
sudo ovn-sbctl list Port_Binding > sbdb-port-binding.txt 2>&1 || true
|
|
sudo ovn-sbctl list Chassis > sbdb-chassis.txt 2>&1 || true
|
|
sudo ovn-nbctl list NAT > nbdb-nat.txt 2>&1 || true
|
|
sudo ovn-nbctl list Logical_Router > nbdb-lr.txt 2>&1 || true
|
|
sudo ovn-nbctl list Logical_Router_Policy > nbdb-lr-policy.txt 2>&1 || true
|
|
sudo ovn-nbctl list Logical_Router_Static_Route > nbdb-lr-static-route.txt 2>&1 || true
|
|
sudo ovn-appctl -t ovn-controller ct-zone-list > ovn-ctzones.txt 2>&1 || true
|
|
sudo ip -d link > ip-link.txt 2>&1 || true
|
|
sudo ip addr > ip-addr.txt 2>&1 || true
|
|
sudo ip route > ip-route.txt 2>&1 || true
|
|
sudo ip neigh > ip-neigh.txt 2>&1 || true
|
|
sudo bridge fdb show > bridge-fdb.txt 2>&1 || true
|
|
sudo bridge link > bridge-link.txt 2>&1 || true
|
|
for f in /sys/class/net/br-wan/brif/*/hairpin_mode /sys/class/net/br-wan/brif/*/learning /sys/class/net/br-wan/brif/*/flood; do
|
|
[ -e "$f" ] && echo "$f $(cat $f)"
|
|
done > hairpin-modes.txt 2>&1 || true
|
|
sudo iptables-save > iptables.txt 2>&1 || true
|
|
sudo nft list ruleset > nft.txt 2>&1 || true
|
|
sudo dmesg --ctime > dmesg.txt 2>&1 || true
|
|
sudo virsh list --all > virsh-list.txt 2>&1 || true
|
|
ls -la /var/lib/spinifex /var/log/spinifex 2>&1 > spinifex-dirs.txt || true
|
|
tar czf /tmp/spinifex-diag.tar.gz -C /tmp spinifex-diag
|
|
REMOTE
|
|
NODE_COUNT=$(echo "$INVENTORY" | jq '.nodes | length')
|
|
for i in $(seq 0 $((NODE_COUNT - 1))); do
|
|
NODE_IP=$(echo "$INVENTORY" | jq -r ".nodes[$i].wan_ip")
|
|
NODE_DIR="$DIAG_DIR/node${i}-${NODE_IP}"
|
|
mkdir -p "$NODE_DIR"
|
|
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" tf-user@${NODE_IP} 'bash -s' < "${RUNNER_TEMP}/collect-diag.sh" > "$NODE_DIR/collect.log" 2>&1 || true
|
|
scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 -i "$SSH_KEY" \
|
|
tf-user@${NODE_IP}:/tmp/spinifex-diag.tar.gz "$NODE_DIR/spinifex-diag.tar.gz" || true
|
|
done
|
|
|
|
- name: Upload Diagnostics
|
|
if: always()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: e2e-diag-multi-${{ github.run_id }}
|
|
path: ${{ runner.temp }}/diag-multi/
|
|
if-no-files-found: warn
|
|
retention-days: 14
|
|
|
|
- name: Teardown
|
|
if: always()
|
|
working-directory: mulga/scripts/tofu-cluster
|
|
run: |
|
|
tofu workspace select -or-create "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1
|
|
tofu destroy -var-file=envs/${SPINIFEX_CI_ENV}.tfvars -auto-approve -input=false >> "$TOFU_LOG" 2>&1
|
|
./scrub-env.sh "${SPINIFEX_CI_ENV}" >> "$TOFU_LOG" 2>&1 || true
|
|
|
|
e2e_status:
|
|
name: Report E2E Status
|
|
if: always() && inputs.sha
|
|
needs: [ e2e_single, e2e_pseudo_multi, e2e_real_multi ]
|
|
runs-on: [ self-hosted, ci-single ]
|
|
timeout-minutes: 5
|
|
steps:
|
|
- name: Report Status to PR
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
run: |
|
|
if [[ "${{ needs.e2e_single.result }}" == "success" && \
|
|
"${{ needs.e2e_pseudo_multi.result }}" == "success" && \
|
|
"${{ needs.e2e_real_multi.result }}" == "success" ]]; then
|
|
STATE="success"
|
|
DESC="All E2E suites passed"
|
|
else
|
|
STATE="failure"
|
|
DESC="E2E failed: single=${{ needs.e2e_single.result }} pseudo=${{ needs.e2e_pseudo_multi.result }} multi=${{ needs.e2e_real_multi.result }}"
|
|
fi
|
|
gh api "repos/${{ github.repository }}/statuses/${{ inputs.sha }}" \
|
|
-f state="$STATE" \
|
|
-f context=e2e \
|
|
-f description="$DESC" \
|
|
-f target_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|