gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(elbv2): collect LB microVM guest console in CI + surface NLB probe failures

Browse Source 
NLB target health stalls at 0/N in CI while the data plane works locally.
The lb-agent's nginx engine activation, any nginx -t/reload error, and its
active-probe results are logged to the LB microVM serial console — which the
host journal never captures and the GetConsoleOutput API path reports as
NotFound, so the failure is undiagnosable from CI artifacts.

- e2e.yml: pull /run/spinifex/console-*.log off each node in the single- and
  multi-node collect steps (files persist past VM teardown) into lb-console/.
- lbagent healthcheck: log probe failures at Warn (was Debug) so unreachable
  targets surface at the agent's default level on the captured console.
This commit is contained in:
joshsiv-mulga
2026-06-05 09:30:24 +10:00
parent dae0b521da
commit af690b3299
2 changed files with 23 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/e2e.yml
+21
View File
@@ -270,6 +270,15 @@ jobs:
ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines"
# LB/system microVM serial consoles. The lb-agent logs its nginx (NLB)
# engine activation and any `nginx -t`/reload error + active-probe
# results to the guest serial console, which the host journal never
# captures. These files persist in /run/spinifex past VM teardown, so
# collecting them here makes an NLB "0/N healthy" timeout diagnosable.
mkdir -p "${ARTIFACT_DIR}/lb-console"
ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
| tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
ok "lb-console: $(ls "${ARTIFACT_DIR}/lb-console" 2>/dev/null | wc -l) files"
- name: Convert test logs to JUnit XML
if: always()
@@ -657,6 +666,18 @@ jobs:
ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
> "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
done
# LB/system microVM serial consoles, per node. The lb-agent logs its
# nginx (NLB) engine activation and any `nginx -t`/reload error +
# active-probe results to the guest serial console (never in the host
# journal). Files persist in /run/spinifex past VM teardown.
for i in "${!NODE_LIST[@]}"; do
idx=$((i + 1))
ip="${NODE_LIST[$i]}"
mkdir -p "${ARTIFACT_DIR}/lb-console/node${idx}"
ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
| tar -C "${ARTIFACT_DIR}/lb-console/node${idx}" -xf - 2>/dev/null
ok "lb-console node${idx}: $(ls "${ARTIFACT_DIR}/lb-console/node${idx}" 2>/dev/null | wc -l) files"
done
- name: Convert test logs to JUnit XML
if: always()
spinifex/lbagent/healthcheck.go
+2 -2
View File
@@ -56,7 +56,7 @@ func probeOne(t HealthTarget) bool {
default: // TCP and anything else falls back to a connect probe
conn, err := net.DialTimeout("tcp", t.Address, probeTimeout)
if err != nil {
slog.Debug("TCP probe failed", "addr", t.Address, "err", err)
slog.Warn("TCP probe failed", "addr", t.Address, "err", err)
return false
}
_ = conn.Close()
@@ -86,7 +86,7 @@ func probeHTTP(t HealthTarget) bool {
url := fmt.Sprintf("%s://%s%s", scheme, t.Address, path)
resp, err := client.Get(url) //nolint:noctx // short fixed-timeout probe
if err != nil {
slog.Debug("HTTP probe failed", "url", url, "err", err)
slog.Warn("HTTP probe failed", "url", url, "err", err)
return false
}
defer resp.Body.Close()