diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9488937e0..2fd055829 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -270,6 +270,15 @@ jobs: ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \ > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines" + # LB/system microVM serial consoles. The lb-agent logs its nginx (NLB) + # engine activation and any `nginx -t`/reload error + active-probe + # results to the guest serial console, which the host journal never + # captures. These files persist in /run/spinifex past VM teardown, so + # collecting them here makes an NLB "0/N healthy" timeout diagnosable. + mkdir -p "${ARTIFACT_DIR}/lb-console" + ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \ + | tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null + ok "lb-console: $(ls "${ARTIFACT_DIR}/lb-console" 2>/dev/null | wc -l) files" - name: Convert test logs to JUnit XML if: always() @@ -657,6 +666,18 @@ jobs: ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \ > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null done + # LB/system microVM serial consoles, per node. The lb-agent logs its + # nginx (NLB) engine activation and any `nginx -t`/reload error + + # active-probe results to the guest serial console (never in the host + # journal). Files persist in /run/spinifex past VM teardown. + for i in "${!NODE_LIST[@]}"; do + idx=$((i + 1)) + ip="${NODE_LIST[$i]}" + mkdir -p "${ARTIFACT_DIR}/lb-console/node${idx}" + ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \ + | tar -C "${ARTIFACT_DIR}/lb-console/node${idx}" -xf - 2>/dev/null + ok "lb-console node${idx}: $(ls "${ARTIFACT_DIR}/lb-console/node${idx}" 2>/dev/null | wc -l) files" + done - name: Convert test logs to JUnit XML if: always() diff --git a/spinifex/lbagent/healthcheck.go b/spinifex/lbagent/healthcheck.go index c3d8deea1..94ad0371e 100644 --- a/spinifex/lbagent/healthcheck.go +++ b/spinifex/lbagent/healthcheck.go @@ -56,7 +56,7 @@ func probeOne(t HealthTarget) bool { default: // TCP and anything else falls back to a connect probe conn, err := net.DialTimeout("tcp", t.Address, probeTimeout) if err != nil { - slog.Debug("TCP probe failed", "addr", t.Address, "err", err) + slog.Warn("TCP probe failed", "addr", t.Address, "err", err) return false } _ = conn.Close() @@ -86,7 +86,7 @@ func probeHTTP(t HealthTarget) bool { url := fmt.Sprintf("%s://%s%s", scheme, t.Address, path) resp, err := client.Get(url) //nolint:noctx // short fixed-timeout probe if err != nil { - slog.Debug("HTTP probe failed", "url", url, "err", err) + slog.Warn("HTTP probe failed", "url", url, "err", err) return false } defer resp.Body.Close()