From af690b3299466c44b5e666b93ba186eb3bc44b29 Mon Sep 17 00:00:00 2001
From: joshsiv-mulga <mrk@mulgadc.com>
Date: Fri, 5 Jun 2026 09:30:24 +1000
Subject: [PATCH] test(elbv2): collect LB microVM guest console in CI + surface
 NLB probe failures
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

NLB target health stalls at 0/N in CI while the data plane works locally.
The lb-agent's nginx engine activation, any nginx -t/reload error, and its
active-probe results are logged to the LB microVM serial console — which the
host journal never captures and the GetConsoleOutput API path reports as
NotFound, so the failure is undiagnosable from CI artifacts.

- e2e.yml: pull /run/spinifex/console-*.log off each node in the single- and
  multi-node collect steps (files persist past VM teardown) into lb-console/.
- lbagent healthcheck: log probe failures at Warn (was Debug) so unreachable
  targets surface at the agent's default level on the captured console.
---
 .github/workflows/e2e.yml       | 21 +++++++++++++++++++++
 spinifex/lbagent/healthcheck.go |  4 ++--
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
index 9488937e0..2fd055829 100644
--- a/.github/workflows/e2e.yml
+++ b/.github/workflows/e2e.yml
@@ -270,6 +270,15 @@ jobs:
           ssh "${SSH_OPTS[@]}" "$VM" 'sudo journalctl -u "spinifex-*" --no-pager --output=short-iso --since "60 min ago"' \
             > "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null
           ok "journal: $(wc -l < "${ARTIFACT_DIR}/spinifex-journal.log" 2>/dev/null || echo 0) lines"
+          # LB/system microVM serial consoles. The lb-agent logs its nginx (NLB)
+          # engine activation and any `nginx -t`/reload error + active-probe
+          # results to the guest serial console, which the host journal never
+          # captures. These files persist in /run/spinifex past VM teardown, so
+          # collecting them here makes an NLB "0/N healthy" timeout diagnosable.
+          mkdir -p "${ARTIFACT_DIR}/lb-console"
+          ssh "${SSH_OPTS[@]}" "$VM" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
+            | tar -C "${ARTIFACT_DIR}/lb-console" -xf - 2>/dev/null
+          ok "lb-console: $(ls "${ARTIFACT_DIR}/lb-console" 2>/dev/null | wc -l) files"
 
       - name: Convert test logs to JUnit XML
         if: always()
@@ -657,6 +666,18 @@ jobs:
             ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo journalctl -u openvswitch-ipsec.service --no-pager --output=short-iso --since "60 min ago"' \
               > "${ARTIFACT_DIR}/openvswitch-ipsec-node${idx}.log" 2>/dev/null
           done
+          # LB/system microVM serial consoles, per node. The lb-agent logs its
+          # nginx (NLB) engine activation and any `nginx -t`/reload error +
+          # active-probe results to the guest serial console (never in the host
+          # journal). Files persist in /run/spinifex past VM teardown.
+          for i in "${!NODE_LIST[@]}"; do
+            idx=$((i + 1))
+            ip="${NODE_LIST[$i]}"
+            mkdir -p "${ARTIFACT_DIR}/lb-console/node${idx}"
+            ssh "${SSH_OPTS[@]}" tf-user@"$ip" 'sudo sh -c "cd /run/spinifex 2>/dev/null && tar -cf - console-*.log 2>/dev/null"' \
+              | tar -C "${ARTIFACT_DIR}/lb-console/node${idx}" -xf - 2>/dev/null
+            ok "lb-console node${idx}: $(ls "${ARTIFACT_DIR}/lb-console/node${idx}" 2>/dev/null | wc -l) files"
+          done
 
       - name: Convert test logs to JUnit XML
         if: always()
diff --git a/spinifex/lbagent/healthcheck.go b/spinifex/lbagent/healthcheck.go
index c3d8deea1..94ad0371e 100644
--- a/spinifex/lbagent/healthcheck.go
+++ b/spinifex/lbagent/healthcheck.go
@@ -56,7 +56,7 @@ func probeOne(t HealthTarget) bool {
 	default: // TCP and anything else falls back to a connect probe
 		conn, err := net.DialTimeout("tcp", t.Address, probeTimeout)
 		if err != nil {
-			slog.Debug("TCP probe failed", "addr", t.Address, "err", err)
+			slog.Warn("TCP probe failed", "addr", t.Address, "err", err)
 			return false
 		}
 		_ = conn.Close()
@@ -86,7 +86,7 @@ func probeHTTP(t HealthTarget) bool {
 	url := fmt.Sprintf("%s://%s%s", scheme, t.Address, path)
 	resp, err := client.Get(url) //nolint:noctx // short fixed-timeout probe
 	if err != nil {
-		slog.Debug("HTTP probe failed", "url", url, "err", err)
+		slog.Warn("HTTP probe failed", "url", url, "err", err)
 		return false
 	}
 	defer resp.Body.Close()