diff --git a/scripts/build-microvm-image.sh b/scripts/build-microvm-image.sh index e547c1993..df3a18e11 100755 --- a/scripts/build-microvm-image.sh +++ b/scripts/build-microvm-image.sh @@ -173,6 +173,21 @@ else echo "[build-microvm-image] fw_cfg: module found in lib/modules" fi +# --- Assert nginx + stream module (NLB L4 data plane) --- +# The lb-agent runs `nginx -c` with a config that `load_module`s the stream +# module; if either the binary or the .so is missing the agent never enters +# nginx mode, never probes, and NLB targets stay at 0/N healthy. Fail the build +# loudly here rather than discover it as an opaque e2e health timeout. +if [ ! -x "$CHROOT_DIR/usr/sbin/nginx" ] && [ ! -x "$CHROOT_DIR/usr/bin/nginx" ]; then + echo "ERROR: nginx binary missing from image — NLB data plane will not start" >&2 + exit 1 +fi +if ! find "$CHROOT_DIR/usr/lib/nginx/modules" -name "ngx_stream_module.so" 2>/dev/null | grep -q .; then + echo "ERROR: ngx_stream_module.so missing from image — NLB stream config fails to load" >&2 + exit 1 +fi +echo "[build-microvm-image] nginx: binary + stream module present" + # --- Strip kernel modules (Phase 2 keep-list approach) --- # Only modules needed by a virtio-mmio microvm guest are kept: # virtio*.ko* — virtio bus, net, blk, rng, console, and helpers diff --git a/spinifex/lbagent/agent.go b/spinifex/lbagent/agent.go index 0cf8a9d3c..229910f2f 100644 --- a/spinifex/lbagent/agent.go +++ b/spinifex/lbagent/agent.go @@ -413,6 +413,15 @@ func reloadHAProxy(configPath, pidPath string) error { // `nginx -t` fails closed before any reload, mirroring HAProxy's // validate-before-bind guarantee (a bad config never replaces a good one). func reloadNginx(configPath, pidPath string) error { + // On a stripped Alpine microvm initramfs the nginx runtime prefix dirs may + // be absent, so `nginx -t`/start fails closed on a missing pid dir or temp + // path — and the agent never enters nginx mode, never probes, and reports + // empty health (NLB targets stuck at 0/N). Pre-create them, mirroring + // reloadHAProxy's socket-dir pre-create. + for _, dir := range []string{filepath.Dir(pidPath), "/var/lib/nginx", "/var/lib/nginx/tmp", "/var/log/nginx"} { + _ = os.MkdirAll(dir, 0o750) + } + test := exec.Command("nginx", "-t", "-c", configPath) if out, err := test.CombinedOutput(); err != nil { return fmt.Errorf("nginx -t: %s: %w", strings.TrimSpace(string(out)), err) diff --git a/tests/e2e/harness/diag.go b/tests/e2e/harness/diag.go index a6de32495..1ac307dcf 100644 --- a/tests/e2e/harness/diag.go +++ b/tests/e2e/harness/diag.go @@ -92,6 +92,32 @@ func dumpConsoleOutput(t *testing.T, c *AWSClient, instanceID string) { } } +// DumpInstanceConsole writes an instance's full serial console (base64-decoded) +// to /. The lb-agent logs its data-plane engine activation and any +// `reload nginx:` error to the guest console, which the host daemon journal +// never sees — capturing it makes an NLB health-check timeout diagnosable from +// CI artifacts alone. Best-effort: failures log, they don't fail the test. +func DumpInstanceConsole(t *testing.T, c *AWSClient, instanceID, dir, name string) { + t.Helper() + out, err := c.EC2.GetConsoleOutput(&ec2.GetConsoleOutputInput{ + InstanceId: aws.String(instanceID), + }) + if err != nil { + t.Logf("console capture: GetConsoleOutput(%s) failed: %v", instanceID, err) + return + } + encoded := aws.StringValue(out.Output) + if encoded == "" { + t.Logf("console capture: %s empty", instanceID) + return + } + raw, derr := base64.StdEncoding.DecodeString(encoded) + if derr != nil { + raw = []byte(encoded) + } + DumpFile(t, dir, name, raw) +} + func dumpOVNState(t *testing.T, instanceID string) { t.Helper() if _, err := exec.LookPath("ovn-nbctl"); err != nil { diff --git a/tests/e2e/lb/lb_test.go b/tests/e2e/lb/lb_test.go index 15a459f08..eec36eade 100644 --- a/tests/e2e/lb/lb_test.go +++ b/tests/e2e/lb/lb_test.go @@ -506,6 +506,9 @@ func runLBSuite(t *testing.T, c *harness.AWSClient, f *sharedFixture, kind lbKin } harness.WaitForLBActive(t, c, lb.ARN, label, 5*time.Minute) + if kind == kindNLB { + captureLBConsoleOnFailure(t, c, eniDescPrefix, lb) + } harness.WaitForTargetsHealthy(t, c, tgArn, 2, label, 2*time.Minute) eni := lbENI(t, c, eniDescPrefix, lb) @@ -554,6 +557,7 @@ func runUDPNLBSuite(t *testing.T, c *harness.AWSClient, f *sharedFixture) { assert.Contains(t, lb.ARN, "/net/", label+" ARN must contain /net/") harness.WaitForLBActive(t, c, lb.ARN, label, 5*time.Minute) + captureLBConsoleOnFailure(t, c, "net", lb) // The key assertion: NLB targets reach healthy via the agent's active // prober. Pre-feature this timed out (nginx reported empty server lists). harness.WaitForTargetsHealthy(t, c, tgArn, 2, label, 2*time.Minute) @@ -1300,6 +1304,27 @@ func lbENI(t *testing.T, c *harness.AWSClient, prefix string, lb lbInfo) *ec2.Ne return eni } +// captureLBConsoleOnFailure registers an on-failure dump of the NLB VM's serial +// console to the subtest artifact dir. The lb-agent's nginx engine activation +// (and any `reload nginx:` error) lands on the guest console, not in the host +// daemon journal — so an NLB "0/N healthy" timeout is otherwise undiagnosable +// from CI artifacts. Call after the LB is active (the VM/ENI exists by then). +func captureLBConsoleOnFailure(t *testing.T, c *harness.AWSClient, eniDescPrefix string, lb lbInfo) { + t.Helper() + eni := lbENI(t, c, eniDescPrefix, lb) + if eni.Attachment == nil { + return + } + instanceID := aws.StringValue(eni.Attachment.InstanceId) + if instanceID == "" { + return + } + dir := harness.ArtifactDir(t, harness.LoadEnv(t)) + harness.OnFailure(t, func() { + harness.DumpInstanceConsole(t, c, instanceID, dir, "lb-vm-console.log") + }) +} + func publicIP(eni *ec2.NetworkInterface) string { if eni == nil || eni.Association == nil { return ""