From eac6f449fcea4033cf564e7a78ddd6b8ec80dd23 Mon Sep 17 00:00:00 2001 From: Josh Siviour Date: Tue, 7 Jul 2026 21:12:24 +1000 Subject: [PATCH] fix(e2e): hard-cap runSSHQuiet ssh process to stop EIP probe hang (#481) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EIP "verify unreachable after disassociate" step polls runSSHQuiet inside EventuallyErr(90s), but a wedged ssh made os/exec Wait block for 14 minutes — EventuallyErr cannot interrupt an in-flight CombinedOutput, so one probe consumed the whole 30-min test-binary budget and panicked. The panic skipped t.Cleanup, leaking the EIP and NAT-gateway VMs, which held single-node capacity and starved the lb suite's sys.micro launch (the daemon unsubscribes the full topic, so NATS returns no-responders). Bound the ssh child at the process layer with exec.CommandContext (30s deadline) plus WaitDelay, returning a descriptive timeout error. This is version- and phase-independent, unlike ssh's own ConnectTimeout. 30s is safe for all callers (worst legit call is ~15s) and well under the 90s poll budget, so a wedged probe now yields a clean bounded failure with cleanup intact. --- tests/e2e/single/natgw_test.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/e2e/single/natgw_test.go b/tests/e2e/single/natgw_test.go index 5792c3bdf..572c2549c 100644 --- a/tests/e2e/single/natgw_test.go +++ b/tests/e2e/single/natgw_test.go @@ -3,6 +3,7 @@ package single import ( + "context" "fmt" "os/exec" "strconv" @@ -443,6 +444,12 @@ func scpKey(t *testing.T, keyPath, host string) { } } +// sshQuietHardTimeout hard-caps one runSSHQuiet call. ConnectTimeout bounds only +// the TCP connect; a post-connect banner/kex wedge (e.g. probing an EIP whose +// DNAT was just torn down) is otherwise unbounded and can outlast the caller's +// EventuallyErr budget. The ctx deadline kills ssh regardless of which phase hangs. +const sshQuietHardTimeout = 30 * time.Second + // runSSHQuiet is the EventuallyErr-friendly variant of runSSH: it returns // stdout+stderr and the error rather than calling t.Fatal, so polling // loops can iterate on transient failures (cloud-init not done, OVN @@ -459,6 +466,14 @@ func runSSHQuiet(tgt harness.SSHTarget, command string) (string, error) { tgt.User + "@" + tgt.Host, command, } - out, err := exec.Command("ssh", args...).CombinedOutput() + ctx, cancel := context.WithTimeout(context.Background(), sshQuietHardTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "ssh", args...) + cmd.WaitDelay = 5 * time.Second // return even if ssh ignores the kill + out, err := cmd.CombinedOutput() + if ctx.Err() == context.DeadlineExceeded { + return string(out), fmt.Errorf("ssh %s@%s:%d timed out after %s: %w", + tgt.User, tgt.Host, tgt.Port, sshQuietHardTimeout, ctx.Err()) + } return string(out), err }