fix(e2e): hard-cap runSSHQuiet ssh process to stop EIP probe hang (#481)

The EIP "verify unreachable after disassociate" step polls runSSHQuiet
inside EventuallyErr(90s), but a wedged ssh made os/exec Wait block for
14 minutes — EventuallyErr cannot interrupt an in-flight CombinedOutput,
so one probe consumed the whole 30-min test-binary budget and panicked.
The panic skipped t.Cleanup, leaking the EIP and NAT-gateway VMs, which
held single-node capacity and starved the lb suite's sys.micro launch
(the daemon unsubscribes the full topic, so NATS returns no-responders).

Bound the ssh child at the process layer with exec.CommandContext (30s
deadline) plus WaitDelay, returning a descriptive timeout error. This is
version- and phase-independent, unlike ssh's own ConnectTimeout. 30s is
safe for all callers (worst legit call is ~15s) and well under the 90s
poll budget, so a wedged probe now yields a clean bounded failure with
cleanup intact.
This commit is contained in:
Josh Siviour
2026-07-07 21:12:24 +10:00
committed by Julian Sommer
parent e1c4f85a36
commit eac6f449fc
+16 -1
View File
@@ -3,6 +3,7 @@
package single
import (
"context"
"fmt"
"os/exec"
"strconv"
@@ -443,6 +444,12 @@ func scpKey(t *testing.T, keyPath, host string) {
}
}
// sshQuietHardTimeout hard-caps one runSSHQuiet call. ConnectTimeout bounds only
// the TCP connect; a post-connect banner/kex wedge (e.g. probing an EIP whose
// DNAT was just torn down) is otherwise unbounded and can outlast the caller's
// EventuallyErr budget. The ctx deadline kills ssh regardless of which phase hangs.
const sshQuietHardTimeout = 30 * time.Second
// runSSHQuiet is the EventuallyErr-friendly variant of runSSH: it returns
// stdout+stderr and the error rather than calling t.Fatal, so polling
// loops can iterate on transient failures (cloud-init not done, OVN
@@ -459,6 +466,14 @@ func runSSHQuiet(tgt harness.SSHTarget, command string) (string, error) {
tgt.User + "@" + tgt.Host,
command,
}
out, err := exec.Command("ssh", args...).CombinedOutput()
ctx, cancel := context.WithTimeout(context.Background(), sshQuietHardTimeout)
defer cancel()
cmd := exec.CommandContext(ctx, "ssh", args...)
cmd.WaitDelay = 5 * time.Second // return even if ssh ignores the kill
out, err := cmd.CombinedOutput()
if ctx.Err() == context.DeadlineExceeded {
return string(out), fmt.Errorf("ssh %s@%s:%d timed out after %s: %w",
tgt.User, tgt.Host, tgt.Port, sshQuietHardTimeout, ctx.Err())
}
return string(out), err
}