mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
fix(e2e): hard-cap runSSHQuiet ssh process to stop EIP probe hang (#481)
The EIP "verify unreachable after disassociate" step polls runSSHQuiet inside EventuallyErr(90s), but a wedged ssh made os/exec Wait block for 14 minutes — EventuallyErr cannot interrupt an in-flight CombinedOutput, so one probe consumed the whole 30-min test-binary budget and panicked. The panic skipped t.Cleanup, leaking the EIP and NAT-gateway VMs, which held single-node capacity and starved the lb suite's sys.micro launch (the daemon unsubscribes the full topic, so NATS returns no-responders). Bound the ssh child at the process layer with exec.CommandContext (30s deadline) plus WaitDelay, returning a descriptive timeout error. This is version- and phase-independent, unlike ssh's own ConnectTimeout. 30s is safe for all callers (worst legit call is ~15s) and well under the 90s poll budget, so a wedged probe now yields a clean bounded failure with cleanup intact.
This commit is contained in:
committed by
Julian Sommer
parent
e1c4f85a36
commit
eac6f449fc
@@ -3,6 +3,7 @@
|
||||
package single
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
@@ -443,6 +444,12 @@ func scpKey(t *testing.T, keyPath, host string) {
|
||||
}
|
||||
}
|
||||
|
||||
// sshQuietHardTimeout hard-caps one runSSHQuiet call. ConnectTimeout bounds only
|
||||
// the TCP connect; a post-connect banner/kex wedge (e.g. probing an EIP whose
|
||||
// DNAT was just torn down) is otherwise unbounded and can outlast the caller's
|
||||
// EventuallyErr budget. The ctx deadline kills ssh regardless of which phase hangs.
|
||||
const sshQuietHardTimeout = 30 * time.Second
|
||||
|
||||
// runSSHQuiet is the EventuallyErr-friendly variant of runSSH: it returns
|
||||
// stdout+stderr and the error rather than calling t.Fatal, so polling
|
||||
// loops can iterate on transient failures (cloud-init not done, OVN
|
||||
@@ -459,6 +466,14 @@ func runSSHQuiet(tgt harness.SSHTarget, command string) (string, error) {
|
||||
tgt.User + "@" + tgt.Host,
|
||||
command,
|
||||
}
|
||||
out, err := exec.Command("ssh", args...).CombinedOutput()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), sshQuietHardTimeout)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "ssh", args...)
|
||||
cmd.WaitDelay = 5 * time.Second // return even if ssh ignores the kill
|
||||
out, err := cmd.CombinedOutput()
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return string(out), fmt.Errorf("ssh %s@%s:%d timed out after %s: %w",
|
||||
tgt.User, tgt.Host, tgt.Port, sshQuietHardTimeout, ctx.Err())
|
||||
}
|
||||
return string(out), err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user