From 153980085ad3c545a261cd3b3a0c1fbc8d8360a5 Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Mon, 25 May 2026 10:35:54 +1000 Subject: [PATCH] test(e2e): sticky-skip downstream SSH tests when probe fails MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When TestSSHProbe times out (3 min, typically because of mulga-siv-134 datapath bug), four downstream tests each re-probe SSH on the same broken datapath and time out identically — single-node job runs ~24 min instead of ~12 min, with the same error logged five times. First waitForSSHReady timeout now sets a package-level atomic flag; subsequent waitForSSHReady / trySSHReady / requireSSHHealthy calls skip immediately. Preserves the original red signal on TestSSHProbe and on any test that genuinely needed SSH (Skip ≠ Pass). Bool variant (trySSHReady) consults but does not set the flag — its caller already treats False as advisory warning, not fatal. TestSGToSGDatapath has its own EventuallyErr SSH wait (not via waitForSSHReady) and needs an explicit requireSSHHealthy(t) gate at the top of runSGToSGDatapath. TestNATGateway already calls waitForSSHReady first so the gate fires naturally. Refs: mulga-siv-137 --- tests/e2e/single/datapath_test.go | 1 + tests/e2e/single/lifecycle_test.go | 42 ++++++++++++++++++++---------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/tests/e2e/single/datapath_test.go b/tests/e2e/single/datapath_test.go index 3c50ae0f7..65aab9cc9 100644 --- a/tests/e2e/single/datapath_test.go +++ b/tests/e2e/single/datapath_test.go @@ -45,6 +45,7 @@ systemd-run --unit=sge-http --description="Phase 8e HTTP server" \ func runSGToSGDatapath(t *testing.T, fix *Fixture) { harness.Phase(t, "Single — Security Group SG-to-SG Datapath (OVN)") harness.SkipIfNoOVN(t) + requireSSHHealthy(t) // Bootstrap every prereq up front. runSGEInstance / primaryENI use // `fix.AMIID / InstanceType / KeyName / KeyPath` indirectly; resolve diff --git a/tests/e2e/single/lifecycle_test.go b/tests/e2e/single/lifecycle_test.go index bd4ef1e26..d987c212f 100644 --- a/tests/e2e/single/lifecycle_test.go +++ b/tests/e2e/single/lifecycle_test.go @@ -3,11 +3,11 @@ package single import ( - "fmt" "net" "os/exec" "strconv" "strings" + "sync/atomic" "testing" "time" @@ -18,6 +18,23 @@ import ( "github.com/stretchr/testify/require" ) +// sshDatapathBroken is set the first time a single-node SSH probe times +// out. Downstream SSH-dependent tests call requireSSHHealthy(t) and skip +// rather than re-running the same 3-minute timeout against the same +// broken datapath (mulga-siv-134, mulga-siv-137). +var sshDatapathBroken atomic.Bool + +// requireSSHHealthy skips the calling test if a prior SSH probe on this +// single-node VM has already failed. Cuts the cascade cost of a broken +// datapath from 4×3min down to one fail + four skips. +func requireSSHHealthy(t *testing.T) { + t.Helper() + if sshDatapathBroken.Load() { + t.Skipf("skipping: earlier single-node SSH probe failed " + + "(mulga-siv-134); not retrying to keep suite time bounded") + } +} + // runStopStart stops fix.Instance and waits for it to reach the // "stopped" state, then asserts that rebooting a stopped instance is // rejected with IncorrectInstanceState. Leaves the instance stopped so @@ -394,21 +411,15 @@ func runRunInstancesMultiCount(t *testing.T, fix *Fixture) { // Proper hoist to harness.WaitForSSH is tracked under mulga-siv-101. func waitForSSHReady(t *testing.T, host string, port int, keyPath string) { t.Helper() + requireSSHHealthy(t) addr := net.JoinHostPort(host, strconv.Itoa(port)) harness.Step(t, "waiting for SSH handshake %s", addr) - harness.Eventually(t, func() bool { - cmd := exec.Command("ssh", - "-o", "StrictHostKeyChecking=no", - "-o", "UserKnownHostsFile=/dev/null", - "-o", "LogLevel=ERROR", - "-o", "ConnectTimeout=5", - "-o", "BatchMode=yes", - "-p", strconv.Itoa(port), - "-i", keyPath, - "ec2-user@"+host, - "true") - return cmd.Run() == nil - }, 3*time.Minute, 3*time.Second, fmt.Sprintf("SSH handshake %s never completed", addr)) + if !trySSHReady(host, port, keyPath, 3*time.Minute) { + sshDatapathBroken.Store(true) + t.Fatalf("Eventually: condition not met within 3m0s: "+ + "[SSH handshake %s never completed] "+ + "(sticky-skip enabled for downstream — see mulga-siv-134)", addr) + } } // waitForSSHHandshake is an alias kept for instance_test.go's call site. @@ -423,6 +434,9 @@ func waitForSSHHandshake(t *testing.T, host string, port int, keyPath string) { // flow installation is timing-sensitive on shared CI runners and not part // of the EC2 surface contract this test exists to validate. func trySSHReady(host string, port int, keyPath string, timeout time.Duration) bool { + if sshDatapathBroken.Load() { + return false + } deadline := time.Now().Add(timeout) for { cmd := exec.Command("ssh",