From 3298d89c2aab2d42de1dd7f41d0672debc12c5f3 Mon Sep 17 00:00:00 2001 From: joshsiv-mulga Date: Mon, 18 May 2026 08:17:39 +1000 Subject: [PATCH] feat(e2e): add Go test harness + cert scenario, port run-cert-e2e.sh Lands step 1 of docs/development/improvements/e2e-go-harness.md: shared harness primitives (env, wait, artifacts, cert, tls) plus the TestCertIssuance scenario that supersedes run-cert-e2e.sh. All files gated by //go:build e2e so default builds and `make preflight` are unaffected. Bash script stays in place for one CI cycle before deletion. --- tests/e2e/Makefile | 24 ++++ tests/e2e/cert/cert_test.go | 199 +++++++++++++++++++++++++++++++++ tests/e2e/harness/artifacts.go | 64 +++++++++++ tests/e2e/harness/cert.go | 129 +++++++++++++++++++++ tests/e2e/harness/env.go | 123 ++++++++++++++++++++ tests/e2e/harness/tls.go | 67 +++++++++++ tests/e2e/harness/wait.go | 46 ++++++++ 7 files changed, 652 insertions(+) create mode 100644 tests/e2e/Makefile create mode 100644 tests/e2e/cert/cert_test.go create mode 100644 tests/e2e/harness/artifacts.go create mode 100644 tests/e2e/harness/cert.go create mode 100644 tests/e2e/harness/env.go create mode 100644 tests/e2e/harness/tls.go create mode 100644 tests/e2e/harness/wait.go diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile new file mode 100644 index 000000000..03324b5b6 --- /dev/null +++ b/tests/e2e/Makefile @@ -0,0 +1,24 @@ +PKG ?= ./cert/... ./lb/... ./multinode/... ./baremetal/... ./reboot/... ./tofu/... +ARTIFACT_DIR ?= /tmp/spinifex-e2e-artifacts +TIMEOUT ?= 30m +MODE ?= single + +.PHONY: e2e e2e-cert e2e-lb e2e-multinode e2e-bm e2e-reboot e2e-tofu e2e-all clean + +e2e: + @mkdir -p $(ARTIFACT_DIR) + SPINIFEX_E2E=1 \ + SPINIFEX_MODE=$(MODE) \ + ARTIFACT_DIR=$(ARTIFACT_DIR) \ + go test -tags=e2e -v -timeout=$(TIMEOUT) $(PKG) + +e2e-cert: ; $(MAKE) e2e PKG=./cert/... +e2e-lb: ; $(MAKE) e2e PKG=./lb/... +e2e-multinode: ; $(MAKE) e2e PKG=./multinode/... +e2e-bm: ; $(MAKE) e2e PKG=./baremetal/... MODE=baremetal +e2e-reboot: ; $(MAKE) e2e PKG=./reboot/... +e2e-tofu: ; $(MAKE) e2e PKG=./tofu/... +e2e-all: ; $(MAKE) e2e + +clean: + rm -rf $(ARTIFACT_DIR) diff --git a/tests/e2e/cert/cert_test.go b/tests/e2e/cert/cert_test.go new file mode 100644 index 000000000..ba559c435 --- /dev/null +++ b/tests/e2e/cert/cert_test.go @@ -0,0 +1,199 @@ +//go:build e2e + +package cert + +import ( + "crypto/x509" + "fmt" + "os" + "os/exec" + "strings" + "testing" + + "github.com/mulgadc/spinifex/tests/e2e/harness" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestCertIssuance is the Go port of run-cert-e2e.sh. +// +// Validates that server certificates contain the correct SANs and that TLS +// connections succeed without InsecureSkipVerify when the Spinifex CA is +// trusted. Assumes the cluster is bootstrapped (services up, certs generated, +// CA installed in system trust store via bootstrap.sh step 8). +func TestCertIssuance(t *testing.T) { + env := harness.LoadEnv(t) + artifacts := harness.ArtifactDir(t, env) + + caPath, err := harness.ResolveCACert(env) + require.NoError(t, err, "Spinifex CA cert must exist") + t.Logf("CA cert: %s", caPath) + t.Logf("mode=%s nodeIPs=%v serviceIPs=%v", env.Mode, env.NodeIPs, env.ServiceIPs) + + caCert, err := harness.ParseCertFile(caPath) + require.NoError(t, err) + + t.Run("SANContainsNodeIPs", func(t *testing.T) { + for _, ip := range env.NodeIPs { + ip := ip + t.Run(ip, func(t *testing.T) { + cert := loadCert(t, env, ip, artifacts) + assert.True(t, harness.CertHasIPSAN(cert, ip), + "cert for %s missing IP SAN %s (got IPs=%v DNS=%v)", + ip, ip, cert.IPAddresses, cert.DNSNames) + }) + } + }) + + t.Run("HostnameInDNSSANs", func(t *testing.T) { + hostname, err := os.Hostname() + require.NoError(t, err) + if hostname == "" || hostname == "localhost" { + t.Skipf("hostname %q not testable", hostname) + } + if env.Mode == harness.ModeMultinode || env.Mode == harness.ModeBaremetal { + t.Skip("hostname SAN check requires local cert path") + } + referenceIP := env.NodeIPs[0] + if env.Mode == harness.ModePseudo { + referenceIP = "10.11.12.1" + } + cert := loadCert(t, env, referenceIP, artifacts) + assert.True(t, harness.CertHasDNSSAN(cert, hostname), + "cert missing DNS SAN for hostname %q (DNS=%v)", hostname, cert.DNSNames) + }) + + t.Run("TLSHandshakeWithCAFile", func(t *testing.T) { + pool, err := harness.LoadCAPool(caPath) + require.NoError(t, err) + for _, ip := range env.ServiceIPs { + ip := ip + t.Run(ip, func(t *testing.T) { + url := fmt.Sprintf("https://%s:%d/", ip, env.AWSGWPort) + code, _, err := harness.HTTPSGet(url, pool, env.DefaultTimeout) + require.NoError(t, err, "TLS handshake to %s", url) + assert.NotZero(t, code, "expected HTTP response from %s", url) + }) + } + }) + + t.Run("SystemTrustStore", func(t *testing.T) { + t.Run("CAInstalled", func(t *testing.T) { + _, err := os.Stat(harness.SystemCAPath) + require.NoError(t, err, "CA cert must be installed at %s", harness.SystemCAPath) + }) + + t.Run("FingerprintMatchesConfig", func(t *testing.T) { + sysCert, err := harness.ParseCertFile(harness.SystemCAPath) + require.NoError(t, err) + assert.True(t, harness.FingerprintMatches(sysCert, caCert), + "system CA fingerprint differs from %s — system trust store stale", caPath) + }) + + t.Run("CurlWithoutCAFlag", func(t *testing.T) { + testIP := env.ServiceIPs[0] + url := fmt.Sprintf("https://%s:%d/", testIP, env.AWSGWPort) + code, _, err := harness.HTTPSGet(url, nil, env.DefaultTimeout) + require.NoError(t, err, "system trust store: GET %s failed — CA not in system bundle?", url) + assert.NotZero(t, code) + }) + }) + + t.Run("OpenSSLVerify", func(t *testing.T) { + for _, ip := range env.ServiceIPs { + ip := ip + t.Run(ip, func(t *testing.T) { + require.NoError(t, harness.OpenSSLVerify(t, caPath, ip, env.AWSGWPort)) + }) + } + }) + + t.Run("CADownloadEndpoint", func(t *testing.T) { + uiIP := env.NodeIPs[0] + if env.Mode == harness.ModeSingle { + uiIP = "127.0.0.1" + } + pool, err := harness.LoadCAPool(caPath) + require.NoError(t, err) + url := fmt.Sprintf("https://%s:%d/api/ca.pem", uiIP, env.UIPort) + code, body, err := harness.HTTPSGet(url, pool, env.DefaultTimeout) + if err != nil { + t.Skipf("UI not reachable at %s: %v (pseudo-multinode disables UI)", url, err) + } + require.Equalf(t, 200, code, "GET %s returned %d", url, code) + downloaded, err := harness.ParseCertPEM(body) + require.NoError(t, err, "CA download must be a PEM cert") + assert.True(t, harness.FingerprintMatches(downloaded, caCert), + "downloaded CA fingerprint differs from local CA at %s", caPath) + }) + + t.Run("CertMetadata", func(t *testing.T) { + if env.Mode == harness.ModeMultinode || env.Mode == harness.ModeBaremetal { + t.Skip("metadata check requires local cert path") + } + referenceIP := env.NodeIPs[0] + if env.Mode == harness.ModePseudo { + referenceIP = "10.11.12.1" + } + cert := loadCert(t, env, referenceIP, artifacts) + + assert.Contains(t, cert.Subject.CommonName, "Spinifex Server", + "server cert CN should be 'Spinifex Server' (got %q)", cert.Subject.CommonName) + assert.Contains(t, cert.Issuer.CommonName, "Spinifex Local CA", + "server cert issuer should be 'Spinifex Local CA' (got %q)", cert.Issuer.CommonName) + assert.False(t, cert.NotAfter.IsZero(), "cert must have an expiry") + t.Logf("cert expiry: %s", cert.NotAfter) + }) + + t.Run("CrossNodeIssuerSharedCA", func(t *testing.T) { + if env.Mode == harness.ModeSingle || len(env.ServiceIPs) < 2 { + t.Skip("requires multi-node deployment") + } + for _, ip := range env.ServiceIPs { + ip := ip + t.Run(ip, func(t *testing.T) { + served, err := harness.FetchServedCert(ip, env.AWSGWPort, env.DefaultTimeout) + require.NoError(t, err) + assert.Contains(t, served.Issuer.CommonName, "Spinifex Local CA", + "node %s cert has unexpected issuer %q", ip, served.Issuer.CommonName) + }) + } + }) + + harness.OnFailure(t, func() { + harness.DumpCmd(t, artifacts, "ip-addr.txt", "ip", "-4", "addr", "show") + harness.DumpCmd(t, artifacts, "ca-text.txt", "openssl", "x509", "-in", caPath, "-text", "-noout") + if _, err := os.Stat(harness.SystemCAPath); err == nil { + harness.DumpCmd(t, artifacts, "system-ca-text.txt", + "openssl", "x509", "-in", harness.SystemCAPath, "-text", "-noout") + } + }) +} + +// loadCert resolves the cert for ip: local file when available, served cert +// otherwise. Dumps cert text to artifacts on failure. +func loadCert(t *testing.T, env *harness.Env, ip, artifacts string) *x509.Certificate { + t.Helper() + if path := harness.ServerCertPath(env, ip); path != "" { + cert, err := harness.ParseCertFile(path) + require.NoErrorf(t, err, "parse local cert %s", path) + harness.OnFailure(t, func() { + harness.DumpCmd(t, artifacts, sanitize("cert-"+ip+".txt"), + "openssl", "x509", "-in", path, "-text", "-noout") + }) + return cert + } + served, err := harness.FetchServedCert(ip, env.AWSGWPort, env.DefaultTimeout) + require.NoErrorf(t, err, "fetch served cert %s:%d", ip, env.AWSGWPort) + harness.OnFailure(t, func() { + out, err := exec.Command("openssl", "s_client", "-connect", + fmt.Sprintf("%s:%d", ip, env.AWSGWPort), "-servername", ip).CombinedOutput() + if err != nil { + out = append(out, []byte(fmt.Sprintf("\n(exit: %v)", err))...) + } + harness.DumpFile(t, artifacts, sanitize("served-"+ip+".txt"), out) + }) + return served +} + +func sanitize(s string) string { return strings.ReplaceAll(s, "/", "_") } diff --git a/tests/e2e/harness/artifacts.go b/tests/e2e/harness/artifacts.go new file mode 100644 index 000000000..4d595d7d5 --- /dev/null +++ b/tests/e2e/harness/artifacts.go @@ -0,0 +1,64 @@ +//go:build e2e + +package harness + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// ArtifactDir returns the per-test artifact directory and registers a Cleanup +// that prunes the directory if the test passed (to avoid filling disk on CI). +// Call once per test. Safe to call from subtests — uses t.Name() which is +// unique per subtest. +func ArtifactDir(t *testing.T, env *Env) string { + t.Helper() + safeName := strings.ReplaceAll(t.Name(), "/", "_") + dir := filepath.Join(env.ArtifactDir, safeName) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("artifact dir: %v", err) + } + t.Cleanup(func() { + if !t.Failed() { + _ = os.RemoveAll(dir) + } + }) + return dir +} + +// DumpFile writes content to /. Errors are logged but +// do not fail the test — artifacts are best-effort diagnostics. +func DumpFile(t *testing.T, dir, name string, content []byte) { + t.Helper() + path := filepath.Join(dir, name) + if err := os.WriteFile(path, content, 0o644); err != nil { + t.Logf("dump %s: %v", path, err) + } +} + +// DumpCmd runs cmd and writes stdout+stderr to /. +// Best-effort — does not fail the test. +func DumpCmd(t *testing.T, dir, name string, cmd string, args ...string) { + t.Helper() + out, err := exec.Command(cmd, args...).CombinedOutput() + header := fmt.Sprintf("$ %s %s\n", cmd, strings.Join(args, " ")) + if err != nil { + header += fmt.Sprintf("(exit error: %v)\n", err) + } + DumpFile(t, dir, name, append([]byte(header), out...)) +} + +// OnFailure registers fn to run only if the test failed. Use to dump +// expensive state (journal logs, VM consoles, predastore listings). +func OnFailure(t *testing.T, fn func()) { + t.Helper() + t.Cleanup(func() { + if t.Failed() { + fn() + } + }) +} diff --git a/tests/e2e/harness/cert.go b/tests/e2e/harness/cert.go new file mode 100644 index 000000000..248d08a9e --- /dev/null +++ b/tests/e2e/harness/cert.go @@ -0,0 +1,129 @@ +//go:build e2e + +package harness + +import ( + "crypto/x509" + "encoding/pem" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// ResolveCACert walks the standard config locations for the Spinifex CA cert +// (matching the bash resolve_ca_cert helper) and returns the first hit. +func ResolveCACert(env *Env) (string, error) { + candidates := []string{ + filepath.Join(env.ConfigDir, "ca.pem"), + "/etc/spinifex/ca.pem", + os.ExpandEnv("$HOME/spinifex/config/ca.pem"), + os.ExpandEnv("$HOME/node1/config/ca.pem"), + } + for _, c := range candidates { + if c == "" { + continue + } + if _, err := os.Stat(c); err == nil { + return c, nil + } + } + return "", fmt.Errorf("CA cert not found in any candidate location: %v", candidates) +} + +// SystemCAPath is the canonical install location used by bootstrap.sh. +const SystemCAPath = "/usr/local/share/ca-certificates/spinifex-ca.crt" + +// ServerCertPath returns the local file path for a node's server cert, +// or empty string if the cert must be fetched over the wire instead. +func ServerCertPath(env *Env, nodeIP string) string { + switch env.Mode { + case ModeSingle: + return filepath.Join(env.ConfigDir, "server.pem") + case ModePseudo: + octet := nodeIP[strings.LastIndex(nodeIP, ".")+1:] + return os.ExpandEnv("$HOME/node" + octet + "/config/server.pem") + default: + return "" + } +} + +// ParseCertFile reads a PEM cert from disk and returns the parsed x509. +func ParseCertFile(path string) (*x509.Certificate, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read %s: %w", path, err) + } + return ParseCertPEM(raw) +} + +// ParseCertPEM parses the first CERTIFICATE block in raw. +func ParseCertPEM(raw []byte) (*x509.Certificate, error) { + block, _ := pem.Decode(raw) + if block == nil || block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("no CERTIFICATE PEM block") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("parse certificate: %w", err) + } + return cert, nil +} + +// CertHasIPSAN returns true if cert has ip in its IP SANs. +func CertHasIPSAN(cert *x509.Certificate, ip string) bool { + target := net.ParseIP(ip) + if target == nil { + return false + } + for _, san := range cert.IPAddresses { + if san.Equal(target) { + return true + } + } + return false +} + +// CertHasDNSSAN returns true if cert has name (case-insensitive) in its DNS SANs. +func CertHasDNSSAN(cert *x509.Certificate, name string) bool { + want := strings.ToLower(name) + for _, san := range cert.DNSNames { + if strings.ToLower(san) == want { + return true + } + } + return false +} + +// OpenSSLVerify shells out to `openssl s_client -verify_return_error` against +// host:port using caFile as the trust anchor. Returns nil if verify code = 0. +// Kept as a shell-out (not crypto/tls) so we exercise the same code path the +// AWS SDK clients use in production tooling. +func OpenSSLVerify(t *testing.T, caFile, host string, port int) error { + t.Helper() + target := fmt.Sprintf("%s:%d", host, port) + cmd := exec.Command("openssl", "s_client", + "-CAfile", caFile, + "-connect", target, + "-verify_return_error", + "-servername", host, + ) + cmd.Stdin = strings.NewReader("") + out, err := cmd.CombinedOutput() + if err == nil && strings.Contains(string(out), "Verify return code: 0") { + return nil + } + return fmt.Errorf("openssl verify %s failed: %v\noutput:\n%s", target, err, out) +} + +// FingerprintMatches returns true if the two PEM certs share the SHA-256 +// fingerprint that openssl x509 -fingerprint prints. +func FingerprintMatches(a, b *x509.Certificate) bool { + if a == nil || b == nil { + return false + } + return string(a.Raw) == string(b.Raw) +} diff --git a/tests/e2e/harness/env.go b/tests/e2e/harness/env.go new file mode 100644 index 000000000..29a0b2209 --- /dev/null +++ b/tests/e2e/harness/env.go @@ -0,0 +1,123 @@ +//go:build e2e + +// Package harness provides shared E2E test primitives for the Spinifex scenario +// suites (cert, lb, multinode, baremetal, reboot, tofu). Each scenario package +// imports this harness; bash drivers under spinifex/tests/e2e/ are being +// retired in favour of these Go scenarios — see +// docs/development/improvements/e2e-go-harness.md. +package harness + +import ( + "os" + "strings" + "testing" + "time" +) + +type Mode string + +const ( + ModeSingle Mode = "single" + ModePseudo Mode = "pseudo" + ModeMultinode Mode = "multinode" + ModeBaremetal Mode = "baremetal" +) + +type Env struct { + Mode Mode + NodeIPs []string + ServiceIPs []string + ConfigDir string + AWSGWPort int + UIPort int + ArtifactDir string + DefaultTimeout time.Duration + DefaultPoll time.Duration +} + +func LoadEnv(t *testing.T) *Env { + t.Helper() + if os.Getenv("SPINIFEX_E2E") == "" { + t.Skip("SPINIFEX_E2E not set; skipping E2E scenario") + } + + mode := Mode(getenv("SPINIFEX_MODE", string(ModeSingle))) + nodeIPs := splitCSV(os.Getenv("SPINIFEX_NODE_IPS")) + if len(nodeIPs) == 0 { + nodeIPs = []string{"127.0.0.1"} + } + serviceIPs := splitCSV(os.Getenv("SPINIFEX_SERVICE_IPS")) + if len(serviceIPs) == 0 { + serviceIPs = nodeIPs + } + + configDir := getenv("SPINIFEX_CONFIG_DIR", "") + if configDir == "" { + for _, c := range []string{"/etc/spinifex", os.ExpandEnv("$HOME/spinifex/config")} { + if stat, err := os.Stat(c); err == nil && stat.IsDir() { + configDir = c + break + } + } + } + + return &Env{ + Mode: mode, + NodeIPs: nodeIPs, + ServiceIPs: serviceIPs, + ConfigDir: configDir, + AWSGWPort: atoiOr("SPINIFEX_AWSGW_PORT", 9999), + UIPort: atoiOr("SPINIFEX_UI_PORT", 3000), + ArtifactDir: getenv("ARTIFACT_DIR", "/tmp/spinifex-e2e-artifacts"), + DefaultTimeout: durationOr("SPINIFEX_DEFAULT_TIMEOUT", 30*time.Second), + DefaultPoll: durationOr("SPINIFEX_DEFAULT_POLL", 500*time.Millisecond), + } +} + +func getenv(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func splitCSV(s string) []string { + if s == "" { + return nil + } + parts := strings.Split(s, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + if v := strings.TrimSpace(p); v != "" { + out = append(out, v) + } + } + return out +} + +func atoiOr(key string, def int) int { + v := os.Getenv(key) + if v == "" { + return def + } + n := 0 + for _, c := range v { + if c < '0' || c > '9' { + return def + } + n = n*10 + int(c-'0') + } + return n +} + +func durationOr(key string, def time.Duration) time.Duration { + v := os.Getenv(key) + if v == "" { + return def + } + d, err := time.ParseDuration(v) + if err != nil { + return def + } + return d +} diff --git a/tests/e2e/harness/tls.go b/tests/e2e/harness/tls.go new file mode 100644 index 000000000..68ecb7258 --- /dev/null +++ b/tests/e2e/harness/tls.go @@ -0,0 +1,67 @@ +//go:build e2e + +package harness + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "net" + "net/http" + "os" + "time" +) + +// FetchServedCert opens a TLS connection to host:port and returns the +// leaf certificate the server presented. Trust is bypassed (the caller +// decides whether to validate against a CA pool afterwards). +func FetchServedCert(host string, port int, timeout time.Duration) (*x509.Certificate, error) { + dialer := &net.Dialer{Timeout: timeout} + conn, err := tls.DialWithDialer(dialer, "tcp", fmt.Sprintf("%s:%d", host, port), &tls.Config{ + InsecureSkipVerify: true, + ServerName: host, + }) + if err != nil { + return nil, fmt.Errorf("tls dial %s:%d: %w", host, port, err) + } + defer conn.Close() + certs := conn.ConnectionState().PeerCertificates + if len(certs) == 0 { + return nil, fmt.Errorf("no peer certificates from %s:%d", host, port) + } + return certs[0], nil +} + +// HTTPSGet probes url using the given CA pool (nil = system trust store). +// Returns the HTTP status code and body. Connection errors return status 0. +func HTTPSGet(url string, caPool *x509.CertPool, timeout time.Duration) (int, []byte, error) { + client := &http.Client{ + Timeout: timeout, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: caPool, + }, + }, + } + resp, err := client.Get(url) + if err != nil { + return 0, nil, err + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + return resp.StatusCode, body, nil +} + +// LoadCAPool reads a PEM-encoded CA bundle from path and returns an x509 pool. +func LoadCAPool(path string) (*x509.CertPool, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read CA %s: %w", path, err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(raw) { + return nil, fmt.Errorf("no certs parsed from %s", path) + } + return pool, nil +} diff --git a/tests/e2e/harness/wait.go b/tests/e2e/harness/wait.go new file mode 100644 index 000000000..db36be947 --- /dev/null +++ b/tests/e2e/harness/wait.go @@ -0,0 +1,46 @@ +//go:build e2e + +package harness + +import ( + "testing" + "time" +) + +// Eventually polls cond until it returns true or the timeout expires. +// Replaces every `sleep N; ` loop in the bash scripts. +func Eventually(t *testing.T, cond func() bool, timeout, interval time.Duration, msgAndArgs ...any) { + t.Helper() + deadline := time.Now().Add(timeout) + for { + if cond() { + return + } + if time.Now().After(deadline) { + if len(msgAndArgs) == 0 { + t.Fatalf("Eventually: condition not met within %s", timeout) + } + t.Fatalf("Eventually: condition not met within %s: %v", timeout, msgAndArgs) + } + time.Sleep(interval) + } +} + +// EventuallyErr is like Eventually but lets cond return an error for the +// failure message. Useful when the probe itself yields diagnostic detail. +func EventuallyErr(t *testing.T, cond func() error, timeout, interval time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + var lastErr error + for { + if err := cond(); err == nil { + return + } else { + lastErr = err + } + if time.Now().After(deadline) { + t.Fatalf("EventuallyErr: condition not met within %s: %v", timeout, lastErr) + } + time.Sleep(interval) + } +}