fix: wait for the server span in provider tracing tests

Servers reply before ending their span, so the client can return before
the span reaches the recorder. SpansByKind now polls until a server span
has ended and fails the test if none does within 5s.

spinifex/ebsprovider/natsserve/server_test.go

@@ -20,7 +20,6 @@ func spansByKind(t *testing.T, recorder *tracetest.SpanRecorder) (client, server
 	t.Helper()
 	client, server = testutil.SpansByKind(t, recorder)
 	require.NotNil(t, client, "no client span recorded")
-	require.NotNil(t, server, "no server span recorded")
 	return client, server
 }
 

spinifex/services/viperblockd/provider_tracing_test.go

@@ -27,7 +27,6 @@ func TestProviderTracing_ServerSpanJoinsTheCallersTrace(t *testing.T) {
 
 	clientSpan, serverSpan := testutil.SpansByKind(t, recorder)
 	require.NotNil(t, clientSpan, "no client span recorded")
-	require.NotNil(t, serverSpan, "no server span recorded")
 	assert.Equal(t, "ebs.capabilities", clientSpan.Name())
 	assert.Equal(t, "ebs.capabilities", serverSpan.Name())
 	assert.Equal(t, clientSpan.SpanContext().TraceID(), serverSpan.SpanContext().TraceID())
@@ -65,7 +64,6 @@ func TestProviderTracing_SpanNamesDropTheVolumeID(t *testing.T) {
 	require.NotNil(t, response.Error)
 
 	_, serverSpan := testutil.SpansByKind(t, recorder)
-	require.NotNil(t, serverSpan, "no server span recorded")
 	assert.Equal(t, "ebs.volume.describe", serverSpan.Name())
 	assert.NotContains(t, serverSpan.Name(), volumeID)
 	assert.Equal(t, ownerSubject, testutil.SpanAttribute(t, serverSpan, "ebs.provider.subject").AsString())

spinifex/testutil/tracing.go

@@ -2,6 +2,7 @@ package testutil
 
 import (
 	"testing"
+	"time"
 
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
@@ -42,12 +43,26 @@ func SpanAttribute(t *testing.T, span sdktrace.ReadOnlySpan, key string) attribu
 	return attribute.Value{}
 }
 
-// SpansByKind returns the last recorded span of each kind. Owner-first
-// routing can produce more than one client span for a single call, and the
-// leg that answered is the last one.
+// SpansByKind returns the last recorded span of each kind (owner-first routing
+// can record several client legs; the last answered). Servers reply before
+// ending their span, so this waits for one and fails the test if none ends.
 func SpansByKind(t *testing.T, recorder *tracetest.SpanRecorder) (client, server sdktrace.ReadOnlySpan) {
 	t.Helper()
-	for _, span := range recorder.Ended() {
+	deadline := time.Now().Add(5 * time.Second)
+	for {
+		client, server = lastSpansByKind(recorder.Ended())
+		if server != nil {
+			return client, server
+		}
+		if time.Now().After(deadline) {
+			t.Fatal("no server span recorded within 5s")
+		}
+		time.Sleep(time.Millisecond)
+	}
+}
+
+func lastSpansByKind(spans []sdktrace.ReadOnlySpan) (client, server sdktrace.ReadOnlySpan) {
+	for _, span := range spans {
 		switch span.SpanKind() {
 		case trace.SpanKindClient:
 			client = span