From 98efe62d7c8fc357e16b6e4596bdbcfc7d22a780 Mon Sep 17 00:00:00 2001 From: "pulse-triage[bot]" <249995291+pulse-triage[bot]@users.noreply.github.com> Date: Tue, 8 Sep 2026 21:14:29 +0100 Subject: [PATCH] test(api): distinguish stress setup and first-request latency The failed qualification and isolated API samples used the same setup but lacked comparable phase and per-worker request evidence. Retain UTC setup/load boundaries, execution context and first-attempt versus subsequent successful resource latency without altering workloads, aggregate reports or verdicts. These are observational partitions, not cold-tenant or historical-cause claims. Focused instrumented API stress passed; JSON assertions verify ordered phases and partition totals. Historical exact-candidate failure remains unresolved. Change-source: pulse-maintainer --- internal/api/multi_tenant_stress_test.go | 58 ++++++++++++++++++------ 1 file changed, 43 insertions(+), 15 deletions(-) diff --git a/internal/api/multi_tenant_stress_test.go b/internal/api/multi_tenant_stress_test.go index 075acec13..3a746d22f 100644 --- a/internal/api/multi_tenant_stress_test.go +++ b/internal/api/multi_tenant_stress_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "reflect" + "runtime" "sync" "testing" "time" @@ -32,6 +33,8 @@ import ( func TestMultiTenant_ConcurrentAPIStress(t *testing.T) { skipUnderRace(t) suppressTestLogs(t) + setupStarted := time.Now() + t.Logf("stress phase=setup at=%s gomaxprocs=%d goroutines=%d", setupStarted.UTC().Format(time.RFC3339Nano), runtime.GOMAXPROCS(0), runtime.NumGoroutine()) const ( numOrgs = 20 @@ -168,19 +171,23 @@ func TestMultiTenant_ConcurrentAPIStress(t *testing.T) { } } + t.Logf("stress phase=setup-complete at=%s elapsed=%v gomaxprocs=%d goroutines=%d", time.Now().UTC().Format(time.RFC3339Nano), time.Since(setupStarted), runtime.GOMAXPROCS(0), runtime.NumGoroutine()) + // Concurrent stress test: 3 workers per org × 20 orgs = 60 goroutines. // Each worker randomly hits one of three endpoints for its assigned org. type orgResult struct { - orgID string - resourceLats []time.Duration - historyLats []time.Duration - statsLats []time.Duration - resourceErrors int64 - historyErrors int64 - statsErrors int64 - resourceCount int64 - historyCount int64 - statsCount int64 + orgID string + resourceLats []time.Duration + firstResourceLats []time.Duration + laterResourceLats []time.Duration + historyLats []time.Duration + statsLats []time.Duration + resourceErrors int64 + historyErrors int64 + statsErrors int64 + resourceCount int64 + historyCount int64 + statsCount int64 } results := make([]orgResult, numOrgs) @@ -195,6 +202,9 @@ func TestMultiTenant_ConcurrentAPIStress(t *testing.T) { var mu sync.Mutex // Protects per-org result slices. + // Record before spawning workers: the ready barrier can release them before + // the parent observes ready.Wait returning. This is an envelope, not a common deadline. + t.Logf("stress phase=workers-launch at=%s", time.Now().UTC().Format(time.RFC3339Nano)) for orgIdx := 0; orgIdx < numOrgs; orgIdx++ { org := orgs[orgIdx] @@ -217,6 +227,8 @@ func TestMultiTenant_ConcurrentAPIStress(t *testing.T) { deadline := time.Now().Add(duration) var localResourceLats, localHistoryLats, localStatsLats []time.Duration + var firstResourceLats, laterResourceLats []time.Duration + resourceAttempted := false var localResourceErrs, localHistoryErrs, localStatsErrs int64 for time.Now().Before(deadline) { @@ -235,7 +247,13 @@ func TestMultiTenant_ConcurrentAPIStress(t *testing.T) { localResourceErrs++ } else { localResourceLats = append(localResourceLats, elapsed) + if !resourceAttempted { + firstResourceLats = append(firstResourceLats, elapsed) + } else { + laterResourceLats = append(laterResourceLats, elapsed) + } } + resourceAttempted = true case roll < 8: // Metrics history idx := rng.Intn(len(historyURLs)) @@ -265,6 +283,8 @@ func TestMultiTenant_ConcurrentAPIStress(t *testing.T) { } mu.Lock() + results[workerOrgIdx].firstResourceLats = append(results[workerOrgIdx].firstResourceLats, firstResourceLats...) + results[workerOrgIdx].laterResourceLats = append(results[workerOrgIdx].laterResourceLats, laterResourceLats...) results[workerOrgIdx].resourceLats = append(results[workerOrgIdx].resourceLats, localResourceLats...) results[workerOrgIdx].historyLats = append(results[workerOrgIdx].historyLats, localHistoryLats...) results[workerOrgIdx].statsLats = append(results[workerOrgIdx].statsLats, localStatsLats...) @@ -295,17 +315,21 @@ func TestMultiTenant_ConcurrentAPIStress(t *testing.T) { t.Fatal("multi-tenant stress test timed out (possible deadlock)") } wallTime := time.Since(start) + t.Logf("stress phase=workers-complete at=%s goroutines=%d", time.Now().UTC().Format(time.RFC3339Nano), runtime.NumGoroutine()) // Aggregate results across all orgs. var ( - allResourceLats []time.Duration - allHistoryLats []time.Duration - allStatsLats []time.Duration - totalErrors int64 - totalRequests int64 + allResourceLats []time.Duration + firstResourceLats, laterResourceLats []time.Duration + allHistoryLats []time.Duration + allStatsLats []time.Duration + totalErrors int64 + totalRequests int64 ) for _, r := range results { + firstResourceLats = append(firstResourceLats, r.firstResourceLats...) + laterResourceLats = append(laterResourceLats, r.laterResourceLats...) allResourceLats = append(allResourceLats, r.resourceLats...) allHistoryLats = append(allHistoryLats, r.historyLats...) allStatsLats = append(allStatsLats, r.statsLats...) @@ -313,6 +337,10 @@ func TestMultiTenant_ConcurrentAPIStress(t *testing.T) { totalRequests += r.resourceCount + r.historyCount + r.statsCount } + // First means first attempt per worker, not proven cold construction per tenant. + // Keep these observational partitions out of all latency/throughput verdicts. + t.Logf("resources first-per-worker: count=%d p95=%v; subsequent: count=%d p95=%v", len(firstResourceLats), percentile(firstResourceLats, 0.95), len(laterResourceLats), percentile(laterResourceLats, 0.95)) + rps := float64(totalRequests) / wallTime.Seconds() t.Logf("multi-tenant stress: %d orgs, %d workers, %d requests in %v (%.1f rps)", numOrgs, totalWorkers, totalRequests, wallTime, rps)