pulse/internal/api/resourceapi/load_test.go

package resourceapi

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"sort"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/config"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
)

// TestLoad_500Node_ConcurrentResources validates the resource query service
// with 500 nodes and 2,500 VMs under 50-way concurrent request load.
func TestLoad_500Node_ConcurrentResources(t *testing.T) {
	if raceEnabled {
		t.Skip("skipping latency test under -race")
	}

	state := buildResourceLoadState(t, 500)
	handlers := NewQueryService(&config.Config{DataPath: t.TempDir()})
	handlers.SetStateProvider(&resourceLoadStateProvider{state: state})

	req := httptest.NewRequest(http.MethodGet, "/api/resources", nil)
	rec := httptest.NewRecorder()
	handlers.HandleListResources(rec, req)
	if rec.Code != http.StatusOK {
		t.Fatalf("warmup failed: status %d, body: %s", rec.Code, rec.Body.String())
	}
	var warmupResp map[string]interface{}
	if err := json.Unmarshal(rec.Body.Bytes(), &warmupResp); err != nil {
		t.Fatalf("warmup unmarshal: %v", err)
	}
	meta, _ := warmupResp["meta"].(map[string]interface{})
	total, _ := meta["total"].(float64)
	if int(total) != 3000 {
		t.Fatalf("warmup: expected 3000 total resources, got %v", total)
	}

	const concurrency = 50
	const duration = 2 * time.Second
	latencies := make([][]time.Duration, concurrency)
	var errors, totalCount int64
	var ready, workers sync.WaitGroup
	ready.Add(concurrency)

	for index := 0; index < concurrency; index++ {
		workers.Add(1)
		index := index
		latencies[index] = make([]time.Duration, 0, 200)
		go func() {
			defer workers.Done()
			ready.Done()
			ready.Wait()
			deadline := time.Now().Add(duration)
			for time.Now().Before(deadline) {
				started := time.Now()
				req := httptest.NewRequest(http.MethodGet, "/api/resources?limit=50", nil)
				rec := httptest.NewRecorder()
				handlers.HandleListResources(rec, req)
				if rec.Code != http.StatusOK {
					atomic.AddInt64(&errors, 1)
					continue
				}
				latencies[index] = append(latencies[index], time.Since(started))
				atomic.AddInt64(&totalCount, 1)
			}
		}()
	}
	ready.Wait()
	started := time.Now()
	workers.Wait()
	wallTime := time.Since(started)
	if errors > 0 {
		t.Errorf("got %d error responses", errors)
	}

	all := mergeResourceLoadLatencies(latencies)
	if len(all) == 0 {
		t.Fatal("no successful requests recorded")
	}
	p50 := resourceLoadPercentile(all, 0.50)
	p95 := resourceLoadPercentile(all, 0.95)
	p99 := resourceLoadPercentile(all, 0.99)
	t.Logf("resources 500-node load: %d requests in %v (%.1f rps)", totalCount, wallTime, float64(totalCount)/wallTime.Seconds())
	t.Logf("p50=%v p95=%v p99=%v", p50, p95, p99)

	target := 3 * time.Second
	if os.Getenv("GITHUB_ACTIONS") == "true" {
		target = 4 * time.Second
	}
	if p95 > target {
		resourceLoadOverrun(t, "p95 latency %v exceeds %v budget for 500-node concurrent resources load", p95, target)
	}
	minimum := int64(100)
	if os.Getenv("GITHUB_ACTIONS") == "true" {
		minimum = 40
	}
	if totalCount < minimum {
		resourceLoadOverrun(t, "completed only %d requests, expected at least %d for 500-node concurrent resources load", totalCount, minimum)
	}
}

func buildResourceLoadState(t *testing.T, numNodes int) *models.State {
	t.Helper()
	state := models.NewState()
	const numInstances = 10
	basePerInstance, remainder := numNodes/numInstances, numNodes%numInstances
	nodesSoFar := 0
	for instance := 0; instance < numInstances; instance++ {
		instanceName := fmt.Sprintf("pve%d", instance)
		count := basePerInstance
		if instance < remainder {
			count++
		}
		if count == 0 {
			continue
		}
		nodes := make([]models.Node, count)
		for node := range nodes {
			globalIndex := nodesSoFar + node
			nodes[node] = models.Node{
				ID: fmt.Sprintf("%s:node%d", instanceName, node), Name: fmt.Sprintf("node-%d", globalIndex), Instance: instanceName,
				Status: "online", CPU: float64(globalIndex%80+10) / 100,
				Memory: models.Memory{Usage: float64(globalIndex%60 + 20), Total: 64 << 30, Used: 32 << 30},
				Disk:   models.Disk{Usage: float64(globalIndex%40 + 30), Total: 500 << 30, Used: 250 << 30},
			}
		}
		state.UpdateNodesForInstance(instanceName, nodes)
		vms := make([]models.VM, count*5)
		for vm := range vms {
			nodeIndex := vm / 5
			globalIndex := (nodesSoFar+nodeIndex)*5 + vm%5
			vms[vm] = models.VM{
				ID: fmt.Sprintf("%s:node%d:%d", instanceName, nodeIndex, 1000+globalIndex), VMID: 1000 + globalIndex,
				Name: fmt.Sprintf("vm-%d", globalIndex), Node: fmt.Sprintf("node%d", nodeIndex), Instance: instanceName,
				Status: "running", Type: "qemu", CPU: float64(globalIndex%80+10) / 100,
				Memory: models.Memory{Usage: float64(globalIndex%60 + 20), Total: 4 << 30, Used: 2 << 30},
				Disk:   models.Disk{Usage: float64(globalIndex%40 + 30), Total: 50 << 30, Used: 25 << 30},
			}
		}
		state.UpdateVMsForInstance(instanceName, vms)
		nodesSoFar += count
	}
	return state
}

func resourceLoadOverrun(t *testing.T, format string, args ...interface{}) {
	t.Helper()
	if os.Getenv("GITHUB_ACTIONS") == "true" {
		t.Errorf(format, args...)
		return
	}
	t.Skipf("%s (host CPU contention can cause this locally; CI enforces the budget)", fmt.Sprintf(format, args...))
}

func mergeResourceLoadLatencies(groups [][]time.Duration) []time.Duration {
	var merged []time.Duration
	for _, group := range groups {
		merged = append(merged, group...)
	}
	return merged
}

func resourceLoadPercentile(durations []time.Duration, percentile float64) time.Duration {
	sorted := append([]time.Duration(nil), durations...)
	sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] })
	return sorted[int(float64(len(sorted)-1)*percentile)]
}

type resourceLoadStateProvider struct{ state *models.State }

func (p *resourceLoadStateProvider) ReadSnapshot() models.StateSnapshot { return p.state.GetSnapshot() }