Files
pulse/internal/monitoring/metrics_target_resolution_test.go
T
rcourtman ad122ed150 Stop mock target lookups rebuilding the estate view per tick
Alert evaluation resolves a metrics target for every resource and
windowed metric. In mock mode that went through the unified view cache,
keyed on the fixture data version, which advances on every 2 s metric
tick. Building the view for the default 1,776-resource estate costs about
half a second of CPU, so a process starved of CPU published each build
under an already-stale version and the next lookup built again. Startup
runs the first alert pass before the listener opens, so it never got
there: 675 of 929 view requests missed at a 16 s average build, and the
process had not served at 606 s.

A target depends only on a resource's type and source mappings, which a
metric tick never changes. Key the lookup on a new structure revision
(mock.FixtureStructureRevision, the existing structural-only counter)
and the link list instead, and answer from the targets the cached view
captured. An ID the estate does not list resolves to nothing without a
build. The view itself stays keyed on the data version for every other
consumer. The fixture now advances its data version before its structure
revision, and a cache fill reads the revision first, so a reader that sees
a new revision cannot cache the previous estate's snapshot under it.

This is not a regression from 1a094646d7..d2ac0c5112: both commits stall
identically under equal CPU contention and both start in 14 to 16 s at
normal load. Real mode does not pay the cost in steady state either; a
full store pass over 1,694 resources takes about 108 ms.

On pulse-dev, with the process pinned to one core behind four busy
loops, the fixed binary serves at 148 s where the previous behaviour had
not served at 606 s. The remaining startup time there is one view build
per consumer per tick, and the monitor's repeated full resource-store
passes; neither is changed here.
2026-10-10 07:44:50 +01:00

159 lines
5.0 KiB
Go

package monitoring
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
// useMockEstate enables the default mock estate with the given node count,
// whose fixture ticks every interval, and restores the previous mock mode and
// configuration when the test ends.
func useMockEstate(tb testing.TB, nodes int, interval time.Duration) {
tb.Helper()
previous := mock.IsMockEnabled()
previousConfig := mock.GetConfig()
if previous {
mustSetMockEnabled(tb, false)
}
// Start from the default estate, not the ambient one: PULSE_MOCK_* settings
// can remove the standalone agents and clusters these tests look for.
config := mock.DefaultConfig
config.NodeCount = nodes
config.UpdateInterval = interval
mock.SetMockConfig(config)
mustSetMockEnabled(tb, true)
tb.Cleanup(func() {
mustSetMockEnabled(tb, false)
mock.SetMockConfig(previousConfig)
mustSetMockEnabled(tb, previous)
})
}
func newMockEstateMonitor() *Monitor {
return &Monitor{
state: models.NewState(),
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(unifiedresources.NewMemoryStore())),
}
}
// publishedMockView returns the view the monitor currently caches for mock
// mode without building one.
func publishedMockView(m *Monitor) (monitorUnifiedStateView, bool) {
m.mockUnifiedViewMu.Lock()
defer m.mockUnifiedViewMu.Unlock()
return m.mockUnifiedView, m.mockUnifiedViewValid
}
// sameMetricsTarget reports whether two lookups resolved the same target (or
// both resolved none).
func sameMetricsTarget(a, b *unifiedresources.MetricsTarget) bool {
if a == nil || b == nil {
return a == b
}
return *a == *b
}
func waitForFixtureTicks(tb testing.TB, ticks uint64) {
tb.Helper()
target := mock.FixtureDataVersion() + ticks
// A starved host ticks slowly; the deadline only catches a ticker that
// is not running at all.
deadline := time.Now().Add(5 * time.Minute)
for mock.FixtureDataVersion() < target {
if time.Now().After(deadline) {
tb.Fatalf("fixture data version stayed below %d (at %d) for five minutes; the mock ticker is not running", target, mock.FixtureDataVersion())
}
time.Sleep(20 * time.Millisecond)
}
}
func resourceIDsWithTargets(tb testing.TB, view monitorUnifiedStateView) []string {
tb.Helper()
ids := make([]string, 0, len(view.resources))
for _, resource := range view.resources {
if view.metricsTargets.MetricsTargetForResource(resource.ID) != nil {
ids = append(ids, resource.ID)
}
}
if len(ids) == 0 {
tb.Fatal("expected the mock estate to list resources with metrics targets")
}
return ids
}
// BenchmarkMockMetricsTargetResolutionAfterTick measures one alert pass's
// lookups (every resource, three windowed metrics) against a view that was
// built before the fixture last ticked. It must stay a map read per lookup, so
// a pass costs microseconds however slowly the process runs, not an estate
// build per lookup.
func BenchmarkMockMetricsTargetResolutionAfterTick(b *testing.B) {
useMockEstate(b, mock.DefaultConfig.NodeCount, time.Second)
m := newMockEstateMonitor()
ids := resourceIDsWithTargets(b, m.currentUnifiedStateView())
waitForFixtureTicks(b, 1)
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
for _, id := range ids {
for metric := 0; metric < 3; metric++ {
if m.MetricsTargetForResource(id) == nil {
b.Fatalf("no target for %q", id)
}
}
}
}
b.ReportMetric(float64(len(ids)*3), "lookups/op")
}
// BenchmarkLiveRegistryMetricsTargetResolution measures the same pass against a
// store-backed registry, as real mode resolves it, right after an ingest (the
// registry's typed views not yet built) and once they are.
func BenchmarkLiveRegistryMetricsTargetResolution(b *testing.B) {
useMockEstate(b, mock.DefaultConfig.NodeCount, 5*time.Minute)
graph := mock.CurrentFixtureGraph()
snapshot := unifiedresources.SnapshotWithoutSources(graph.State, mock.SupplementalOwnedSources())
mustSetMockEnabled(b, false)
for _, views := range []string{"after-ingest", "views-built"} {
b.Run(views, func(b *testing.B) {
registry := unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(snapshot)
adapter := unifiedresources.NewMonitorAdapter(registry)
m := &Monitor{state: models.NewState(), resourceStore: adapter}
listed := registry.List()
ids := make([]string, 0, len(listed))
for _, resource := range listed {
ids = append(ids, resource.ID)
}
if views == "views-built" {
adapter.VMs()
}
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if views == "after-ingest" {
b.StopTimer()
registry = unifiedresources.NewRegistry(nil)
registry.IngestSnapshot(snapshot)
adapter = unifiedresources.NewMonitorAdapter(registry)
m.resourceStore = adapter
b.StartTimer()
}
for _, id := range ids {
for metric := 0; metric < 3; metric++ {
m.MetricsTargetForResource(id)
}
}
}
b.ReportMetric(float64(len(ids)*3), "lookups/op")
})
}
}