mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-27 05:57:27 +00:00
ad122ed150
Alert evaluation resolves a metrics target for every resource and windowed metric. In mock mode that went through the unified view cache, keyed on the fixture data version, which advances on every 2 s metric tick. Building the view for the default 1,776-resource estate costs about half a second of CPU, so a process starved of CPU published each build under an already-stale version and the next lookup built again. Startup runs the first alert pass before the listener opens, so it never got there: 675 of 929 view requests missed at a 16 s average build, and the process had not served at 606 s. A target depends only on a resource's type and source mappings, which a metric tick never changes. Key the lookup on a new structure revision (mock.FixtureStructureRevision, the existing structural-only counter) and the link list instead, and answer from the targets the cached view captured. An ID the estate does not list resolves to nothing without a build. The view itself stays keyed on the data version for every other consumer. The fixture now advances its data version before its structure revision, and a cache fill reads the revision first, so a reader that sees a new revision cannot cache the previous estate's snapshot under it. This is not a regression from 1a094646d7..d2ac0c5112: both commits stall identically under equal CPU contention and both start in 14 to 16 s at normal load. Real mode does not pay the cost in steady state either; a full store pass over 1,694 resources takes about 108 ms. On pulse-dev, with the process pinned to one core behind four busy loops, the fixed binary serves at 148 s where the previous behaviour had not served at 606 s. The remaining startup time there is one view build per consumer per tick, and the monitor's repeated full resource-store passes; neither is changed here.
159 lines
5.0 KiB
Go
159 lines
5.0 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
)
|
|
|
|
// useMockEstate enables the default mock estate with the given node count,
|
|
// whose fixture ticks every interval, and restores the previous mock mode and
|
|
// configuration when the test ends.
|
|
func useMockEstate(tb testing.TB, nodes int, interval time.Duration) {
|
|
tb.Helper()
|
|
previous := mock.IsMockEnabled()
|
|
previousConfig := mock.GetConfig()
|
|
if previous {
|
|
mustSetMockEnabled(tb, false)
|
|
}
|
|
// Start from the default estate, not the ambient one: PULSE_MOCK_* settings
|
|
// can remove the standalone agents and clusters these tests look for.
|
|
config := mock.DefaultConfig
|
|
config.NodeCount = nodes
|
|
config.UpdateInterval = interval
|
|
mock.SetMockConfig(config)
|
|
mustSetMockEnabled(tb, true)
|
|
tb.Cleanup(func() {
|
|
mustSetMockEnabled(tb, false)
|
|
mock.SetMockConfig(previousConfig)
|
|
mustSetMockEnabled(tb, previous)
|
|
})
|
|
}
|
|
|
|
func newMockEstateMonitor() *Monitor {
|
|
return &Monitor{
|
|
state: models.NewState(),
|
|
resourceStore: unifiedresources.NewMonitorAdapter(unifiedresources.NewRegistry(unifiedresources.NewMemoryStore())),
|
|
}
|
|
}
|
|
|
|
// publishedMockView returns the view the monitor currently caches for mock
|
|
// mode without building one.
|
|
func publishedMockView(m *Monitor) (monitorUnifiedStateView, bool) {
|
|
m.mockUnifiedViewMu.Lock()
|
|
defer m.mockUnifiedViewMu.Unlock()
|
|
return m.mockUnifiedView, m.mockUnifiedViewValid
|
|
}
|
|
|
|
// sameMetricsTarget reports whether two lookups resolved the same target (or
|
|
// both resolved none).
|
|
func sameMetricsTarget(a, b *unifiedresources.MetricsTarget) bool {
|
|
if a == nil || b == nil {
|
|
return a == b
|
|
}
|
|
return *a == *b
|
|
}
|
|
|
|
func waitForFixtureTicks(tb testing.TB, ticks uint64) {
|
|
tb.Helper()
|
|
target := mock.FixtureDataVersion() + ticks
|
|
// A starved host ticks slowly; the deadline only catches a ticker that
|
|
// is not running at all.
|
|
deadline := time.Now().Add(5 * time.Minute)
|
|
for mock.FixtureDataVersion() < target {
|
|
if time.Now().After(deadline) {
|
|
tb.Fatalf("fixture data version stayed below %d (at %d) for five minutes; the mock ticker is not running", target, mock.FixtureDataVersion())
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func resourceIDsWithTargets(tb testing.TB, view monitorUnifiedStateView) []string {
|
|
tb.Helper()
|
|
ids := make([]string, 0, len(view.resources))
|
|
for _, resource := range view.resources {
|
|
if view.metricsTargets.MetricsTargetForResource(resource.ID) != nil {
|
|
ids = append(ids, resource.ID)
|
|
}
|
|
}
|
|
if len(ids) == 0 {
|
|
tb.Fatal("expected the mock estate to list resources with metrics targets")
|
|
}
|
|
return ids
|
|
}
|
|
|
|
// BenchmarkMockMetricsTargetResolutionAfterTick measures one alert pass's
|
|
// lookups (every resource, three windowed metrics) against a view that was
|
|
// built before the fixture last ticked. It must stay a map read per lookup, so
|
|
// a pass costs microseconds however slowly the process runs, not an estate
|
|
// build per lookup.
|
|
func BenchmarkMockMetricsTargetResolutionAfterTick(b *testing.B) {
|
|
useMockEstate(b, mock.DefaultConfig.NodeCount, time.Second)
|
|
m := newMockEstateMonitor()
|
|
ids := resourceIDsWithTargets(b, m.currentUnifiedStateView())
|
|
waitForFixtureTicks(b, 1)
|
|
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
for _, id := range ids {
|
|
for metric := 0; metric < 3; metric++ {
|
|
if m.MetricsTargetForResource(id) == nil {
|
|
b.Fatalf("no target for %q", id)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
b.ReportMetric(float64(len(ids)*3), "lookups/op")
|
|
}
|
|
|
|
// BenchmarkLiveRegistryMetricsTargetResolution measures the same pass against a
|
|
// store-backed registry, as real mode resolves it, right after an ingest (the
|
|
// registry's typed views not yet built) and once they are.
|
|
func BenchmarkLiveRegistryMetricsTargetResolution(b *testing.B) {
|
|
useMockEstate(b, mock.DefaultConfig.NodeCount, 5*time.Minute)
|
|
graph := mock.CurrentFixtureGraph()
|
|
snapshot := unifiedresources.SnapshotWithoutSources(graph.State, mock.SupplementalOwnedSources())
|
|
mustSetMockEnabled(b, false)
|
|
|
|
for _, views := range []string{"after-ingest", "views-built"} {
|
|
b.Run(views, func(b *testing.B) {
|
|
registry := unifiedresources.NewRegistry(nil)
|
|
registry.IngestSnapshot(snapshot)
|
|
adapter := unifiedresources.NewMonitorAdapter(registry)
|
|
m := &Monitor{state: models.NewState(), resourceStore: adapter}
|
|
|
|
listed := registry.List()
|
|
ids := make([]string, 0, len(listed))
|
|
for _, resource := range listed {
|
|
ids = append(ids, resource.ID)
|
|
}
|
|
if views == "views-built" {
|
|
adapter.VMs()
|
|
}
|
|
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
if views == "after-ingest" {
|
|
b.StopTimer()
|
|
registry = unifiedresources.NewRegistry(nil)
|
|
registry.IngestSnapshot(snapshot)
|
|
adapter = unifiedresources.NewMonitorAdapter(registry)
|
|
m.resourceStore = adapter
|
|
b.StartTimer()
|
|
}
|
|
for _, id := range ids {
|
|
for metric := 0; metric < 3; metric++ {
|
|
m.MetricsTargetForResource(id)
|
|
}
|
|
}
|
|
}
|
|
b.ReportMetric(float64(len(ids)*3), "lookups/op")
|
|
})
|
|
}
|
|
}
|