mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-27 05:57:27 +00:00
d086054b7e
A tenant monitor's loop only sees its cancelled context after startMockMetricsSampler returns, and seeding the 50-node demo estate took 4-11s on CI runners (about 2.7s on an M-series Mac). Organization deletion cancels the loop and waits for it, so in mock mode every deletion of a just-used organization was held for the whole seed, and under the 2s budget #2518 introduced it was refused with tenant_shutdown_incomplete (Core E2E 13-api-token-scope since 15:03 UTC on 2026-10-05). Seeding now checks the context before each resource and returns ctx.Err() once cancelled, so the loop exits within a millisecond of the cancel. Series already sampled are recorded whole. A cancelled seed is never stored as the shared template other tenants reuse, and a monitor cancelled while it waited on the template lock returns without seeding. startMockMetricsSampler then skips the chart-cache warm-up and the sampler goroutine. #2529 separately gives deletion a longer bounded wait for any slow loop; this removes the mock-mode cause of the wait. Contract-Neutral: Mock-mode seeding stops early on monitor cancellation; no route, field, schema, persistence format or provider behaviour changes.
200 lines
5.6 KiB
Go
200 lines
5.6 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
)
|
|
|
|
func resetMockMetricsSeedCacheForTest(t *testing.T) {
|
|
t.Helper()
|
|
mockMetricsSeedCache.Lock()
|
|
previousKey := mockMetricsSeedCache.key
|
|
previousHistory := mockMetricsSeedCache.history
|
|
previousGeneration := mockMetricsSeedCache.generation
|
|
mockMetricsSeedCache.generation++
|
|
mockMetricsSeedCache.key = mockMetricsSeedCacheKey{}
|
|
mockMetricsSeedCache.history = nil
|
|
mockMetricsSeedCache.Unlock()
|
|
t.Cleanup(func() {
|
|
mockMetricsSeedCache.Lock()
|
|
mockMetricsSeedCache.generation++
|
|
mockMetricsSeedCache.key = previousKey
|
|
mockMetricsSeedCache.history = previousHistory
|
|
if previousHistory != nil {
|
|
mockMetricsSeedCache.generation = previousGeneration
|
|
}
|
|
mockMetricsSeedCache.Unlock()
|
|
})
|
|
}
|
|
|
|
func TestPrepareMockMetricsHistoryReusesRevisionSeedWithIndependentHistories(t *testing.T) {
|
|
resetMockMetricsSeedCacheForTest(t)
|
|
|
|
now := time.Now().UTC().Truncate(time.Minute)
|
|
const (
|
|
fixtureRevision = uint64(41)
|
|
seedDuration = time.Hour
|
|
sampleInterval = time.Minute
|
|
maxDataPoints = 3500
|
|
)
|
|
graph := fixtureGraphWithState(models.StateSnapshot{
|
|
VMs: []models.VM{
|
|
{
|
|
ID: "vm-cache",
|
|
Status: "running",
|
|
CPU: 0.42,
|
|
Memory: models.Memory{Usage: 55, Total: 1024},
|
|
Disk: models.Disk{Usage: 61, Total: 1024, Used: 625},
|
|
},
|
|
},
|
|
})
|
|
|
|
first, firstCacheHit, _ := prepareMockMetricsHistory(
|
|
context.Background(),
|
|
graph,
|
|
fixtureRevision,
|
|
now,
|
|
seedDuration,
|
|
sampleInterval,
|
|
maxDataPoints,
|
|
nil,
|
|
)
|
|
if firstCacheHit {
|
|
t.Fatal("first seed unexpectedly reported a cache hit")
|
|
}
|
|
second, secondCacheHit, _ := prepareMockMetricsHistory(
|
|
context.Background(),
|
|
graph,
|
|
fixtureRevision,
|
|
now,
|
|
seedDuration,
|
|
sampleInterval,
|
|
maxDataPoints,
|
|
nil,
|
|
)
|
|
if !secondCacheHit {
|
|
t.Fatal("second seed did not reuse the matching fixture revision")
|
|
}
|
|
if first == second {
|
|
t.Fatal("cached seed returned shared mutable MetricsHistory pointers")
|
|
}
|
|
|
|
mockMetricsSeedCache.Lock()
|
|
firstGeneration := mockMetricsSeedCache.generation
|
|
mockMetricsSeedCache.Unlock()
|
|
|
|
_, revisionCacheHit, _ := prepareMockMetricsHistory(
|
|
context.Background(),
|
|
graph,
|
|
fixtureRevision+1,
|
|
now,
|
|
seedDuration,
|
|
sampleInterval,
|
|
maxDataPoints,
|
|
nil,
|
|
)
|
|
if revisionCacheHit {
|
|
t.Fatal("new fixture revision reused a stale seed")
|
|
}
|
|
mockMetricsSeedCache.Lock()
|
|
replacementGeneration := mockMetricsSeedCache.generation
|
|
mockMetricsSeedCache.Unlock()
|
|
|
|
expireMockMetricsSeedCache(firstGeneration)
|
|
mockMetricsSeedCache.Lock()
|
|
replacementRetained := mockMetricsSeedCache.history != nil && mockMetricsSeedCache.key.fixtureRevision == fixtureRevision+1
|
|
mockMetricsSeedCache.Unlock()
|
|
if !replacementRetained {
|
|
t.Fatal("stale seed expiry cleared a newer cached fixture revision")
|
|
}
|
|
|
|
expireMockMetricsSeedCache(replacementGeneration)
|
|
mockMetricsSeedCache.Lock()
|
|
cacheReleased := mockMetricsSeedCache.history == nil
|
|
mockMetricsSeedCache.Unlock()
|
|
if !cacheReleased {
|
|
t.Fatal("seed template remained cached after its startup reuse window expired")
|
|
}
|
|
|
|
firstCPU := first.GetGuestMetrics("vm-cache", "cpu", seedDuration)
|
|
secondCPU := second.GetGuestMetrics("vm-cache", "cpu", seedDuration)
|
|
if len(firstCPU) == 0 || len(firstCPU) != len(secondCPU) {
|
|
t.Fatalf("cached seed coverage mismatch: first=%d second=%d", len(firstCPU), len(secondCPU))
|
|
}
|
|
|
|
first.AddGuestMetric("vm-cache", "cpu", 99, now.Add(sampleInterval))
|
|
if got := len(second.GetGuestMetrics("vm-cache", "cpu", seedDuration)); got != len(secondCPU) {
|
|
t.Fatalf("mutating one tenant history changed cached sibling coverage: got=%d want=%d", got, len(secondCPU))
|
|
}
|
|
|
|
}
|
|
|
|
func TestPrepareMockMetricsHistoryBoundsFullDemoEstateBeforeCaching(t *testing.T) {
|
|
resetMockMetricsSeedCacheForTest(t)
|
|
|
|
cfg := mock.DefaultConfig
|
|
cfg.NodeCount = 50
|
|
cfg.VMsPerNode = 10
|
|
cfg.LXCsPerNode = 8
|
|
|
|
previousEnabled := mock.IsMockEnabled()
|
|
previousConfig := mock.GetConfig()
|
|
t.Cleanup(func() {
|
|
mustSetMockEnabled(t, false)
|
|
mock.SetMockConfig(previousConfig)
|
|
if previousEnabled {
|
|
mustSetMockEnabled(t, true)
|
|
}
|
|
})
|
|
mustSetMockEnabled(t, false)
|
|
mock.SetMockConfig(cfg)
|
|
mustSetMockEnabled(t, true)
|
|
|
|
full := mock.CurrentFixtureGraph()
|
|
fullPVEGuests := make(map[string]struct{}, len(full.State.VMs)+len(full.State.Containers))
|
|
for _, vm := range full.State.VMs {
|
|
fullPVEGuests[vm.ID] = struct{}{}
|
|
}
|
|
for _, container := range full.State.Containers {
|
|
fullPVEGuests[container.ID] = struct{}{}
|
|
}
|
|
if len(fullPVEGuests) <= mockEagerHistoryPVEGuestLimit {
|
|
t.Fatalf("fixture contains %d PVE guests, want more than eager limit %d", len(fullPVEGuests), mockEagerHistoryPVEGuestLimit)
|
|
}
|
|
|
|
now := time.Now().UTC().Truncate(time.Minute)
|
|
history, cacheHit, _ := prepareMockMetricsHistory(context.Background(), full, 99, now, time.Hour, time.Minute, 3500, nil)
|
|
if cacheHit {
|
|
t.Fatal("first full-estate seed unexpectedly reported a cache hit")
|
|
}
|
|
|
|
countEagerPVEGuests := func(name string, candidate *MetricsHistory) int {
|
|
t.Helper()
|
|
candidate.mu.RLock()
|
|
defer candidate.mu.RUnlock()
|
|
count := 0
|
|
for id := range candidate.guestMetrics {
|
|
if _, exists := fullPVEGuests[id]; exists {
|
|
count++
|
|
}
|
|
}
|
|
if count != mockEagerHistoryPVEGuestLimit {
|
|
t.Fatalf("%s eagerly retained %d PVE guests, want %d", name, count, mockEagerHistoryPVEGuestLimit)
|
|
}
|
|
return count
|
|
}
|
|
|
|
countEagerPVEGuests("active history", history)
|
|
mockMetricsSeedCache.Lock()
|
|
cached := mockMetricsSeedCache.history
|
|
mockMetricsSeedCache.Unlock()
|
|
if cached == nil {
|
|
t.Fatal("full-estate seed did not populate the reusable cache")
|
|
}
|
|
countEagerPVEGuests("cached template", cached)
|
|
}
|