mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-24 04:33:17 +00:00
Stop mock metrics seeding when its monitor is cancelled
A tenant monitor's loop only sees its cancelled context after startMockMetricsSampler returns, and seeding the 50-node demo estate took 4-11s on CI runners (about 2.7s on an M-series Mac). Organization deletion cancels the loop and waits for it, so in mock mode every deletion of a just-used organization was held for the whole seed, and under the 2s budget #2518 introduced it was refused with tenant_shutdown_incomplete (Core E2E 13-api-token-scope since 15:03 UTC on 2026-10-05). Seeding now checks the context before each resource and returns ctx.Err() once cancelled, so the loop exits within a millisecond of the cancel. Series already sampled are recorded whole. A cancelled seed is never stored as the shared template other tenants reuse, and a monitor cancelled while it waited on the template lock returns without seeding. startMockMetricsSampler then skips the chart-cache warm-up and the sampler goroutine. #2529 separately gives deletion a longer bounded wait for any slow loop; this removes the mock-mode cause of the wait. Contract-Neutral: Mock-mode seeding stops early on monitor cancellation; no route, field, schema, persistence format or provider behaviour changes.
This commit is contained in:
@@ -540,13 +540,18 @@ func canonicalMetricSeriesWithSampler(sampler mock.MetricSampler, resourceType,
|
||||
return sampler.SampleMetricSeries(resourceType, resourceID, metric, timestamps)
|
||||
}
|
||||
|
||||
func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.FixtureGraph, now time.Time, seedDuration, interval time.Duration) {
|
||||
// seedMockMetricsHistory returns ctx.Err() when ctx is cancelled partway. A
|
||||
// large demo estate takes several seconds to seed, and a tenant monitor's loop
|
||||
// only sees its cancelled context once seeding returns, so organization
|
||||
// deletion would otherwise wait for the whole estate. Each series that was
|
||||
// sampled is recorded whole; the rest are skipped.
|
||||
func seedMockMetricsHistory(ctx context.Context, mh *MetricsHistory, ms *metrics.Store, graph mock.FixtureGraph, now time.Time, seedDuration, interval time.Duration) error {
|
||||
if mh == nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
state := graph.State
|
||||
if seedDuration <= 0 || interval <= 0 {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
sampler := mock.NewMetricSampler(graph)
|
||||
now = normalizeMockMetricTimestamp(now, interval)
|
||||
@@ -669,6 +674,9 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
|
||||
}
|
||||
}
|
||||
recordStorageTimeline := func(storageID string, currentTotal float64) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(storageID) == "" || currentTotal <= 0 || numPoints == 0 {
|
||||
return
|
||||
}
|
||||
@@ -693,6 +701,9 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
|
||||
}
|
||||
|
||||
recordNode := func(node models.Node) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
if node.ID == "" {
|
||||
return
|
||||
}
|
||||
@@ -719,7 +730,7 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
|
||||
includeDiskIO bool,
|
||||
includeNetwork bool,
|
||||
) {
|
||||
if len(metricIDs) == 0 || storeID == "" {
|
||||
if ctx.Err() != nil || len(metricIDs) == 0 || storeID == "" {
|
||||
return
|
||||
}
|
||||
uniqueMetricIDs := make([]string, 0, len(metricIDs))
|
||||
@@ -784,7 +795,7 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
|
||||
seedStoreSeries(storeType, storeID, storeMetrics...)
|
||||
}
|
||||
recordGuestTemperature := func(metricIDs []string, storeType, storeID string) {
|
||||
if len(metricIDs) == 0 || strings.TrimSpace(storeID) == "" {
|
||||
if ctx.Err() != nil || len(metricIDs) == 0 || strings.TrimSpace(storeID) == "" {
|
||||
return
|
||||
}
|
||||
temperatureSeries := canonicalMetricSeriesWithSampler(
|
||||
@@ -802,7 +813,7 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
|
||||
seedStoreSeries(storeType, storeID, "temperature")
|
||||
}
|
||||
recordGuestMemoryUsed := func(metricIDs []string, storeType, storeID string, memoryTotal float64) {
|
||||
if len(metricIDs) == 0 || strings.TrimSpace(storeID) == "" || memoryTotal <= 0 {
|
||||
if ctx.Err() != nil || len(metricIDs) == 0 || strings.TrimSpace(storeID) == "" || memoryTotal <= 0 {
|
||||
return
|
||||
}
|
||||
percentSeries := canonicalMetricSeriesWithSampler(sampler, storeType, storeID, "memory", seedTimestamps)
|
||||
@@ -915,6 +926,9 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
|
||||
// series for one physical disk, shared by the native and TrueNAS fixture
|
||||
// disk loops below.
|
||||
seedDiskTelemetry := func(resourceID string) {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
tempSeries := canonicalMetricSeriesWithSampler(sampler, "disk", resourceID, "smart_temp", seedTimestamps)
|
||||
busySeries := canonicalMetricSeriesWithSampler(sampler, "disk", resourceID, "disk", seedTimestamps)
|
||||
diskReadSeries := canonicalMetricSeriesWithSampler(sampler, "disk", resourceID, "diskread", seedTimestamps)
|
||||
@@ -1025,6 +1039,9 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
|
||||
recordGuestTemperature(systemMetricIDs, "agent", trueNASFixtures.System.Hostname)
|
||||
|
||||
for _, pool := range trueNASFixtures.Pools {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
poolKey := mock.TrueNASPoolMetricID(trueNASFixtures.System.Hostname, pool.Name)
|
||||
diskSeries := canonicalMetricSeriesWithSampler(sampler, "storage", poolKey, "usage", seedTimestamps)
|
||||
mh.addGuestMetricSeries(poolKey, "disk", diskSeries, seedTimestamps)
|
||||
@@ -1032,6 +1049,9 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
|
||||
}
|
||||
|
||||
for _, dataset := range trueNASFixtures.Datasets {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
dsKey := mock.TrueNASDatasetMetricID(trueNASFixtures.System.Hostname, dataset.Name)
|
||||
totalBytes := dataset.UsedBytes + dataset.AvailBytes
|
||||
diskSeries := canonicalMetricSeriesWithSampler(sampler, "storage", dsKey, "usage", seedTimestamps)
|
||||
@@ -1115,6 +1135,9 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
|
||||
}
|
||||
|
||||
flushStoreBatch()
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
if ms != nil {
|
||||
log.Info().
|
||||
Int("rows", storeRows).
|
||||
@@ -1122,9 +1145,11 @@ func seedMockMetricsHistory(mh *MetricsHistory, ms *metrics.Store, graph mock.Fi
|
||||
Msg("mock seeding: metrics store backfill completed")
|
||||
}
|
||||
log.Debug().Msg("mock seeding: completed")
|
||||
return nil
|
||||
}
|
||||
|
||||
func prepareMockMetricsHistory(
|
||||
ctx context.Context,
|
||||
graph mock.FixtureGraph,
|
||||
fixtureRevision uint64,
|
||||
now time.Time,
|
||||
@@ -1132,7 +1157,7 @@ func prepareMockMetricsHistory(
|
||||
interval time.Duration,
|
||||
maxDataPoints int,
|
||||
storeSink *metrics.Store,
|
||||
) (*MetricsHistory, bool) {
|
||||
) (*MetricsHistory, bool, error) {
|
||||
// Enforce the eager-history cardinality limit at the allocation/cache
|
||||
// boundary. Callers may retain the complete fixture graph for inventory and
|
||||
// realtime updates, but neither the active history nor the reusable seed
|
||||
@@ -1141,8 +1166,10 @@ func prepareMockMetricsHistory(
|
||||
seedAt := normalizeMockMetricTimestamp(now, interval)
|
||||
if storeSink != nil {
|
||||
history := NewMetricsHistory(maxDataPoints, seedDuration)
|
||||
seedMockMetricsHistory(history, storeSink, graph, seedAt, seedDuration, interval)
|
||||
return history, false
|
||||
if err := seedMockMetricsHistory(ctx, history, storeSink, graph, seedAt, seedDuration, interval); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return history, false, nil
|
||||
}
|
||||
|
||||
key := mockMetricsSeedCacheKey{
|
||||
@@ -1155,12 +1182,20 @@ func prepareMockMetricsHistory(
|
||||
|
||||
mockMetricsSeedCache.Lock()
|
||||
defer mockMetricsSeedCache.Unlock()
|
||||
// The lock is held for a whole seed, so a monitor may have been cancelled
|
||||
// while it waited behind another tenant's.
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if mockMetricsSeedCache.history != nil && mockMetricsSeedCache.key == key {
|
||||
return mockMetricsSeedCache.history.clone(), true
|
||||
return mockMetricsSeedCache.history.clone(), true, nil
|
||||
}
|
||||
|
||||
template := NewMetricsHistory(maxDataPoints, seedDuration)
|
||||
seedMockMetricsHistory(template, nil, graph, seedAt, seedDuration, interval)
|
||||
if err := seedMockMetricsHistory(ctx, template, nil, graph, seedAt, seedDuration, interval); err != nil {
|
||||
// A partial template must never be reused by another tenant.
|
||||
return nil, false, err
|
||||
}
|
||||
mockMetricsSeedCache.key = key
|
||||
mockMetricsSeedCache.history = template
|
||||
mockMetricsSeedCache.generation++
|
||||
@@ -1168,7 +1203,7 @@ func prepareMockMetricsHistory(
|
||||
time.AfterFunc(mockMetricsSeedCacheTTL, func() {
|
||||
expireMockMetricsSeedCache(generation)
|
||||
})
|
||||
return template.clone(), false
|
||||
return template.clone(), false, nil
|
||||
}
|
||||
|
||||
// recordTrueNASFixturesMetrics records live fixture ticks for TrueNAS host,
|
||||
@@ -1911,7 +1946,8 @@ func (m *Monitor) startMockMetricsSampler(ctx context.Context) {
|
||||
log.Warn().Msg("PULSE_MOCK_SEED_METRICS_STORE is set but the metrics store is unavailable; mock report history will stay empty")
|
||||
}
|
||||
}
|
||||
history, cacheHit := prepareMockMetricsHistory(
|
||||
history, cacheHit, err := prepareMockMetricsHistory(
|
||||
samplerCtx,
|
||||
historyGraph,
|
||||
fixtureRevision,
|
||||
time.Now(),
|
||||
@@ -1920,6 +1956,13 @@ func (m *Monitor) startMockMetricsSampler(ctx context.Context) {
|
||||
maxPoints,
|
||||
storeSink,
|
||||
)
|
||||
if err != nil {
|
||||
// The monitor is stopping. Return before warming chart caches or
|
||||
// starting the sampler so its loop sees the cancellation now; the
|
||||
// caller's stopMockMetricsSampler releases mockMetricsCancel.
|
||||
log.Info().Err(err).Msg("Mock metrics sampler: seeding cancelled")
|
||||
return
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.metricsHistory = history
|
||||
m.mu.Unlock()
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
||||
)
|
||||
|
||||
// cancelAfterChecks reports cancellation once Err has been consulted more
|
||||
// than allowed times, so a test can cancel seeding partway through without
|
||||
// racing a timer.
|
||||
type cancelAfterChecks struct {
|
||||
context.Context
|
||||
remaining atomic.Int32
|
||||
}
|
||||
|
||||
func newCancelAfterChecks(allowed int32) *cancelAfterChecks {
|
||||
ctx := &cancelAfterChecks{Context: context.Background()}
|
||||
ctx.remaining.Store(allowed)
|
||||
return ctx
|
||||
}
|
||||
|
||||
func (c *cancelAfterChecks) Err() error {
|
||||
if c.remaining.Add(-1) < 0 {
|
||||
return context.Canceled
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func threeNodeMockGraph() mock.FixtureGraph {
|
||||
return fixtureGraphWithState(models.StateSnapshot{
|
||||
Nodes: []models.Node{
|
||||
{ID: "node-a", Name: "node-a", Status: "online", CPU: 0.2, Memory: models.Memory{Usage: 40, Total: 1024}},
|
||||
{ID: "node-b", Name: "node-b", Status: "online", CPU: 0.3, Memory: models.Memory{Usage: 50, Total: 1024}},
|
||||
{ID: "node-c", Name: "node-c", Status: "online", CPU: 0.4, Memory: models.Memory{Usage: 60, Total: 1024}},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
// Org deletion refused with tenant_shutdown_incomplete because a tenant
|
||||
// monitor's loop only sees its cancelled context after mock seeding returns,
|
||||
// and seeding a large demo estate took 4-11s in CI. Seeding must stop at the
|
||||
// next resource instead of finishing the estate.
|
||||
func TestSeedMockMetricsHistoryStopsPartwayWhenCancelled(t *testing.T) {
|
||||
now := time.Now().UTC().Truncate(time.Minute)
|
||||
mh := NewMetricsHistory(3500, time.Hour)
|
||||
|
||||
err := seedMockMetricsHistory(newCancelAfterChecks(1), mh, nil, threeNodeMockGraph(), now, time.Hour, time.Minute)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatalf("seedMockMetricsHistory error = %v, want context.Canceled", err)
|
||||
}
|
||||
if len(mh.GetNodeMetrics("node-a", "cpu", time.Hour)) == 0 {
|
||||
t.Fatal("node seeded before cancellation has no history")
|
||||
}
|
||||
for _, nodeID := range []string{"node-b", "node-c"} {
|
||||
if got := len(mh.GetNodeMetrics(nodeID, "cpu", time.Hour)); got != 0 {
|
||||
t.Fatalf("node %s seeded %d points after cancellation", nodeID, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareMockMetricsHistoryDoesNotCacheCancelledSeed(t *testing.T) {
|
||||
resetMockMetricsSeedCacheForTest(t)
|
||||
now := time.Now().UTC().Truncate(time.Minute)
|
||||
graph := threeNodeMockGraph()
|
||||
|
||||
// One check passes the cache lock, one seeds node-a, then cancellation.
|
||||
history, cacheHit, err := prepareMockMetricsHistory(newCancelAfterChecks(2), graph, 7, now, time.Hour, time.Minute, 3500, nil)
|
||||
if !errors.Is(err, context.Canceled) || history != nil || cacheHit {
|
||||
t.Fatalf("cancelled prepare = (%v, %v, %v), want (nil, false, context.Canceled)", history, cacheHit, err)
|
||||
}
|
||||
mockMetricsSeedCache.Lock()
|
||||
cached := mockMetricsSeedCache.history
|
||||
mockMetricsSeedCache.Unlock()
|
||||
if cached != nil {
|
||||
t.Fatal("cancelled seed left a partial template for other tenants to reuse")
|
||||
}
|
||||
|
||||
history, cacheHit, err = prepareMockMetricsHistory(context.Background(), graph, 7, now, time.Hour, time.Minute, 3500, nil)
|
||||
if err != nil || cacheHit {
|
||||
t.Fatalf("seed after cancellation = (cacheHit %v, err %v), want a fresh seed", cacheHit, err)
|
||||
}
|
||||
if len(history.GetNodeMetrics("node-c", "cpu", time.Hour)) == 0 {
|
||||
t.Fatal("fresh seed after cancellation is incomplete")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStartMockMetricsSamplerSkipsSeedingWhenMonitorStopping(t *testing.T) {
|
||||
setMockSamplerTestEnv(t, time.Hour, time.Minute)
|
||||
resetMockMetricsSeedCacheForTest(t)
|
||||
previousEnabled := mock.IsMockEnabled()
|
||||
previousConfig := mock.GetConfig()
|
||||
t.Cleanup(func() {
|
||||
mustSetMockEnabled(t, false)
|
||||
mock.SetMockConfig(previousConfig)
|
||||
if previousEnabled {
|
||||
mustSetMockEnabled(t, true)
|
||||
}
|
||||
})
|
||||
mustSetMockEnabled(t, false)
|
||||
mock.SetMockConfig(compactMockFixtureConfig())
|
||||
mustSetMockEnabled(t, true)
|
||||
|
||||
nodes := mock.CurrentFixtureGraph().State.Nodes
|
||||
if len(nodes) == 0 {
|
||||
t.Fatal("expected mock fixture nodes")
|
||||
}
|
||||
monitor := &Monitor{
|
||||
metricsHistory: NewMetricsHistory(1000, time.Hour),
|
||||
state: models.NewState(),
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
monitor.startMockMetricsSampler(ctx)
|
||||
t.Cleanup(monitor.stopMockMetricsSampler)
|
||||
|
||||
if got := len(monitor.metricsHistory.GetNodeMetrics(nodes[0].ID, "cpu", time.Hour)); got != 0 {
|
||||
t.Fatalf("stopping monitor seeded %d points for node %s", got, nodes[0].ID)
|
||||
}
|
||||
}
|
||||
@@ -176,7 +176,7 @@ func TestSeedMockMetricsHistory_PopulatesSeries(t *testing.T) {
|
||||
}
|
||||
|
||||
mh := NewMetricsHistory(1000, 24*time.Hour)
|
||||
seedMockMetricsHistory(mh, nil, fixtureGraphWithState(state), now, time.Hour, interval)
|
||||
seedMockMetricsHistory(context.Background(), mh, nil, fixtureGraphWithState(state), now, time.Hour, interval)
|
||||
|
||||
nodeCPU := mh.GetNodeMetrics("node-1", "cpu", time.Hour)
|
||||
if len(nodeCPU) < 10 {
|
||||
@@ -285,7 +285,7 @@ func TestSeedMockMetricsHistory_AppendsSingleTerminalNowPoint(t *testing.T) {
|
||||
}
|
||||
|
||||
mh := NewMetricsHistory(1000, 24*time.Hour)
|
||||
seedMockMetricsHistory(mh, nil, fixtureGraphWithState(state), now, time.Hour, interval)
|
||||
seedMockMetricsHistory(context.Background(), mh, nil, fixtureGraphWithState(state), now, time.Hour, interval)
|
||||
|
||||
memorySeries := mh.GetGuestMetrics("docker:cont-1", "memory", time.Hour)
|
||||
if len(memorySeries) < 2 {
|
||||
@@ -334,7 +334,7 @@ func TestSeedMockMetricsHistory_PopulatesKubernetesPodSeries(t *testing.T) {
|
||||
}
|
||||
|
||||
mh := NewMetricsHistory(1000, 24*time.Hour)
|
||||
seedMockMetricsHistory(mh, nil, fixtureGraphWithState(state), now, time.Hour, interval)
|
||||
seedMockMetricsHistory(context.Background(), mh, nil, fixtureGraphWithState(state), now, time.Hour, interval)
|
||||
|
||||
metricID := kubernetesPodMetricID(state.KubernetesClusters[0], state.KubernetesClusters[0].Pods[0])
|
||||
if metricID == "" {
|
||||
@@ -387,7 +387,7 @@ func TestSeedMockMetricsHistory_PopulatesKubernetesClusterNodeAndDeploymentSerie
|
||||
}
|
||||
|
||||
mh := NewMetricsHistory(1000, 24*time.Hour)
|
||||
seedMockMetricsHistory(mh, nil, fixtureGraphWithState(state), now, time.Hour, interval)
|
||||
seedMockMetricsHistory(context.Background(), mh, nil, fixtureGraphWithState(state), now, time.Hour, interval)
|
||||
|
||||
clusterMetricID := kubernetesClusterMetricID(state.KubernetesClusters[0])
|
||||
nodeMetricID := kubernetesNodeMetricID(state.KubernetesClusters[0], state.KubernetesClusters[0].Nodes[0])
|
||||
@@ -468,7 +468,7 @@ func TestSeedMockMetricsHistory_SeedsMetricsStore(t *testing.T) {
|
||||
defer store.Close()
|
||||
|
||||
mh := NewMetricsHistory(1000, seedDuration)
|
||||
seedMockMetricsHistory(mh, store, fixtureGraphWithState(state), now, seedDuration, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), mh, store, fixtureGraphWithState(state), now, seedDuration, time.Minute)
|
||||
|
||||
points, err := store.Query("vm", "vm-100", "cpu", now.Add(-seedDuration), now, 3600)
|
||||
if err != nil {
|
||||
@@ -531,7 +531,7 @@ func TestSeedMockMetricsHistory_StoreSeedGapFillsWithoutDuplicates(t *testing.T)
|
||||
return len(points)
|
||||
}
|
||||
|
||||
seedMockMetricsHistory(NewMetricsHistory(1000, seedDuration), store, fixtureGraphWithState(state), now, seedDuration, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), NewMetricsHistory(1000, seedDuration), store, fixtureGraphWithState(state), now, seedDuration, time.Minute)
|
||||
first := countPoints(now)
|
||||
if first == 0 {
|
||||
t.Fatal("expected initial store seed to write minute-tier points")
|
||||
@@ -539,7 +539,7 @@ func TestSeedMockMetricsHistory_StoreSeedGapFillsWithoutDuplicates(t *testing.T)
|
||||
|
||||
// Re-seeding at the same instant must not add rows: timestamps sit on the
|
||||
// tier spacing grids and existing coverage short-circuits the backfill.
|
||||
seedMockMetricsHistory(NewMetricsHistory(1000, seedDuration), store, fixtureGraphWithState(state), now, seedDuration, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), NewMetricsHistory(1000, seedDuration), store, fixtureGraphWithState(state), now, seedDuration, time.Minute)
|
||||
if got := countPoints(now); got != first {
|
||||
t.Fatalf("expected idempotent reseed to keep %d points, got %d", first, got)
|
||||
}
|
||||
@@ -560,7 +560,7 @@ func TestSeedMockMetricsHistory_StoreSeedGapFillsWithoutDuplicates(t *testing.T)
|
||||
if gapPoints == 0 {
|
||||
t.Fatal("expected the advanced clock to expose new minute-tier grid points")
|
||||
}
|
||||
seedMockMetricsHistory(NewMetricsHistory(1000, seedDuration), store, fixtureGraphWithState(state), later, seedDuration, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), NewMetricsHistory(1000, seedDuration), store, fixtureGraphWithState(state), later, seedDuration, time.Minute)
|
||||
if got, want := countPoints(later), first+gapPoints; got != want {
|
||||
t.Fatalf("expected gap fill to append exactly the missing grid points (%d), got %d", want, got)
|
||||
}
|
||||
@@ -597,7 +597,7 @@ func TestSeedMockMetricsHistory_SeedsDiskTemperatureMetricsStore(t *testing.T) {
|
||||
defer store.Close()
|
||||
|
||||
mh := NewMetricsHistory(1000, seedDuration)
|
||||
seedMockMetricsHistory(mh, store, fixtureGraphWithState(state), now, seedDuration, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), mh, store, fixtureGraphWithState(state), now, seedDuration, time.Minute)
|
||||
|
||||
points, err := store.Query("disk", "SERIAL-001", "smart_temp", now.Add(-seedDuration), now, 3600)
|
||||
if err != nil {
|
||||
@@ -634,7 +634,7 @@ func TestSeedMockMetricsHistory_SeedsVMwareMetricsStore(t *testing.T) {
|
||||
},
|
||||
}
|
||||
sampler := mock.NewMetricSampler(graph)
|
||||
seedMockMetricsHistory(mh, store, graph, now, seedDuration, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), mh, store, graph, now, seedDuration, time.Minute)
|
||||
|
||||
hostPoints, err := store.Query("agent", "vc-mock-1:host:host-101", "cpu", now.Add(-seedDuration), now, 3600)
|
||||
if err != nil {
|
||||
@@ -707,7 +707,7 @@ func TestSeedMockMetricsHistory_SeedsTrueNASMetricsStore(t *testing.T) {
|
||||
},
|
||||
}
|
||||
sampler := mock.NewMetricSampler(graph)
|
||||
seedMockMetricsHistory(mh, store, graph, now, seedDuration, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), mh, store, graph, now, seedDuration, time.Minute)
|
||||
|
||||
systemPoints, err := store.Query("agent", fixtures.System.Hostname, "disk", now.Add(-seedDuration), now, 3600)
|
||||
if err != nil {
|
||||
@@ -814,7 +814,7 @@ func TestSeedMockMetricsHistory_UsesCanonicalMockFixtureGraphForLegacyAndProvide
|
||||
defer store.Close()
|
||||
|
||||
mh := NewMetricsHistory(1000, historyRetention)
|
||||
seedMockMetricsHistory(mh, store, graph, now, seedDuration, sampleInterval)
|
||||
seedMockMetricsHistory(context.Background(), mh, store, graph, now, seedDuration, sampleInterval)
|
||||
|
||||
nodePoints, err := store.Query("node", graph.State.Nodes[0].ID, "cpu", now.Add(-seedDuration), now, 3600)
|
||||
if err != nil {
|
||||
@@ -1455,7 +1455,7 @@ func TestSeedMockMetricsHistory_StaysContinuousWithSubsequentLiveMockTicks(t *te
|
||||
}
|
||||
|
||||
mh := NewMetricsHistory(5000, seedDuration)
|
||||
seedMockMetricsHistory(mh, nil, fixtureGraphWithState(seedState), now, seedDuration, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), mh, nil, fixtureGraphWithState(seedState), now, seedDuration, time.Minute)
|
||||
|
||||
vmCPUSeeded := mh.GetGuestMetrics("vm-tail", "cpu", seedDuration)
|
||||
if len(vmCPUSeeded) == 0 {
|
||||
@@ -1626,7 +1626,7 @@ func TestRecordMockStateToMetricsHistory_ContinuesCanonicalKubernetesClusterNode
|
||||
deploymentMetricID := kubernetesDeploymentMetricID(seedState.KubernetesClusters[0], seedState.KubernetesClusters[0].Deployments[0])
|
||||
|
||||
mh := NewMetricsHistory(5000, seedDuration)
|
||||
seedMockMetricsHistory(mh, nil, fixtureGraphWithState(seedState), now, seedDuration, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), mh, nil, fixtureGraphWithState(seedState), now, seedDuration, time.Minute)
|
||||
recordMockStateToMetricsHistory(mh, nil, fixtureGraphWithState(seedState), next)
|
||||
|
||||
for name, metricID := range map[string]string{
|
||||
@@ -1664,7 +1664,7 @@ func TestMockMetricsHistoryKeepsGraphPersonaAcrossSeedAndLiveBoundary(t *testing
|
||||
sampler := mock.NewMetricSampler(graph)
|
||||
|
||||
mh := NewMetricsHistory(5000, boundedMockHistoryProofWindow)
|
||||
seedMockMetricsHistory(mh, nil, graph, now, boundedMockHistoryProofWindow, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), mh, nil, graph, now, boundedMockHistoryProofWindow, time.Minute)
|
||||
recordMockStateToMetricsHistory(mh, nil, graph, next)
|
||||
|
||||
series := mh.GetGuestMetrics(resourceID, "memory", boundedMockHistoryProofWindow)
|
||||
@@ -1699,7 +1699,7 @@ func TestSeedMockMetricsHistory_UsesCanonicalMetricModelForPlatformFixtures(t *t
|
||||
}
|
||||
|
||||
mh := NewMetricsHistory(20_000, 4*time.Hour)
|
||||
seedMockMetricsHistory(mh, nil, graph, ts, 4*time.Hour, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), mh, nil, graph, ts, 4*time.Hour, time.Minute)
|
||||
|
||||
assertCanonicalSeries := func(series []MetricPoint, resourceType, resourceID, metric string) {
|
||||
t.Helper()
|
||||
@@ -1765,7 +1765,7 @@ func TestSeedMockMetricsHistory_NormalizesTerminalTimestampToCanonicalGrid(t *te
|
||||
})
|
||||
|
||||
mh := NewMetricsHistory(4_000, 2*time.Hour)
|
||||
seedMockMetricsHistory(mh, nil, graph, now, 2*time.Hour, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), mh, nil, graph, now, 2*time.Hour, time.Minute)
|
||||
|
||||
points := mh.GetGuestMetrics("vm-grid", "cpu", 2*time.Hour)
|
||||
if len(points) == 0 {
|
||||
@@ -2030,7 +2030,7 @@ func TestSeedMockMetricsHistory_DiskTelemetryParityAcrossNativeAndTrueNAS(t *tes
|
||||
}
|
||||
|
||||
mh := NewMetricsHistory(1000, seedDuration)
|
||||
seedMockMetricsHistory(mh, nil, mock.FixtureGraph{
|
||||
seedMockMetricsHistory(context.Background(), mh, nil, mock.FixtureGraph{
|
||||
State: state,
|
||||
PlatformFixtures: mock.PlatformFixtures{
|
||||
TrueNAS: fixtures,
|
||||
@@ -2064,7 +2064,7 @@ func TestMockDockerHostSeedCoversTheSameSeriesAsTheSyntheticGenerator(t *testing
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
seedMockMetricsHistory(history, nil, graph, now, 2*time.Hour, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), history, nil, graph, now, 2*time.Hour, time.Minute)
|
||||
|
||||
hostID := graph.State.DockerHosts[0].ID
|
||||
for _, metricType := range mockHostChartMetricTypes {
|
||||
@@ -2087,7 +2087,7 @@ func TestMockDockerContainerSeedCoversDrawerHistorySeries(t *testing.T) {
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
seedMockMetricsHistory(history, nil, graph, now, 2*time.Hour, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), history, nil, graph, now, 2*time.Hour, time.Minute)
|
||||
|
||||
containerID := graph.State.DockerHosts[0].Containers[0].ID
|
||||
for _, metricType := range mockGuestChartMetricTypes {
|
||||
@@ -2110,7 +2110,7 @@ func TestMockNodeSeedCoversDrawerHistorySeries(t *testing.T) {
|
||||
}
|
||||
|
||||
now := time.Now().UTC()
|
||||
seedMockMetricsHistory(history, nil, graph, now, 2*time.Hour, time.Minute)
|
||||
seedMockMetricsHistory(context.Background(), history, nil, graph, now, 2*time.Hour, time.Minute)
|
||||
|
||||
nodeID := graph.State.Nodes[0].ID
|
||||
for _, metricType := range mockNodeChartMetricTypes {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -52,7 +53,8 @@ func TestPrepareMockMetricsHistoryReusesRevisionSeedWithIndependentHistories(t *
|
||||
},
|
||||
})
|
||||
|
||||
first, firstCacheHit := prepareMockMetricsHistory(
|
||||
first, firstCacheHit, _ := prepareMockMetricsHistory(
|
||||
context.Background(),
|
||||
graph,
|
||||
fixtureRevision,
|
||||
now,
|
||||
@@ -64,7 +66,8 @@ func TestPrepareMockMetricsHistoryReusesRevisionSeedWithIndependentHistories(t *
|
||||
if firstCacheHit {
|
||||
t.Fatal("first seed unexpectedly reported a cache hit")
|
||||
}
|
||||
second, secondCacheHit := prepareMockMetricsHistory(
|
||||
second, secondCacheHit, _ := prepareMockMetricsHistory(
|
||||
context.Background(),
|
||||
graph,
|
||||
fixtureRevision,
|
||||
now,
|
||||
@@ -84,7 +87,8 @@ func TestPrepareMockMetricsHistoryReusesRevisionSeedWithIndependentHistories(t *
|
||||
firstGeneration := mockMetricsSeedCache.generation
|
||||
mockMetricsSeedCache.Unlock()
|
||||
|
||||
_, revisionCacheHit := prepareMockMetricsHistory(
|
||||
_, revisionCacheHit, _ := prepareMockMetricsHistory(
|
||||
context.Background(),
|
||||
graph,
|
||||
fixtureRevision+1,
|
||||
now,
|
||||
@@ -163,7 +167,7 @@ func TestPrepareMockMetricsHistoryBoundsFullDemoEstateBeforeCaching(t *testing.T
|
||||
}
|
||||
|
||||
now := time.Now().UTC().Truncate(time.Minute)
|
||||
history, cacheHit := prepareMockMetricsHistory(full, 99, now, time.Hour, time.Minute, 3500, nil)
|
||||
history, cacheHit, _ := prepareMockMetricsHistory(context.Background(), full, 99, now, time.Hour, time.Minute, 3500, nil)
|
||||
if cacheHit {
|
||||
t.Fatal("first full-estate seed unexpectedly reported a cache hit")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user