From 1d9e66b24c580a80fa2c2ee5a0953b85ac5baa89 Mon Sep 17 00:00:00 2001 From: Pulse Test Date: Sun, 30 Aug 2026 22:10:44 +0100 Subject: [PATCH] fix(mock): keep provider fixtures fresh on slow demo ticks The public demo runs PULSE_MOCK_UPDATE_INTERVAL=60s, which stretches the 10-cohort supplemental rotation to 600s while the unified registry still marks TrueNAS/VMware/availability sources stale after 120s. Every row those sources own spent ~80% of each rotation flagged as a degraded stale source, so the TrueNAS Storage tab read Healthy 0 / Attention 21 (and the vSphere page all-Attention) despite healthy fixture data. Rebase provider-backed fixtures every tick once the rotation would outlive the 120s freshness budget (at slow ticks that is cheaper per wall-clock second than the default 2s config's once-per-20s cadence), and derive mock supplemental stale thresholds from the actual refresh cadence the same way PVE/PBS/PMG thresholds already derive from their polling intervals. Split the supplemental uptime step so per-tick refreshes do not advance PBS uptime by the rotation-scaled step. --- internal/mock/fixture_graph.go | 24 +++++-- internal/mock/generator.go | 9 ++- internal/mock/integration.go | 33 +++++++++ internal/mock/platform_fixtures_test.go | 70 +++++++++++++++++++ .../monitoring/resource_stale_thresholds.go | 27 ++++++- .../resource_stale_thresholds_test.go | 70 +++++++++++++++++++ 6 files changed, 224 insertions(+), 9 deletions(-) create mode 100644 internal/monitoring/resource_stale_thresholds_test.go diff --git a/internal/mock/fixture_graph.go b/internal/mock/fixture_graph.go index 087d64ff9..c0d99e0cf 100644 --- a/internal/mock/fixture_graph.go +++ b/internal/mock/fixture_graph.go @@ -105,16 +105,26 @@ func (g *FixtureGraph) UpdateMetricCohort( selectedNodes[g.State.Nodes[index].Name] = struct{}{} } - includeSupplemental := cohortIndex == 0 + // Provider-backed demo fixtures are much smaller than the PVE estate and + // normally refresh once per full cohort rotation, keeping their timestamps + // live without forcing every provider resource through every socket delta. + // When a slow tick interval stretches the rotation past the registry's + // freshness budget, they refresh every tick instead — otherwise every + // TrueNAS/VMware/availability row spends most of each rotation flagged as + // a stale source (all-Attention badge noise on the public demo). + supplementalEveryTick := supplementalRefreshEveryTick(currentMockUpdateInterval(), cohortCount) + includeSupplemental := cohortIndex == 0 || supplementalEveryTick + supplementalTicks := int64(cohortCount) + if supplementalEveryTick { + supplementalTicks = 1 + } updateFixtureStateMetricsSelectedAt(&g.State, cfg, now, fixtureMetricSelection{ - proxmoxNodeNames: selectedNodes, - includeSupplemental: includeSupplemental, - uptimeStep: currentMockUpdateStepInt64() * int64(cohortCount), + proxmoxNodeNames: selectedNodes, + includeSupplemental: includeSupplemental, + uptimeStep: currentMockUpdateStepInt64() * int64(cohortCount), + supplementalUptimeStep: currentMockUpdateStepInt64() * supplementalTicks, }) if includeSupplemental { - // Provider-backed demo fixtures are much smaller than the PVE estate and - // refresh once per full cohort rotation. Their timestamps stay live - // without forcing every provider resource through every socket delta. g.PlatformFixtures = rebasePlatformFixtures(g.PlatformFixtures, now) g.AvailabilityFixtures = rebaseAvailabilityFixtures(g.AvailabilityFixtures, now) g.DiscoveryFixtures = buildDiscoveryFixtures(g.State, now) diff --git a/internal/mock/generator.go b/internal/mock/generator.go index 09ce9c9c1..16c5f7ce2 100644 --- a/internal/mock/generator.go +++ b/internal/mock/generator.go @@ -6101,6 +6101,10 @@ type fixtureMetricSelection struct { proxmoxNodeNames map[string]struct{} includeSupplemental bool uptimeStep int64 + // supplementalUptimeStep covers supplemental (non-PVE) uptime counters, + // whose refresh cadence can differ from the node cohort's rotation. Zero + // falls back to uptimeStep. + supplementalUptimeStep int64 } func (selection fixtureMetricSelection) includesProxmoxNode(nodeName string) bool { @@ -6123,6 +6127,9 @@ func updateFixtureStateMetricsSelectedAt( if selection.uptimeStep <= 0 { selection.uptimeStep = currentMockUpdateStepInt64() } + if selection.supplementalUptimeStep <= 0 { + selection.supplementalUptimeStep = selection.uptimeStep + } if selection.includeSupplemental { updateDockerHosts(data, config, refreshNow) @@ -6139,7 +6146,7 @@ func updateFixtureStateMetricsSelectedAt( inst.Status = "online" inst.ConnectionHealth = "healthy" inst.LastSeen = refreshNow.Add(-time.Duration(randIntnSafe(12)) * time.Second) - inst.Uptime += step + inst.Uptime += selection.supplementalUptimeStep if data.ConnectionHealth != nil { data.ConnectionHealth[fmt.Sprintf("pbs-%s", inst.Name)] = true diff --git a/internal/mock/integration.go b/internal/mock/integration.go index e8fd097c6..219946857 100644 --- a/internal/mock/integration.go +++ b/internal/mock/integration.go @@ -88,6 +88,39 @@ func currentMockUpdateStepInt64() int64 { return step } +// supplementalFreshnessBudget mirrors the unified registry's default platform +// stale threshold. Provider-backed fixture timestamps (TrueNAS, VMware, +// availability) that age past it read as stale sources, which flips every row +// those sources own to a degraded status. The cohort rotation must therefore +// never let supplemental fixtures age beyond this budget between rebases. +const supplementalFreshnessBudget = 120 * time.Second + +// supplementalRefreshEveryTick reports whether the update loop must rebase +// provider-backed fixtures on every tick. At fast tick rates one rebase per +// cohort rotation keeps them comfortably fresh while bounding per-tick socket +// deltas; once the rotation would outlive the freshness budget (e.g. the +// public demo's 60s PULSE_MOCK_UPDATE_INTERVAL makes a 600s rotation), the +// per-tick rebase is both required for freshness and cheaper per wall-clock +// second than the default configuration's cadence. +func supplementalRefreshEveryTick(interval time.Duration, cohortCount int) bool { + if cohortCount <= 1 { + return true + } + return interval*time.Duration(cohortCount) > supplementalFreshnessBudget +} + +// SupplementalRefreshInterval returns the wall-clock cadence at which the mock +// update loop rebases provider-backed (TrueNAS, VMware, availability) fixture +// timestamps. Stale-threshold derivation uses it the same way real providers +// use their polling intervals. +func SupplementalRefreshInterval() time.Duration { + interval := currentMockUpdateInterval() + if supplementalRefreshEveryTick(interval, mockMetricCohortCount) { + return interval + } + return interval * mockMetricCohortCount +} + // IsMockEnabled returns whether mock mode is enabled. func IsMockEnabled() bool { return enabled.Load() diff --git a/internal/mock/platform_fixtures_test.go b/internal/mock/platform_fixtures_test.go index 1fb6c5e92..947388c7a 100644 --- a/internal/mock/platform_fixtures_test.go +++ b/internal/mock/platform_fixtures_test.go @@ -770,6 +770,76 @@ func TestFixtureGraphMetricCohortsBoundPVEChurnAndCoverTheEstate(t *testing.T) { } } +func TestSupplementalRefreshCadenceTracksFreshnessBudget(t *testing.T) { + t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) }) + + cases := []struct { + name string + interval time.Duration + cohortCount int + everyTick bool + wantInterval time.Duration + }{ + {"default rotation stays within budget", 2 * time.Second, mockMetricCohortCount, false, 20 * time.Second}, + {"rotation at the budget keeps the cohort cadence", 12 * time.Second, mockMetricCohortCount, false, 120 * time.Second}, + {"rotation past the budget refreshes per tick", 13 * time.Second, mockMetricCohortCount, true, 13 * time.Second}, + {"public demo interval refreshes per tick", 60 * time.Second, mockMetricCohortCount, true, 60 * time.Second}, + {"degenerate cohort count refreshes per tick", 2 * time.Second, 1, true, 2 * time.Second}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := supplementalRefreshEveryTick(tc.interval, tc.cohortCount); got != tc.everyTick { + t.Fatalf("supplementalRefreshEveryTick(%s, %d) = %v, want %v", tc.interval, tc.cohortCount, got, tc.everyTick) + } + if tc.cohortCount == mockMetricCohortCount { + setMockUpdateInterval(tc.interval) + if got := SupplementalRefreshInterval(); got != tc.wantInterval { + t.Fatalf("SupplementalRefreshInterval() = %s at %s ticks, want %s", got, tc.interval, tc.wantInterval) + } + } + }) + } +} + +func TestFixtureGraphSlowTicksKeepSupplementalFixturesFresh(t *testing.T) { + t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) }) + + cfg := DefaultConfig + cfg.NodeCount = 4 + cfg.VMsPerNode = 1 + cfg.LXCsPerNode = 1 + cfg.DockerHostCount = 0 + cfg.GenericHostCount = 0 + cfg.K8sClusterCount = 0 + + base := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC) + + // Fast ticks: a non-zero cohort must keep the once-per-rotation cadence. + graph := buildFixtureGraph(cfg, base) + collectedAt := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS) + graph.UpdateMetricCohort(cfg, base.Add(cfg.UpdateInterval), 3, mockMetricCohortCount) + if got := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS); !got.Equal(collectedAt) { + t.Fatalf("fast ticks rebased supplemental fixtures on a non-zero cohort: %s -> %s", collectedAt, got) + } + + // Slow ticks (the public demo's PULSE_MOCK_UPDATE_INTERVAL shape): the + // rotation outlives the registry's freshness budget, so every cohort must + // rebase provider-backed fixtures or their rows read as stale sources. + cfg.UpdateInterval = 60 * time.Second + graph = buildFixtureGraph(cfg, base) + tick := base.Add(cfg.UpdateInterval) + graph.UpdateMetricCohort(cfg, tick, 3, mockMetricCohortCount) + if got := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS); !got.Equal(tick) { + t.Fatalf("slow ticks left TrueNAS fixtures at %s, want rebase to %s", got, tick) + } + if got := graph.PlatformFixtures.VMware.CollectedAt; !got.Equal(tick) { + t.Fatalf("slow ticks left VMware fixtures at %s, want rebase to %s", got, tick) + } + if got := availabilityFixturesFreshness(graph.AvailabilityFixtures); !got.Equal(tick) { + t.Fatalf("slow ticks left availability fixtures at %s, want rebase to %s", got, tick) + } +} + func TestFixtureGraphMetricCohortProducesSparseUnifiedResourceChanges(t *testing.T) { cfg := DefaultConfig cfg.NodeCount = 50 diff --git a/internal/monitoring/resource_stale_thresholds.go b/internal/monitoring/resource_stale_thresholds.go index cc5bccf3b..9e5d63980 100644 --- a/internal/monitoring/resource_stale_thresholds.go +++ b/internal/monitoring/resource_stale_thresholds.go @@ -4,6 +4,7 @@ import ( "time" "github.com/rcourtman/pulse-go-rewrite/internal/config" + "github.com/rcourtman/pulse-go-rewrite/internal/mock" "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" ) @@ -16,7 +17,15 @@ const ( // polling cadence. A source should not be considered stale until it has missed // at least one expected poll cycle plus the normal interval. func ResourceStaleThresholdsForConfig(cfg *config.Config) map[unifiedresources.DataSource]time.Duration { - return map[unifiedresources.DataSource]time.Duration{ + return resourceStaleThresholdsForConfig(cfg, mock.IsMockEnabled(), mock.SupplementalRefreshInterval) +} + +func resourceStaleThresholdsForConfig( + cfg *config.Config, + mockEnabled bool, + mockSupplementalCadence func() time.Duration, +) map[unifiedresources.DataSource]time.Duration { + thresholds := map[unifiedresources.DataSource]time.Duration{ unifiedresources.SourceProxmox: resourceStaleThresholdForPollInterval( effectivePVEPollingIntervalForConfig(cfg), defaultProxmoxResourceStaleThreshold, @@ -30,6 +39,22 @@ func ResourceStaleThresholdsForConfig(cfg *config.Config) map[unifiedresources.D defaultPlatformResourceStaleThreshold, ), } + // Mock mode's provider-backed fixtures (TrueNAS, VMware, availability) + // deliver on the mock update loop's supplemental cadence rather than a + // real poll schedule. Derive their freshness from that cadence the same + // way the entries above derive theirs, so a slow mock tick (e.g. a large + // PULSE_MOCK_UPDATE_INTERVAL on the public demo) does not flag every + // provider-owned row as a stale source between refreshes. + if mockEnabled && mockSupplementalCadence != nil { + supplemental := resourceStaleThresholdForPollInterval( + mockSupplementalCadence(), + defaultPlatformResourceStaleThreshold, + ) + thresholds[unifiedresources.SourceTrueNAS] = supplemental + thresholds[unifiedresources.SourceVMware] = supplemental + thresholds[unifiedresources.SourceAvailability] = supplemental + } + return thresholds } func (m *Monitor) resourceStaleThresholds() map[unifiedresources.DataSource]time.Duration { diff --git a/internal/monitoring/resource_stale_thresholds_test.go b/internal/monitoring/resource_stale_thresholds_test.go new file mode 100644 index 000000000..71507dc5c --- /dev/null +++ b/internal/monitoring/resource_stale_thresholds_test.go @@ -0,0 +1,70 @@ +package monitoring + +import ( + "testing" + "time" + + "github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources" +) + +func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) { + supplementalSources := []unifiedresources.DataSource{ + unifiedresources.SourceTrueNAS, + unifiedresources.SourceVMware, + unifiedresources.SourceAvailability, + } + + t.Run("mock disabled leaves supplemental sources on registry defaults", func(t *testing.T) { + thresholds := resourceStaleThresholdsForConfig(nil, false, func() time.Duration { + t.Fatal("supplemental cadence must not be consulted while mock mode is off") + return 0 + }) + for _, source := range supplementalSources { + if _, ok := thresholds[source]; ok { + t.Fatalf("unexpected %s threshold override with mock disabled", source) + } + } + }) + + t.Run("fast mock cadence keeps the default platform threshold", func(t *testing.T) { + thresholds := resourceStaleThresholdsForConfig(nil, true, func() time.Duration { + return 20 * time.Second + }) + for _, source := range supplementalSources { + if got := thresholds[source]; got != defaultPlatformResourceStaleThreshold { + t.Fatalf("%s threshold = %s, want default %s", source, got, defaultPlatformResourceStaleThreshold) + } + } + }) + + // The public demo runs PULSE_MOCK_UPDATE_INTERVAL well above the default; + // a fixed 120s threshold there flagged every TrueNAS/VMware/availability + // row as a stale source for most of each refresh cycle. + t.Run("slow mock cadence widens supplemental thresholds", func(t *testing.T) { + thresholds := resourceStaleThresholdsForConfig(nil, true, func() time.Duration { + return 300 * time.Second + }) + want := 600 * time.Second + for _, source := range supplementalSources { + if got := thresholds[source]; got != want { + t.Fatalf("%s threshold = %s, want %s", source, got, want) + } + } + }) + + t.Run("supplemental derivation leaves polled sources untouched", func(t *testing.T) { + base := resourceStaleThresholdsForConfig(nil, false, nil) + derived := resourceStaleThresholdsForConfig(nil, true, func() time.Duration { + return 300 * time.Second + }) + for _, source := range []unifiedresources.DataSource{ + unifiedresources.SourceProxmox, + unifiedresources.SourcePBS, + unifiedresources.SourcePMG, + } { + if base[source] != derived[source] { + t.Fatalf("%s threshold changed with mock enabled: %s -> %s", source, base[source], derived[source]) + } + } + }) +}