fix(mock): keep provider fixtures fresh on slow demo ticks

The public demo runs PULSE_MOCK_UPDATE_INTERVAL=60s, which stretches the
10-cohort supplemental rotation to 600s while the unified registry still
marks TrueNAS/VMware/availability sources stale after 120s. Every row those
sources own spent ~80% of each rotation flagged as a degraded stale source,
so the TrueNAS Storage tab read Healthy 0 / Attention 21 (and the vSphere
page all-Attention) despite healthy fixture data.

Rebase provider-backed fixtures every tick once the rotation would outlive
the 120s freshness budget (at slow ticks that is cheaper per wall-clock
second than the default 2s config's once-per-20s cadence), and derive mock
supplemental stale thresholds from the actual refresh cadence the same way
PVE/PBS/PMG thresholds already derive from their polling intervals. Split
the supplemental uptime step so per-tick refreshes do not advance PBS
uptime by the rotation-scaled step.
This commit is contained in:
Pulse Test
2026-08-30 22:10:44 +01:00
parent a8bc2e044b
commit 1d9e66b24c
6 changed files with 224 additions and 9 deletions
+17 -7
View File
@@ -105,16 +105,26 @@ func (g *FixtureGraph) UpdateMetricCohort(
selectedNodes[g.State.Nodes[index].Name] = struct{}{}
}
includeSupplemental := cohortIndex == 0
// Provider-backed demo fixtures are much smaller than the PVE estate and
// normally refresh once per full cohort rotation, keeping their timestamps
// live without forcing every provider resource through every socket delta.
// When a slow tick interval stretches the rotation past the registry's
// freshness budget, they refresh every tick instead — otherwise every
// TrueNAS/VMware/availability row spends most of each rotation flagged as
// a stale source (all-Attention badge noise on the public demo).
supplementalEveryTick := supplementalRefreshEveryTick(currentMockUpdateInterval(), cohortCount)
includeSupplemental := cohortIndex == 0 || supplementalEveryTick
supplementalTicks := int64(cohortCount)
if supplementalEveryTick {
supplementalTicks = 1
}
updateFixtureStateMetricsSelectedAt(&g.State, cfg, now, fixtureMetricSelection{
proxmoxNodeNames: selectedNodes,
includeSupplemental: includeSupplemental,
uptimeStep: currentMockUpdateStepInt64() * int64(cohortCount),
proxmoxNodeNames: selectedNodes,
includeSupplemental: includeSupplemental,
uptimeStep: currentMockUpdateStepInt64() * int64(cohortCount),
supplementalUptimeStep: currentMockUpdateStepInt64() * supplementalTicks,
})
if includeSupplemental {
// Provider-backed demo fixtures are much smaller than the PVE estate and
// refresh once per full cohort rotation. Their timestamps stay live
// without forcing every provider resource through every socket delta.
g.PlatformFixtures = rebasePlatformFixtures(g.PlatformFixtures, now)
g.AvailabilityFixtures = rebaseAvailabilityFixtures(g.AvailabilityFixtures, now)
g.DiscoveryFixtures = buildDiscoveryFixtures(g.State, now)
+8 -1
View File
@@ -6101,6 +6101,10 @@ type fixtureMetricSelection struct {
proxmoxNodeNames map[string]struct{}
includeSupplemental bool
uptimeStep int64
// supplementalUptimeStep covers supplemental (non-PVE) uptime counters,
// whose refresh cadence can differ from the node cohort's rotation. Zero
// falls back to uptimeStep.
supplementalUptimeStep int64
}
func (selection fixtureMetricSelection) includesProxmoxNode(nodeName string) bool {
@@ -6123,6 +6127,9 @@ func updateFixtureStateMetricsSelectedAt(
if selection.uptimeStep <= 0 {
selection.uptimeStep = currentMockUpdateStepInt64()
}
if selection.supplementalUptimeStep <= 0 {
selection.supplementalUptimeStep = selection.uptimeStep
}
if selection.includeSupplemental {
updateDockerHosts(data, config, refreshNow)
@@ -6139,7 +6146,7 @@ func updateFixtureStateMetricsSelectedAt(
inst.Status = "online"
inst.ConnectionHealth = "healthy"
inst.LastSeen = refreshNow.Add(-time.Duration(randIntnSafe(12)) * time.Second)
inst.Uptime += step
inst.Uptime += selection.supplementalUptimeStep
if data.ConnectionHealth != nil {
data.ConnectionHealth[fmt.Sprintf("pbs-%s", inst.Name)] = true
+33
View File
@@ -88,6 +88,39 @@ func currentMockUpdateStepInt64() int64 {
return step
}
// supplementalFreshnessBudget mirrors the unified registry's default platform
// stale threshold. Provider-backed fixture timestamps (TrueNAS, VMware,
// availability) that age past it read as stale sources, which flips every row
// those sources own to a degraded status. The cohort rotation must therefore
// never let supplemental fixtures age beyond this budget between rebases.
const supplementalFreshnessBudget = 120 * time.Second
// supplementalRefreshEveryTick reports whether the update loop must rebase
// provider-backed fixtures on every tick. At fast tick rates one rebase per
// cohort rotation keeps them comfortably fresh while bounding per-tick socket
// deltas; once the rotation would outlive the freshness budget (e.g. the
// public demo's 60s PULSE_MOCK_UPDATE_INTERVAL makes a 600s rotation), the
// per-tick rebase is both required for freshness and cheaper per wall-clock
// second than the default configuration's cadence.
func supplementalRefreshEveryTick(interval time.Duration, cohortCount int) bool {
if cohortCount <= 1 {
return true
}
return interval*time.Duration(cohortCount) > supplementalFreshnessBudget
}
// SupplementalRefreshInterval returns the wall-clock cadence at which the mock
// update loop rebases provider-backed (TrueNAS, VMware, availability) fixture
// timestamps. Stale-threshold derivation uses it the same way real providers
// use their polling intervals.
func SupplementalRefreshInterval() time.Duration {
interval := currentMockUpdateInterval()
if supplementalRefreshEveryTick(interval, mockMetricCohortCount) {
return interval
}
return interval * mockMetricCohortCount
}
// IsMockEnabled returns whether mock mode is enabled.
func IsMockEnabled() bool {
return enabled.Load()
+70
View File
@@ -770,6 +770,76 @@ func TestFixtureGraphMetricCohortsBoundPVEChurnAndCoverTheEstate(t *testing.T) {
}
}
func TestSupplementalRefreshCadenceTracksFreshnessBudget(t *testing.T) {
t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })
cases := []struct {
name string
interval time.Duration
cohortCount int
everyTick bool
wantInterval time.Duration
}{
{"default rotation stays within budget", 2 * time.Second, mockMetricCohortCount, false, 20 * time.Second},
{"rotation at the budget keeps the cohort cadence", 12 * time.Second, mockMetricCohortCount, false, 120 * time.Second},
{"rotation past the budget refreshes per tick", 13 * time.Second, mockMetricCohortCount, true, 13 * time.Second},
{"public demo interval refreshes per tick", 60 * time.Second, mockMetricCohortCount, true, 60 * time.Second},
{"degenerate cohort count refreshes per tick", 2 * time.Second, 1, true, 2 * time.Second},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := supplementalRefreshEveryTick(tc.interval, tc.cohortCount); got != tc.everyTick {
t.Fatalf("supplementalRefreshEveryTick(%s, %d) = %v, want %v", tc.interval, tc.cohortCount, got, tc.everyTick)
}
if tc.cohortCount == mockMetricCohortCount {
setMockUpdateInterval(tc.interval)
if got := SupplementalRefreshInterval(); got != tc.wantInterval {
t.Fatalf("SupplementalRefreshInterval() = %s at %s ticks, want %s", got, tc.interval, tc.wantInterval)
}
}
})
}
}
func TestFixtureGraphSlowTicksKeepSupplementalFixturesFresh(t *testing.T) {
t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })
cfg := DefaultConfig
cfg.NodeCount = 4
cfg.VMsPerNode = 1
cfg.LXCsPerNode = 1
cfg.DockerHostCount = 0
cfg.GenericHostCount = 0
cfg.K8sClusterCount = 0
base := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC)
// Fast ticks: a non-zero cohort must keep the once-per-rotation cadence.
graph := buildFixtureGraph(cfg, base)
collectedAt := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS)
graph.UpdateMetricCohort(cfg, base.Add(cfg.UpdateInterval), 3, mockMetricCohortCount)
if got := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS); !got.Equal(collectedAt) {
t.Fatalf("fast ticks rebased supplemental fixtures on a non-zero cohort: %s -> %s", collectedAt, got)
}
// Slow ticks (the public demo's PULSE_MOCK_UPDATE_INTERVAL shape): the
// rotation outlives the registry's freshness budget, so every cohort must
// rebase provider-backed fixtures or their rows read as stale sources.
cfg.UpdateInterval = 60 * time.Second
graph = buildFixtureGraph(cfg, base)
tick := base.Add(cfg.UpdateInterval)
graph.UpdateMetricCohort(cfg, tick, 3, mockMetricCohortCount)
if got := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS); !got.Equal(tick) {
t.Fatalf("slow ticks left TrueNAS fixtures at %s, want rebase to %s", got, tick)
}
if got := graph.PlatformFixtures.VMware.CollectedAt; !got.Equal(tick) {
t.Fatalf("slow ticks left VMware fixtures at %s, want rebase to %s", got, tick)
}
if got := availabilityFixturesFreshness(graph.AvailabilityFixtures); !got.Equal(tick) {
t.Fatalf("slow ticks left availability fixtures at %s, want rebase to %s", got, tick)
}
}
func TestFixtureGraphMetricCohortProducesSparseUnifiedResourceChanges(t *testing.T) {
cfg := DefaultConfig
cfg.NodeCount = 50
@@ -4,6 +4,7 @@ import (
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
@@ -16,7 +17,15 @@ const (
// polling cadence. A source should not be considered stale until it has missed
// at least one expected poll cycle plus the normal interval.
func ResourceStaleThresholdsForConfig(cfg *config.Config) map[unifiedresources.DataSource]time.Duration {
return map[unifiedresources.DataSource]time.Duration{
return resourceStaleThresholdsForConfig(cfg, mock.IsMockEnabled(), mock.SupplementalRefreshInterval)
}
func resourceStaleThresholdsForConfig(
cfg *config.Config,
mockEnabled bool,
mockSupplementalCadence func() time.Duration,
) map[unifiedresources.DataSource]time.Duration {
thresholds := map[unifiedresources.DataSource]time.Duration{
unifiedresources.SourceProxmox: resourceStaleThresholdForPollInterval(
effectivePVEPollingIntervalForConfig(cfg),
defaultProxmoxResourceStaleThreshold,
@@ -30,6 +39,22 @@ func ResourceStaleThresholdsForConfig(cfg *config.Config) map[unifiedresources.D
defaultPlatformResourceStaleThreshold,
),
}
// Mock mode's provider-backed fixtures (TrueNAS, VMware, availability)
// deliver on the mock update loop's supplemental cadence rather than a
// real poll schedule. Derive their freshness from that cadence the same
// way the entries above derive theirs, so a slow mock tick (e.g. a large
// PULSE_MOCK_UPDATE_INTERVAL on the public demo) does not flag every
// provider-owned row as a stale source between refreshes.
if mockEnabled && mockSupplementalCadence != nil {
supplemental := resourceStaleThresholdForPollInterval(
mockSupplementalCadence(),
defaultPlatformResourceStaleThreshold,
)
thresholds[unifiedresources.SourceTrueNAS] = supplemental
thresholds[unifiedresources.SourceVMware] = supplemental
thresholds[unifiedresources.SourceAvailability] = supplemental
}
return thresholds
}
func (m *Monitor) resourceStaleThresholds() map[unifiedresources.DataSource]time.Duration {
@@ -0,0 +1,70 @@
package monitoring
import (
"testing"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
)
func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {
supplementalSources := []unifiedresources.DataSource{
unifiedresources.SourceTrueNAS,
unifiedresources.SourceVMware,
unifiedresources.SourceAvailability,
}
t.Run("mock disabled leaves supplemental sources on registry defaults", func(t *testing.T) {
thresholds := resourceStaleThresholdsForConfig(nil, false, func() time.Duration {
t.Fatal("supplemental cadence must not be consulted while mock mode is off")
return 0
})
for _, source := range supplementalSources {
if _, ok := thresholds[source]; ok {
t.Fatalf("unexpected %s threshold override with mock disabled", source)
}
}
})
t.Run("fast mock cadence keeps the default platform threshold", func(t *testing.T) {
thresholds := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {
return 20 * time.Second
})
for _, source := range supplementalSources {
if got := thresholds[source]; got != defaultPlatformResourceStaleThreshold {
t.Fatalf("%s threshold = %s, want default %s", source, got, defaultPlatformResourceStaleThreshold)
}
}
})
// The public demo runs PULSE_MOCK_UPDATE_INTERVAL well above the default;
// a fixed 120s threshold there flagged every TrueNAS/VMware/availability
// row as a stale source for most of each refresh cycle.
t.Run("slow mock cadence widens supplemental thresholds", func(t *testing.T) {
thresholds := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {
return 300 * time.Second
})
want := 600 * time.Second
for _, source := range supplementalSources {
if got := thresholds[source]; got != want {
t.Fatalf("%s threshold = %s, want %s", source, got, want)
}
}
})
t.Run("supplemental derivation leaves polled sources untouched", func(t *testing.T) {
base := resourceStaleThresholdsForConfig(nil, false, nil)
derived := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {
return 300 * time.Second
})
for _, source := range []unifiedresources.DataSource{
unifiedresources.SourceProxmox,
unifiedresources.SourcePBS,
unifiedresources.SourcePMG,
} {
if base[source] != derived[source] {
t.Fatalf("%s threshold changed with mock enabled: %s -> %s", source, base[source], derived[source])
}
}
})
}