mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-10 02:25:56 +00:00
fix(mock): keep provider fixtures fresh on slow demo ticks
The public demo runs PULSE_MOCK_UPDATE_INTERVAL=60s, which stretches the 10-cohort supplemental rotation to 600s while the unified registry still marks TrueNAS/VMware/availability sources stale after 120s. Every row those sources own spent ~80% of each rotation flagged as a degraded stale source, so the TrueNAS Storage tab read Healthy 0 / Attention 21 (and the vSphere page all-Attention) despite healthy fixture data. Rebase provider-backed fixtures every tick once the rotation would outlive the 120s freshness budget (at slow ticks that is cheaper per wall-clock second than the default 2s config's once-per-20s cadence), and derive mock supplemental stale thresholds from the actual refresh cadence the same way PVE/PBS/PMG thresholds already derive from their polling intervals. Split the supplemental uptime step so per-tick refreshes do not advance PBS uptime by the rotation-scaled step.
This commit is contained in:
@@ -105,16 +105,26 @@ func (g *FixtureGraph) UpdateMetricCohort(
|
||||
selectedNodes[g.State.Nodes[index].Name] = struct{}{}
|
||||
}
|
||||
|
||||
includeSupplemental := cohortIndex == 0
|
||||
// Provider-backed demo fixtures are much smaller than the PVE estate and
|
||||
// normally refresh once per full cohort rotation, keeping their timestamps
|
||||
// live without forcing every provider resource through every socket delta.
|
||||
// When a slow tick interval stretches the rotation past the registry's
|
||||
// freshness budget, they refresh every tick instead — otherwise every
|
||||
// TrueNAS/VMware/availability row spends most of each rotation flagged as
|
||||
// a stale source (all-Attention badge noise on the public demo).
|
||||
supplementalEveryTick := supplementalRefreshEveryTick(currentMockUpdateInterval(), cohortCount)
|
||||
includeSupplemental := cohortIndex == 0 || supplementalEveryTick
|
||||
supplementalTicks := int64(cohortCount)
|
||||
if supplementalEveryTick {
|
||||
supplementalTicks = 1
|
||||
}
|
||||
updateFixtureStateMetricsSelectedAt(&g.State, cfg, now, fixtureMetricSelection{
|
||||
proxmoxNodeNames: selectedNodes,
|
||||
includeSupplemental: includeSupplemental,
|
||||
uptimeStep: currentMockUpdateStepInt64() * int64(cohortCount),
|
||||
proxmoxNodeNames: selectedNodes,
|
||||
includeSupplemental: includeSupplemental,
|
||||
uptimeStep: currentMockUpdateStepInt64() * int64(cohortCount),
|
||||
supplementalUptimeStep: currentMockUpdateStepInt64() * supplementalTicks,
|
||||
})
|
||||
if includeSupplemental {
|
||||
// Provider-backed demo fixtures are much smaller than the PVE estate and
|
||||
// refresh once per full cohort rotation. Their timestamps stay live
|
||||
// without forcing every provider resource through every socket delta.
|
||||
g.PlatformFixtures = rebasePlatformFixtures(g.PlatformFixtures, now)
|
||||
g.AvailabilityFixtures = rebaseAvailabilityFixtures(g.AvailabilityFixtures, now)
|
||||
g.DiscoveryFixtures = buildDiscoveryFixtures(g.State, now)
|
||||
|
||||
@@ -6101,6 +6101,10 @@ type fixtureMetricSelection struct {
|
||||
proxmoxNodeNames map[string]struct{}
|
||||
includeSupplemental bool
|
||||
uptimeStep int64
|
||||
// supplementalUptimeStep covers supplemental (non-PVE) uptime counters,
|
||||
// whose refresh cadence can differ from the node cohort's rotation. Zero
|
||||
// falls back to uptimeStep.
|
||||
supplementalUptimeStep int64
|
||||
}
|
||||
|
||||
func (selection fixtureMetricSelection) includesProxmoxNode(nodeName string) bool {
|
||||
@@ -6123,6 +6127,9 @@ func updateFixtureStateMetricsSelectedAt(
|
||||
if selection.uptimeStep <= 0 {
|
||||
selection.uptimeStep = currentMockUpdateStepInt64()
|
||||
}
|
||||
if selection.supplementalUptimeStep <= 0 {
|
||||
selection.supplementalUptimeStep = selection.uptimeStep
|
||||
}
|
||||
|
||||
if selection.includeSupplemental {
|
||||
updateDockerHosts(data, config, refreshNow)
|
||||
@@ -6139,7 +6146,7 @@ func updateFixtureStateMetricsSelectedAt(
|
||||
inst.Status = "online"
|
||||
inst.ConnectionHealth = "healthy"
|
||||
inst.LastSeen = refreshNow.Add(-time.Duration(randIntnSafe(12)) * time.Second)
|
||||
inst.Uptime += step
|
||||
inst.Uptime += selection.supplementalUptimeStep
|
||||
|
||||
if data.ConnectionHealth != nil {
|
||||
data.ConnectionHealth[fmt.Sprintf("pbs-%s", inst.Name)] = true
|
||||
|
||||
@@ -88,6 +88,39 @@ func currentMockUpdateStepInt64() int64 {
|
||||
return step
|
||||
}
|
||||
|
||||
// supplementalFreshnessBudget mirrors the unified registry's default platform
|
||||
// stale threshold. Provider-backed fixture timestamps (TrueNAS, VMware,
|
||||
// availability) that age past it read as stale sources, which flips every row
|
||||
// those sources own to a degraded status. The cohort rotation must therefore
|
||||
// never let supplemental fixtures age beyond this budget between rebases.
|
||||
const supplementalFreshnessBudget = 120 * time.Second
|
||||
|
||||
// supplementalRefreshEveryTick reports whether the update loop must rebase
|
||||
// provider-backed fixtures on every tick. At fast tick rates one rebase per
|
||||
// cohort rotation keeps them comfortably fresh while bounding per-tick socket
|
||||
// deltas; once the rotation would outlive the freshness budget (e.g. the
|
||||
// public demo's 60s PULSE_MOCK_UPDATE_INTERVAL makes a 600s rotation), the
|
||||
// per-tick rebase is both required for freshness and cheaper per wall-clock
|
||||
// second than the default configuration's cadence.
|
||||
func supplementalRefreshEveryTick(interval time.Duration, cohortCount int) bool {
|
||||
if cohortCount <= 1 {
|
||||
return true
|
||||
}
|
||||
return interval*time.Duration(cohortCount) > supplementalFreshnessBudget
|
||||
}
|
||||
|
||||
// SupplementalRefreshInterval returns the wall-clock cadence at which the mock
|
||||
// update loop rebases provider-backed (TrueNAS, VMware, availability) fixture
|
||||
// timestamps. Stale-threshold derivation uses it the same way real providers
|
||||
// use their polling intervals.
|
||||
func SupplementalRefreshInterval() time.Duration {
|
||||
interval := currentMockUpdateInterval()
|
||||
if supplementalRefreshEveryTick(interval, mockMetricCohortCount) {
|
||||
return interval
|
||||
}
|
||||
return interval * mockMetricCohortCount
|
||||
}
|
||||
|
||||
// IsMockEnabled returns whether mock mode is enabled.
|
||||
func IsMockEnabled() bool {
|
||||
return enabled.Load()
|
||||
|
||||
@@ -770,6 +770,76 @@ func TestFixtureGraphMetricCohortsBoundPVEChurnAndCoverTheEstate(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSupplementalRefreshCadenceTracksFreshnessBudget(t *testing.T) {
|
||||
t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
interval time.Duration
|
||||
cohortCount int
|
||||
everyTick bool
|
||||
wantInterval time.Duration
|
||||
}{
|
||||
{"default rotation stays within budget", 2 * time.Second, mockMetricCohortCount, false, 20 * time.Second},
|
||||
{"rotation at the budget keeps the cohort cadence", 12 * time.Second, mockMetricCohortCount, false, 120 * time.Second},
|
||||
{"rotation past the budget refreshes per tick", 13 * time.Second, mockMetricCohortCount, true, 13 * time.Second},
|
||||
{"public demo interval refreshes per tick", 60 * time.Second, mockMetricCohortCount, true, 60 * time.Second},
|
||||
{"degenerate cohort count refreshes per tick", 2 * time.Second, 1, true, 2 * time.Second},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := supplementalRefreshEveryTick(tc.interval, tc.cohortCount); got != tc.everyTick {
|
||||
t.Fatalf("supplementalRefreshEveryTick(%s, %d) = %v, want %v", tc.interval, tc.cohortCount, got, tc.everyTick)
|
||||
}
|
||||
if tc.cohortCount == mockMetricCohortCount {
|
||||
setMockUpdateInterval(tc.interval)
|
||||
if got := SupplementalRefreshInterval(); got != tc.wantInterval {
|
||||
t.Fatalf("SupplementalRefreshInterval() = %s at %s ticks, want %s", got, tc.interval, tc.wantInterval)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFixtureGraphSlowTicksKeepSupplementalFixturesFresh(t *testing.T) {
|
||||
t.Cleanup(func() { setMockUpdateInterval(DefaultConfig.UpdateInterval) })
|
||||
|
||||
cfg := DefaultConfig
|
||||
cfg.NodeCount = 4
|
||||
cfg.VMsPerNode = 1
|
||||
cfg.LXCsPerNode = 1
|
||||
cfg.DockerHostCount = 0
|
||||
cfg.GenericHostCount = 0
|
||||
cfg.K8sClusterCount = 0
|
||||
|
||||
base := time.Date(2026, 8, 23, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// Fast ticks: a non-zero cohort must keep the once-per-rotation cadence.
|
||||
graph := buildFixtureGraph(cfg, base)
|
||||
collectedAt := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS)
|
||||
graph.UpdateMetricCohort(cfg, base.Add(cfg.UpdateInterval), 3, mockMetricCohortCount)
|
||||
if got := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS); !got.Equal(collectedAt) {
|
||||
t.Fatalf("fast ticks rebased supplemental fixtures on a non-zero cohort: %s -> %s", collectedAt, got)
|
||||
}
|
||||
|
||||
// Slow ticks (the public demo's PULSE_MOCK_UPDATE_INTERVAL shape): the
|
||||
// rotation outlives the registry's freshness budget, so every cohort must
|
||||
// rebase provider-backed fixtures or their rows read as stale sources.
|
||||
cfg.UpdateInterval = 60 * time.Second
|
||||
graph = buildFixtureGraph(cfg, base)
|
||||
tick := base.Add(cfg.UpdateInterval)
|
||||
graph.UpdateMetricCohort(cfg, tick, 3, mockMetricCohortCount)
|
||||
if got := trueNASCollectedAt(graph.PlatformFixtures.TrueNAS); !got.Equal(tick) {
|
||||
t.Fatalf("slow ticks left TrueNAS fixtures at %s, want rebase to %s", got, tick)
|
||||
}
|
||||
if got := graph.PlatformFixtures.VMware.CollectedAt; !got.Equal(tick) {
|
||||
t.Fatalf("slow ticks left VMware fixtures at %s, want rebase to %s", got, tick)
|
||||
}
|
||||
if got := availabilityFixturesFreshness(graph.AvailabilityFixtures); !got.Equal(tick) {
|
||||
t.Fatalf("slow ticks left availability fixtures at %s, want rebase to %s", got, tick)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFixtureGraphMetricCohortProducesSparseUnifiedResourceChanges(t *testing.T) {
|
||||
cfg := DefaultConfig
|
||||
cfg.NodeCount = 50
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||||
)
|
||||
|
||||
@@ -16,7 +17,15 @@ const (
|
||||
// polling cadence. A source should not be considered stale until it has missed
|
||||
// at least one expected poll cycle plus the normal interval.
|
||||
func ResourceStaleThresholdsForConfig(cfg *config.Config) map[unifiedresources.DataSource]time.Duration {
|
||||
return map[unifiedresources.DataSource]time.Duration{
|
||||
return resourceStaleThresholdsForConfig(cfg, mock.IsMockEnabled(), mock.SupplementalRefreshInterval)
|
||||
}
|
||||
|
||||
func resourceStaleThresholdsForConfig(
|
||||
cfg *config.Config,
|
||||
mockEnabled bool,
|
||||
mockSupplementalCadence func() time.Duration,
|
||||
) map[unifiedresources.DataSource]time.Duration {
|
||||
thresholds := map[unifiedresources.DataSource]time.Duration{
|
||||
unifiedresources.SourceProxmox: resourceStaleThresholdForPollInterval(
|
||||
effectivePVEPollingIntervalForConfig(cfg),
|
||||
defaultProxmoxResourceStaleThreshold,
|
||||
@@ -30,6 +39,22 @@ func ResourceStaleThresholdsForConfig(cfg *config.Config) map[unifiedresources.D
|
||||
defaultPlatformResourceStaleThreshold,
|
||||
),
|
||||
}
|
||||
// Mock mode's provider-backed fixtures (TrueNAS, VMware, availability)
|
||||
// deliver on the mock update loop's supplemental cadence rather than a
|
||||
// real poll schedule. Derive their freshness from that cadence the same
|
||||
// way the entries above derive theirs, so a slow mock tick (e.g. a large
|
||||
// PULSE_MOCK_UPDATE_INTERVAL on the public demo) does not flag every
|
||||
// provider-owned row as a stale source between refreshes.
|
||||
if mockEnabled && mockSupplementalCadence != nil {
|
||||
supplemental := resourceStaleThresholdForPollInterval(
|
||||
mockSupplementalCadence(),
|
||||
defaultPlatformResourceStaleThreshold,
|
||||
)
|
||||
thresholds[unifiedresources.SourceTrueNAS] = supplemental
|
||||
thresholds[unifiedresources.SourceVMware] = supplemental
|
||||
thresholds[unifiedresources.SourceAvailability] = supplemental
|
||||
}
|
||||
return thresholds
|
||||
}
|
||||
|
||||
func (m *Monitor) resourceStaleThresholds() map[unifiedresources.DataSource]time.Duration {
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
package monitoring
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
||||
)
|
||||
|
||||
func TestResourceStaleThresholdsDeriveMockSupplementalCadence(t *testing.T) {
|
||||
supplementalSources := []unifiedresources.DataSource{
|
||||
unifiedresources.SourceTrueNAS,
|
||||
unifiedresources.SourceVMware,
|
||||
unifiedresources.SourceAvailability,
|
||||
}
|
||||
|
||||
t.Run("mock disabled leaves supplemental sources on registry defaults", func(t *testing.T) {
|
||||
thresholds := resourceStaleThresholdsForConfig(nil, false, func() time.Duration {
|
||||
t.Fatal("supplemental cadence must not be consulted while mock mode is off")
|
||||
return 0
|
||||
})
|
||||
for _, source := range supplementalSources {
|
||||
if _, ok := thresholds[source]; ok {
|
||||
t.Fatalf("unexpected %s threshold override with mock disabled", source)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("fast mock cadence keeps the default platform threshold", func(t *testing.T) {
|
||||
thresholds := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {
|
||||
return 20 * time.Second
|
||||
})
|
||||
for _, source := range supplementalSources {
|
||||
if got := thresholds[source]; got != defaultPlatformResourceStaleThreshold {
|
||||
t.Fatalf("%s threshold = %s, want default %s", source, got, defaultPlatformResourceStaleThreshold)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
// The public demo runs PULSE_MOCK_UPDATE_INTERVAL well above the default;
|
||||
// a fixed 120s threshold there flagged every TrueNAS/VMware/availability
|
||||
// row as a stale source for most of each refresh cycle.
|
||||
t.Run("slow mock cadence widens supplemental thresholds", func(t *testing.T) {
|
||||
thresholds := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {
|
||||
return 300 * time.Second
|
||||
})
|
||||
want := 600 * time.Second
|
||||
for _, source := range supplementalSources {
|
||||
if got := thresholds[source]; got != want {
|
||||
t.Fatalf("%s threshold = %s, want %s", source, got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("supplemental derivation leaves polled sources untouched", func(t *testing.T) {
|
||||
base := resourceStaleThresholdsForConfig(nil, false, nil)
|
||||
derived := resourceStaleThresholdsForConfig(nil, true, func() time.Duration {
|
||||
return 300 * time.Second
|
||||
})
|
||||
for _, source := range []unifiedresources.DataSource{
|
||||
unifiedresources.SourceProxmox,
|
||||
unifiedresources.SourcePBS,
|
||||
unifiedresources.SourcePMG,
|
||||
} {
|
||||
if base[source] != derived[source] {
|
||||
t.Fatalf("%s threshold changed with mock enabled: %s -> %s", source, base[source], derived[source])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user