mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-09-11 02:55:51 +00:00
1d9e66b24c
The public demo runs PULSE_MOCK_UPDATE_INTERVAL=60s, which stretches the 10-cohort supplemental rotation to 600s while the unified registry still marks TrueNAS/VMware/availability sources stale after 120s. Every row those sources own spent ~80% of each rotation flagged as a degraded stale source, so the TrueNAS Storage tab read Healthy 0 / Attention 21 (and the vSphere page all-Attention) despite healthy fixture data. Rebase provider-backed fixtures every tick once the rotation would outlive the 120s freshness budget (at slow ticks that is cheaper per wall-clock second than the default 2s config's once-per-20s cadence), and derive mock supplemental stale thresholds from the actual refresh cadence the same way PVE/PBS/PMG thresholds already derive from their polling intervals. Split the supplemental uptime step so per-tick refreshes do not advance PBS uptime by the rotation-scaled step.
458 lines
15 KiB
Go
458 lines
15 KiB
Go
package mock
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/truenas"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/vmware"
|
|
)
|
|
|
|
// FixtureGraph is the canonical mock runtime owner for snapshot-backed and
|
|
// provider-backed fixtures. All mock projections should derive from this graph
|
|
// rather than mixing independent snapshot and provider helpers.
|
|
type FixtureGraph struct {
|
|
State models.StateSnapshot
|
|
AlertHistory []models.Alert
|
|
AlertIncidents []*memory.Incident
|
|
PlatformFixtures PlatformFixtures
|
|
AvailabilityFixtures []AvailabilityFixture
|
|
DiscoveryFixtures []*DiscoveryFixture
|
|
ActionFixtures []ActionFixture
|
|
}
|
|
|
|
func emptyFixtureGraph() FixtureGraph {
|
|
return FixtureGraph{
|
|
State: models.EmptyStateSnapshot(),
|
|
}
|
|
}
|
|
|
|
func buildFixtureGraph(cfg MockConfig, now time.Time) FixtureGraph {
|
|
setMockUpdateInterval(cfg.UpdateInterval)
|
|
graph := FixtureGraph{
|
|
State: buildFixtureState(cfg),
|
|
PlatformFixtures: defaultPlatformFixtures(),
|
|
AvailabilityFixtures: defaultAvailabilityFixtures(now),
|
|
}
|
|
applyDemoScenarioGraph(&graph, cfg, now)
|
|
syncMetricFixtureRegistriesFromGraph(graph)
|
|
graph.UpdateMetrics(cfg, now)
|
|
graph.AlertHistory = buildAlertHistory(graph.State.Nodes, graph.State.VMs, graph.State.Containers)
|
|
graph.AlertHistory, graph.AlertIncidents = buildAlertIncidentFixtures(graph.AlertHistory, now)
|
|
resources, _ := graph.UnifiedResourceSnapshot()
|
|
graph.ActionFixtures = buildActionFixtures(resources, now)
|
|
syncMetricFixtureRegistriesFromGraph(graph)
|
|
return graph
|
|
}
|
|
|
|
func cloneFixtureGraph(in FixtureGraph) FixtureGraph {
|
|
return FixtureGraph{
|
|
State: cloneState(in.State),
|
|
AlertHistory: cloneMockAlerts(in.AlertHistory),
|
|
AlertIncidents: cloneMockIncidents(in.AlertIncidents),
|
|
PlatformFixtures: clonePlatformFixtures(in.PlatformFixtures),
|
|
AvailabilityFixtures: cloneAvailabilityFixtures(in.AvailabilityFixtures),
|
|
DiscoveryFixtures: cloneDiscoveryFixtures(in.DiscoveryFixtures),
|
|
ActionFixtures: cloneActionFixtures(in.ActionFixtures),
|
|
}
|
|
}
|
|
|
|
func (g *FixtureGraph) UpdateMetrics(cfg MockConfig, now time.Time) {
|
|
if g == nil {
|
|
return
|
|
}
|
|
|
|
setMockUpdateInterval(cfg.UpdateInterval)
|
|
applyDemoScenarioGraph(g, cfg, now)
|
|
syncMetricFixtureRegistriesFromGraph(*g)
|
|
updateFixtureStateMetricsAt(&g.State, cfg, now)
|
|
g.PlatformFixtures = rebasePlatformFixtures(g.PlatformFixtures, now)
|
|
g.AvailabilityFixtures = rebaseAvailabilityFixtures(g.AvailabilityFixtures, now)
|
|
applyDemoScenarioGraph(g, cfg, now)
|
|
g.DiscoveryFixtures = buildDiscoveryFixtures(g.State, now)
|
|
syncMetricFixtureRegistriesFromGraph(*g)
|
|
}
|
|
|
|
// UpdateMetricCohort advances one bounded slice of the demo estate. The full
|
|
// UpdateMetrics path remains the fixture-construction and explicit refresh
|
|
// boundary; the long-running mock sampler uses cohorts so a large public demo
|
|
// does not make every resource appear changed on every two-second tick.
|
|
func (g *FixtureGraph) UpdateMetricCohort(
|
|
cfg MockConfig,
|
|
now time.Time,
|
|
cohortIndex int,
|
|
cohortCount int,
|
|
) {
|
|
if g == nil {
|
|
return
|
|
}
|
|
if cohortCount <= 0 {
|
|
cohortCount = 1
|
|
}
|
|
cohortIndex %= cohortCount
|
|
if cohortIndex < 0 {
|
|
cohortIndex += cohortCount
|
|
}
|
|
|
|
setMockUpdateInterval(cfg.UpdateInterval)
|
|
selectedNodes := make(map[string]struct{}, (len(g.State.Nodes)+cohortCount-1)/cohortCount)
|
|
for index := range g.State.Nodes {
|
|
if index%cohortCount != cohortIndex {
|
|
continue
|
|
}
|
|
selectedNodes[g.State.Nodes[index].Name] = struct{}{}
|
|
}
|
|
|
|
// Provider-backed demo fixtures are much smaller than the PVE estate and
|
|
// normally refresh once per full cohort rotation, keeping their timestamps
|
|
// live without forcing every provider resource through every socket delta.
|
|
// When a slow tick interval stretches the rotation past the registry's
|
|
// freshness budget, they refresh every tick instead — otherwise every
|
|
// TrueNAS/VMware/availability row spends most of each rotation flagged as
|
|
// a stale source (all-Attention badge noise on the public demo).
|
|
supplementalEveryTick := supplementalRefreshEveryTick(currentMockUpdateInterval(), cohortCount)
|
|
includeSupplemental := cohortIndex == 0 || supplementalEveryTick
|
|
supplementalTicks := int64(cohortCount)
|
|
if supplementalEveryTick {
|
|
supplementalTicks = 1
|
|
}
|
|
updateFixtureStateMetricsSelectedAt(&g.State, cfg, now, fixtureMetricSelection{
|
|
proxmoxNodeNames: selectedNodes,
|
|
includeSupplemental: includeSupplemental,
|
|
uptimeStep: currentMockUpdateStepInt64() * int64(cohortCount),
|
|
supplementalUptimeStep: currentMockUpdateStepInt64() * supplementalTicks,
|
|
})
|
|
if includeSupplemental {
|
|
g.PlatformFixtures = rebasePlatformFixtures(g.PlatformFixtures, now)
|
|
g.AvailabilityFixtures = rebaseAvailabilityFixtures(g.AvailabilityFixtures, now)
|
|
g.DiscoveryFixtures = buildDiscoveryFixtures(g.State, now)
|
|
}
|
|
syncMetricFixtureRegistriesFromGraph(*g)
|
|
}
|
|
|
|
func (g *FixtureGraph) UpdateAlertSnapshots(active []alerts.Alert, resolved []models.ResolvedAlert) {
|
|
if g == nil {
|
|
return
|
|
}
|
|
|
|
converted := make([]models.Alert, 0, len(active))
|
|
for _, alert := range active {
|
|
converted = append(converted, models.Alert{
|
|
ID: alert.ID,
|
|
Type: alert.Type,
|
|
Level: string(alert.Level),
|
|
ResourceID: alert.ResourceID,
|
|
ResourceName: alert.ResourceName,
|
|
Node: alert.Node,
|
|
Instance: alert.Instance,
|
|
Message: alert.Message,
|
|
Value: alert.Value,
|
|
Threshold: alert.Threshold,
|
|
StartTime: alert.StartTime,
|
|
LastSeen: cloneMockTime(alert.LastSeen),
|
|
Acknowledged: alert.Acknowledged,
|
|
// GetActiveAlerts returns deep clones, so the map is already private.
|
|
Metadata: alert.Metadata,
|
|
})
|
|
}
|
|
|
|
g.State.ActiveAlerts = converted
|
|
g.State.RecentlyResolved = append([]models.ResolvedAlert(nil), resolved...)
|
|
}
|
|
|
|
func CurrentFixtureGraph() FixtureGraph {
|
|
graph, _ := CurrentFixtureGraphWithRevision()
|
|
return graph
|
|
}
|
|
|
|
// CurrentFixtureGraphWithRevision returns one coherent fixture snapshot and
|
|
// its structural revision so downstream caches cannot pair an old graph with
|
|
// a newer invalidation token.
|
|
func CurrentFixtureGraphWithRevision() (FixtureGraph, uint64) {
|
|
dataMu.RLock()
|
|
defer dataMu.RUnlock()
|
|
|
|
if !enabled.Load() {
|
|
return emptyFixtureGraph(), fixtureRevision.Load()
|
|
}
|
|
return cloneFixtureGraph(mockGraph), fixtureRevision.Load()
|
|
}
|
|
|
|
func currentOrDefaultPlatformFixtures() PlatformFixtures {
|
|
if !IsMockEnabled() {
|
|
return defaultPlatformFixtures()
|
|
}
|
|
|
|
return CurrentFixtureGraph().PlatformFixtures
|
|
}
|
|
|
|
func clonePlatformFixtures(in PlatformFixtures) PlatformFixtures {
|
|
return PlatformFixtures{
|
|
TrueNAS: cloneTrueNASFixtureSnapshot(in.TrueNAS),
|
|
VMware: cloneVMwareInventorySnapshot(in.VMware),
|
|
}
|
|
}
|
|
|
|
func cloneTrueNASFixtureSnapshot(in truenas.FixtureSnapshot) truenas.FixtureSnapshot {
|
|
out := in
|
|
out.System = cloneTrueNASSystemInfo(in.System)
|
|
out.Pools = cloneTrueNASPools(in.Pools)
|
|
out.Datasets = append([]truenas.Dataset(nil), in.Datasets...)
|
|
out.Disks = append([]truenas.Disk(nil), in.Disks...)
|
|
out.Alerts = append([]truenas.Alert(nil), in.Alerts...)
|
|
out.Apps = cloneTrueNASApps(in.Apps)
|
|
out.ZFSSnapshots = cloneTrueNASZFSSnapshots(in.ZFSSnapshots)
|
|
out.ReplicationTasks = cloneTrueNASReplicationTasks(in.ReplicationTasks)
|
|
return out
|
|
}
|
|
|
|
func cloneTrueNASPools(in []truenas.Pool) []truenas.Pool {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]truenas.Pool, len(in))
|
|
for i := range in {
|
|
out[i] = in[i]
|
|
if in[i].Scan != nil {
|
|
scan := *in[i].Scan
|
|
out[i].Scan = &scan
|
|
}
|
|
out[i].VDevs = append([]truenas.PoolVDev(nil), in[i].VDevs...)
|
|
out[i].DiskMembers = append([]truenas.PoolDiskMember(nil), in[i].DiskMembers...)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneTrueNASSystemInfo(in truenas.SystemInfo) truenas.SystemInfo {
|
|
out := in
|
|
if len(in.TemperatureCelsius) > 0 {
|
|
out.TemperatureCelsius = make(map[string]float64, len(in.TemperatureCelsius))
|
|
for key, value := range in.TemperatureCelsius {
|
|
out.TemperatureCelsius[key] = value
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneTrueNASApps(in []truenas.App) []truenas.App {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]truenas.App, len(in))
|
|
for i := range in {
|
|
out[i] = in[i]
|
|
out[i].UsedHostIPs = append([]string(nil), in[i].UsedHostIPs...)
|
|
out[i].UsedPorts = cloneTrueNASAppPorts(in[i].UsedPorts)
|
|
out[i].Containers = cloneTrueNASAppContainers(in[i].Containers)
|
|
out[i].Volumes = append([]truenas.AppVolume(nil), in[i].Volumes...)
|
|
out[i].Images = append([]string(nil), in[i].Images...)
|
|
out[i].Networks = cloneTrueNASAppNetworks(in[i].Networks)
|
|
out[i].Stats = cloneTrueNASAppStats(in[i].Stats)
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func cloneTrueNASAppPorts(in []truenas.AppPort) []truenas.AppPort {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]truenas.AppPort, len(in))
|
|
for i := range in {
|
|
out[i] = in[i]
|
|
out[i].HostPorts = append([]truenas.AppHostPort(nil), in[i].HostPorts...)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneTrueNASAppContainers(in []truenas.AppContainer) []truenas.AppContainer {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]truenas.AppContainer, len(in))
|
|
for i := range in {
|
|
out[i] = in[i]
|
|
out[i].PortConfig = cloneTrueNASAppPorts(in[i].PortConfig)
|
|
out[i].VolumeMounts = append([]truenas.AppVolume(nil), in[i].VolumeMounts...)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneTrueNASAppNetworks(in []truenas.AppNetwork) []truenas.AppNetwork {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]truenas.AppNetwork, len(in))
|
|
for i := range in {
|
|
out[i] = in[i]
|
|
out[i].Labels = cloneStringMap(in[i].Labels)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneTrueNASAppStats(in *truenas.AppStats) *truenas.AppStats {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := *in
|
|
out.Interfaces = append([]truenas.AppInterfaceStats(nil), in.Interfaces...)
|
|
return &out
|
|
}
|
|
|
|
func cloneTrueNASZFSSnapshots(in []truenas.ZFSSnapshot) []truenas.ZFSSnapshot {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]truenas.ZFSSnapshot, len(in))
|
|
for i := range in {
|
|
out[i] = in[i]
|
|
out[i].CreatedAt = cloneTimePtr(in[i].CreatedAt)
|
|
out[i].UsedBytes = cloneInt64Ptr(in[i].UsedBytes)
|
|
out[i].Referenced = cloneInt64Ptr(in[i].Referenced)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneTrueNASReplicationTasks(in []truenas.ReplicationTask) []truenas.ReplicationTask {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]truenas.ReplicationTask, len(in))
|
|
for i := range in {
|
|
out[i] = in[i]
|
|
out[i].SourceDatasets = append([]string(nil), in[i].SourceDatasets...)
|
|
out[i].LastRun = cloneTimePtr(in[i].LastRun)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneVMwareInventorySnapshot(in vmware.InventorySnapshot) vmware.InventorySnapshot {
|
|
out := in
|
|
out.Hosts = cloneVMwareInventoryHosts(in.Hosts)
|
|
out.VMs = cloneVMwareInventoryVMs(in.VMs)
|
|
out.Datastores = cloneVMwareInventoryDatastores(in.Datastores)
|
|
out.Networks = cloneVMwareInventoryNetworks(in.Networks)
|
|
out.EnrichmentIssues = append([]vmware.InventoryEnrichmentIssue(nil), in.EnrichmentIssues...)
|
|
return out
|
|
}
|
|
|
|
func cloneVMwareInventoryHosts(in []vmware.InventoryHost) []vmware.InventoryHost {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]vmware.InventoryHost, len(in))
|
|
for i := range in {
|
|
out[i] = in[i]
|
|
out[i].DatastoreIDs = append([]string(nil), in[i].DatastoreIDs...)
|
|
out[i].DatastoreNames = append([]string(nil), in[i].DatastoreNames...)
|
|
out[i].TriggeredAlarms = append([]vmware.InventoryAlarm(nil), in[i].TriggeredAlarms...)
|
|
out[i].RecentTasks = append([]vmware.InventoryTask(nil), in[i].RecentTasks...)
|
|
out[i].RecentEvents = append([]vmware.InventoryEvent(nil), in[i].RecentEvents...)
|
|
out[i].Metrics = cloneVMwareInventoryMetrics(in[i].Metrics)
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func cloneVMwareInventoryVMs(in []vmware.InventoryVM) []vmware.InventoryVM {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]vmware.InventoryVM, len(in))
|
|
for i := range in {
|
|
out[i] = in[i]
|
|
out[i].DatastoreIDs = append([]string(nil), in[i].DatastoreIDs...)
|
|
out[i].DatastoreNames = append([]string(nil), in[i].DatastoreNames...)
|
|
out[i].GuestIPAddresses = append([]string(nil), in[i].GuestIPAddresses...)
|
|
out[i].TriggeredAlarms = append([]vmware.InventoryAlarm(nil), in[i].TriggeredAlarms...)
|
|
out[i].RecentTasks = append([]vmware.InventoryTask(nil), in[i].RecentTasks...)
|
|
out[i].RecentEvents = append([]vmware.InventoryEvent(nil), in[i].RecentEvents...)
|
|
out[i].Metrics = cloneVMwareInventoryMetrics(in[i].Metrics)
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func cloneVMwareInventoryDatastores(in []vmware.InventoryDatastore) []vmware.InventoryDatastore {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]vmware.InventoryDatastore, len(in))
|
|
for i := range in {
|
|
out[i] = in[i]
|
|
out[i].HostIDs = append([]string(nil), in[i].HostIDs...)
|
|
out[i].HostNames = append([]string(nil), in[i].HostNames...)
|
|
out[i].VMIDs = append([]string(nil), in[i].VMIDs...)
|
|
out[i].VMNames = append([]string(nil), in[i].VMNames...)
|
|
out[i].Accessible = cloneBoolPtr(in[i].Accessible)
|
|
out[i].MultipleHostAccess = cloneBoolPtr(in[i].MultipleHostAccess)
|
|
out[i].TriggeredAlarms = append([]vmware.InventoryAlarm(nil), in[i].TriggeredAlarms...)
|
|
out[i].RecentTasks = append([]vmware.InventoryTask(nil), in[i].RecentTasks...)
|
|
out[i].RecentEvents = append([]vmware.InventoryEvent(nil), in[i].RecentEvents...)
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func cloneVMwareInventoryNetworks(in []vmware.InventoryNetwork) []vmware.InventoryNetwork {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]vmware.InventoryNetwork, len(in))
|
|
for i := range in {
|
|
out[i] = in[i]
|
|
out[i].HostIDs = append([]string(nil), in[i].HostIDs...)
|
|
out[i].HostNames = append([]string(nil), in[i].HostNames...)
|
|
out[i].VMIDs = append([]string(nil), in[i].VMIDs...)
|
|
out[i].VMNames = append([]string(nil), in[i].VMNames...)
|
|
out[i].TriggeredAlarms = append([]vmware.InventoryAlarm(nil), in[i].TriggeredAlarms...)
|
|
out[i].RecentTasks = append([]vmware.InventoryTask(nil), in[i].RecentTasks...)
|
|
out[i].RecentEvents = append([]vmware.InventoryEvent(nil), in[i].RecentEvents...)
|
|
}
|
|
|
|
return out
|
|
}
|
|
|
|
func cloneVMwareInventoryMetrics(in *vmware.InventoryMetrics) *vmware.InventoryMetrics {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := *in
|
|
out.CPUPercent = cloneFloat64Ptr(in.CPUPercent)
|
|
out.MemoryPercent = cloneFloat64Ptr(in.MemoryPercent)
|
|
out.MemoryUsedBytes = cloneInt64Ptr(in.MemoryUsedBytes)
|
|
out.MemoryTotalBytes = cloneInt64Ptr(in.MemoryTotalBytes)
|
|
out.NetInBytesPerSecond = cloneFloat64Ptr(in.NetInBytesPerSecond)
|
|
out.NetOutBytesPerSecond = cloneFloat64Ptr(in.NetOutBytesPerSecond)
|
|
out.DiskReadBytesPerSecond = cloneFloat64Ptr(in.DiskReadBytesPerSecond)
|
|
out.DiskWriteBytesPerSecond = cloneFloat64Ptr(in.DiskWriteBytesPerSecond)
|
|
out.UptimeSeconds = cloneInt64Ptr(in.UptimeSeconds)
|
|
out.DiskUsedBytes = cloneInt64Ptr(in.DiskUsedBytes)
|
|
out.DiskTotalBytes = cloneInt64Ptr(in.DiskTotalBytes)
|
|
out.DiskPercent = cloneFloat64Ptr(in.DiskPercent)
|
|
return &out
|
|
}
|
|
|
|
func cloneFloat64Ptr(n *float64) *float64 {
|
|
if n == nil {
|
|
return nil
|
|
}
|
|
value := *n
|
|
return &value
|
|
}
|