Files
pulse/internal/api/connections_alerts.go
T
courtmanr@gmail.com 5a9e20d971 Scale connection staleness cutoff with the configured polling cadence
The connections ledger marked any PVE/PBS/PMG/VMware/TrueNAS connection
stale once its last successful poll was more than a fixed 2 minutes old,
so polling cadences above 120s read as permanently stale and latched the
connection-degraded alert with no path back to active (#1620).

Derive the active->stale cutoff as max(3 x polling interval, 2 minutes),
mirroring how availability probes already scale. PVE/PBS/PMG cadences
come from the server config; TrueNAS uses the per-instance interval and
VMware the poller summary interval. The 3x multiplier absorbs the
evaluation-order skew where connection alerts are checked concurrently
with the poll on the same scheduler tick, leaving LastSuccess roughly
one interval old even when healthy.

Fixes #1620

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-26 12:42:22 +01:00

154 lines
5.2 KiB
Go

package api
import (
"context"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
"github.com/rcourtman/pulse-go-rewrite/internal/config"
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
"github.com/rcourtman/pulse-go-rewrite/internal/models"
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
)
type aggregatorRuntimeSources struct {
orgID string
vmwarePoller *monitoring.VMwarePoller
truenasPoller *monitoring.TrueNASPoller
}
func buildAggregatorInputsWithRuntimeSources(
ctx context.Context,
cfg *config.Config,
persistence *config.ConfigPersistence,
monitor *monitoring.Monitor,
runtime aggregatorRuntimeSources,
) aggregatorInputs {
inputs := aggregatorInputs{now: time.Now()}
if cfg != nil {
inputs.pveInstances = cfg.PVEInstances
inputs.pbsInstances = cfg.PBSInstances
inputs.pmgInstances = cfg.PMGInstances
inputs.apiTokens = append([]config.APITokenRecord(nil), cfg.APITokens...)
inputs.pvePollingInterval = cfg.PVEPollingInterval
inputs.pbsPollingInterval = cfg.PBSPollingInterval
inputs.pmgPollingInterval = cfg.PMGPollingInterval
}
if persistence != nil {
if vmw, err := persistence.LoadVMwareConfig(); err == nil {
inputs.vmwareInstances = vmw
}
if tn, err := persistence.LoadTrueNASConfig(); err == nil {
inputs.truenasInstances = tn
}
if availability, err := persistence.LoadAvailabilityTargets(); err == nil {
inputs.availabilityTargets = availability
}
}
orgID := runtime.orgID
if orgID == "" {
orgID = "default"
}
if runtime.vmwarePoller != nil && len(inputs.vmwareInstances) > 0 {
inputs.vmwareSummaries = runtime.vmwarePoller.ConnectionSummaries(orgID, inputs.vmwareInstances)
}
if runtime.truenasPoller != nil && len(inputs.truenasInstances) > 0 {
inputs.truenasSummaries = runtime.truenasPoller.ConnectionSummaries(orgID, inputs.truenasInstances)
}
if monitor != nil {
inputs.hosts = monitor.HostsSnapshot()
inputs.agentDesiredConfigs = connectionAgentDesiredConfigFingerprints(monitor, inputs.hosts, inputs.apiTokens)
inputs.instanceHealth = instanceHealthByKey(monitor.SchedulerHealth())
inputs.availabilityStatuses = monitor.AvailabilityStatusSnapshot()
} else {
inputs.hosts = []models.Host{}
inputs.instanceHealth = map[string]monitoring.InstanceHealth{}
inputs.availabilityStatuses = map[string]monitoring.AvailabilityProbeStatus{}
}
if mock.IsMockEnabled() {
mockTargets, mockStatuses := mockAvailabilityConnectionInputs()
inputs.availabilityTargets = mergeAvailabilityTargets(inputs.availabilityTargets, mockTargets)
inputs.availabilityStatuses = mergeAvailabilityStatuses(inputs.availabilityStatuses, mockStatuses)
// Mock vSphere/TrueNAS pollers feed the fabric but never persistence,
// so without these the mock ledger has no platform source rows for
// the machines those integrations monitor.
if len(inputs.vmwareInstances) == 0 {
inputs.vmwareInstances, inputs.vmwareSummaries = mockVMwareLedgerInputs()
}
if len(inputs.truenasInstances) == 0 {
inputs.truenasInstances, inputs.truenasSummaries = mockTrueNASLedgerInputs()
}
}
inputs.expectedAgentVersion = currentAgentTargetVersion()
_ = ctx
return inputs
}
// connectionTypeForAlerts maps the wire ConnectionType to the alerts package's
// narrow platform set. Types that fall outside the platform set return false
// so the caller drops the row before invoking CheckConnection.
func connectionTypeForAlerts(t ConnectionType) (alerts.ConnectionType, bool) {
switch t {
case ConnectionTypePVE:
return alerts.ConnectionTypePVE, true
case ConnectionTypePBS:
return alerts.ConnectionTypePBS, true
case ConnectionTypePMG:
return alerts.ConnectionTypePMG, true
case ConnectionTypeVMware:
return alerts.ConnectionTypeVMware, true
case ConnectionTypeTrueNAS:
return alerts.ConnectionTypeTrueNAS, true
default:
return "", false
}
}
// snapshotConnectionsForAlerts translates the aggregator's rich Connection
// rows into the narrow alerts.ConnectionSnapshot view. Non-platform types
// (agent, availability, docker, kubernetes) are dropped because they have
// their own alert lifecycles.
func snapshotConnectionsForAlerts(connections []Connection) []alerts.ConnectionSnapshot {
out := make([]alerts.ConnectionSnapshot, 0, len(connections))
for _, conn := range connections {
alertType, ok := connectionTypeForAlerts(conn.Type)
if !ok {
continue
}
snap := alerts.ConnectionSnapshot{
ID: conn.ID,
Name: conn.Name,
Type: alertType,
State: alerts.ConnectionState(conn.State),
StateReason: conn.StateReason,
Enabled: conn.Enabled,
LastSeen: conn.LastSeen,
}
if conn.LastError != nil {
snap.LastError = &alerts.ConnectionErrorSnapshot{
At: conn.LastError.At,
Message: conn.LastError.Message,
Category: conn.LastError.Category,
}
}
out = append(out, snap)
}
return out
}
func buildAlertConnectionSnapshotsWithRuntimeSources(
ctx context.Context,
cfg *config.Config,
persistence *config.ConfigPersistence,
monitor *monitoring.Monitor,
runtime aggregatorRuntimeSources,
) []alerts.ConnectionSnapshot {
inputs := buildAggregatorInputsWithRuntimeSources(ctx, cfg, persistence, monitor, runtime)
return snapshotConnectionsForAlerts(buildConnections(inputs))
}