mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-21 11:15:36 +00:00
5a9e20d971
The connections ledger marked any PVE/PBS/PMG/VMware/TrueNAS connection stale once its last successful poll was more than a fixed 2 minutes old, so polling cadences above 120s read as permanently stale and latched the connection-degraded alert with no path back to active (#1620). Derive the active->stale cutoff as max(3 x polling interval, 2 minutes), mirroring how availability probes already scale. PVE/PBS/PMG cadences come from the server config; TrueNAS uses the per-instance interval and VMware the poller summary interval. The 3x multiplier absorbs the evaluation-order skew where connection alerts are checked concurrently with the poll on the same scheduler tick, leaving LastSuccess roughly one interval old even when healthy. Fixes #1620 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
154 lines
5.2 KiB
Go
154 lines
5.2 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring"
|
|
)
|
|
|
|
type aggregatorRuntimeSources struct {
|
|
orgID string
|
|
vmwarePoller *monitoring.VMwarePoller
|
|
truenasPoller *monitoring.TrueNASPoller
|
|
}
|
|
|
|
func buildAggregatorInputsWithRuntimeSources(
|
|
ctx context.Context,
|
|
cfg *config.Config,
|
|
persistence *config.ConfigPersistence,
|
|
monitor *monitoring.Monitor,
|
|
runtime aggregatorRuntimeSources,
|
|
) aggregatorInputs {
|
|
inputs := aggregatorInputs{now: time.Now()}
|
|
|
|
if cfg != nil {
|
|
inputs.pveInstances = cfg.PVEInstances
|
|
inputs.pbsInstances = cfg.PBSInstances
|
|
inputs.pmgInstances = cfg.PMGInstances
|
|
inputs.apiTokens = append([]config.APITokenRecord(nil), cfg.APITokens...)
|
|
inputs.pvePollingInterval = cfg.PVEPollingInterval
|
|
inputs.pbsPollingInterval = cfg.PBSPollingInterval
|
|
inputs.pmgPollingInterval = cfg.PMGPollingInterval
|
|
}
|
|
|
|
if persistence != nil {
|
|
if vmw, err := persistence.LoadVMwareConfig(); err == nil {
|
|
inputs.vmwareInstances = vmw
|
|
}
|
|
if tn, err := persistence.LoadTrueNASConfig(); err == nil {
|
|
inputs.truenasInstances = tn
|
|
}
|
|
if availability, err := persistence.LoadAvailabilityTargets(); err == nil {
|
|
inputs.availabilityTargets = availability
|
|
}
|
|
}
|
|
|
|
orgID := runtime.orgID
|
|
if orgID == "" {
|
|
orgID = "default"
|
|
}
|
|
if runtime.vmwarePoller != nil && len(inputs.vmwareInstances) > 0 {
|
|
inputs.vmwareSummaries = runtime.vmwarePoller.ConnectionSummaries(orgID, inputs.vmwareInstances)
|
|
}
|
|
if runtime.truenasPoller != nil && len(inputs.truenasInstances) > 0 {
|
|
inputs.truenasSummaries = runtime.truenasPoller.ConnectionSummaries(orgID, inputs.truenasInstances)
|
|
}
|
|
|
|
if monitor != nil {
|
|
inputs.hosts = monitor.HostsSnapshot()
|
|
inputs.agentDesiredConfigs = connectionAgentDesiredConfigFingerprints(monitor, inputs.hosts, inputs.apiTokens)
|
|
inputs.instanceHealth = instanceHealthByKey(monitor.SchedulerHealth())
|
|
inputs.availabilityStatuses = monitor.AvailabilityStatusSnapshot()
|
|
} else {
|
|
inputs.hosts = []models.Host{}
|
|
inputs.instanceHealth = map[string]monitoring.InstanceHealth{}
|
|
inputs.availabilityStatuses = map[string]monitoring.AvailabilityProbeStatus{}
|
|
}
|
|
if mock.IsMockEnabled() {
|
|
mockTargets, mockStatuses := mockAvailabilityConnectionInputs()
|
|
inputs.availabilityTargets = mergeAvailabilityTargets(inputs.availabilityTargets, mockTargets)
|
|
inputs.availabilityStatuses = mergeAvailabilityStatuses(inputs.availabilityStatuses, mockStatuses)
|
|
// Mock vSphere/TrueNAS pollers feed the fabric but never persistence,
|
|
// so without these the mock ledger has no platform source rows for
|
|
// the machines those integrations monitor.
|
|
if len(inputs.vmwareInstances) == 0 {
|
|
inputs.vmwareInstances, inputs.vmwareSummaries = mockVMwareLedgerInputs()
|
|
}
|
|
if len(inputs.truenasInstances) == 0 {
|
|
inputs.truenasInstances, inputs.truenasSummaries = mockTrueNASLedgerInputs()
|
|
}
|
|
}
|
|
inputs.expectedAgentVersion = currentAgentTargetVersion()
|
|
_ = ctx
|
|
return inputs
|
|
}
|
|
|
|
// connectionTypeForAlerts maps the wire ConnectionType to the alerts package's
|
|
// narrow platform set. Types that fall outside the platform set return false
|
|
// so the caller drops the row before invoking CheckConnection.
|
|
func connectionTypeForAlerts(t ConnectionType) (alerts.ConnectionType, bool) {
|
|
switch t {
|
|
case ConnectionTypePVE:
|
|
return alerts.ConnectionTypePVE, true
|
|
case ConnectionTypePBS:
|
|
return alerts.ConnectionTypePBS, true
|
|
case ConnectionTypePMG:
|
|
return alerts.ConnectionTypePMG, true
|
|
case ConnectionTypeVMware:
|
|
return alerts.ConnectionTypeVMware, true
|
|
case ConnectionTypeTrueNAS:
|
|
return alerts.ConnectionTypeTrueNAS, true
|
|
default:
|
|
return "", false
|
|
}
|
|
}
|
|
|
|
// snapshotConnectionsForAlerts translates the aggregator's rich Connection
|
|
// rows into the narrow alerts.ConnectionSnapshot view. Non-platform types
|
|
// (agent, availability, docker, kubernetes) are dropped because they have
|
|
// their own alert lifecycles.
|
|
func snapshotConnectionsForAlerts(connections []Connection) []alerts.ConnectionSnapshot {
|
|
out := make([]alerts.ConnectionSnapshot, 0, len(connections))
|
|
for _, conn := range connections {
|
|
alertType, ok := connectionTypeForAlerts(conn.Type)
|
|
if !ok {
|
|
continue
|
|
}
|
|
|
|
snap := alerts.ConnectionSnapshot{
|
|
ID: conn.ID,
|
|
Name: conn.Name,
|
|
Type: alertType,
|
|
State: alerts.ConnectionState(conn.State),
|
|
StateReason: conn.StateReason,
|
|
Enabled: conn.Enabled,
|
|
LastSeen: conn.LastSeen,
|
|
}
|
|
if conn.LastError != nil {
|
|
snap.LastError = &alerts.ConnectionErrorSnapshot{
|
|
At: conn.LastError.At,
|
|
Message: conn.LastError.Message,
|
|
Category: conn.LastError.Category,
|
|
}
|
|
}
|
|
out = append(out, snap)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func buildAlertConnectionSnapshotsWithRuntimeSources(
|
|
ctx context.Context,
|
|
cfg *config.Config,
|
|
persistence *config.ConfigPersistence,
|
|
monitor *monitoring.Monitor,
|
|
runtime aggregatorRuntimeSources,
|
|
) []alerts.ConnectionSnapshot {
|
|
inputs := buildAggregatorInputsWithRuntimeSources(ctx, cfg, persistence, monitor, runtime)
|
|
return snapshotConnectionsForAlerts(buildConnections(inputs))
|
|
}
|