mirror of
https://github.com/rcourtman/Pulse.git
synced 2026-10-23 20:25:56 +00:00
rcourtman
06b7f5a0b9
SetMockMode(false) switched the mock package off and then cleared active alerts, but every path that evaluates mode-dependent data checked the mode once, up front. A mock alert pass (started from the ticker with go, so passes overlap), a frontend or ingest refresh evaluating a GetState snapshot, a backup-age check over fixture recovery rollups, or a connection check over the mock ledger could already hold fixture data when the switch happened. Its evaluations then landed after the clear, in live mode, where nothing evaluates or resolves fixture resources, so the alerts stayed until the 24-hour stale sweep. A pass-level lock around checkMockAlerts was tried and reverted: it left the other paths open and made the mock-mode API wait behind a whole pass of synchronous event-log writes. The monitor now owns a mock-mode epoch fence. The mock pass, backup-age and snapshot checks, connection checks, Docker pruning and the resource-store refresh take a scope before reading anything mode-dependent and send each alert-manager call through it. SetMockMode, now serialized, flips the mode, ends the epoch, waits only for calls already admitted, and then clears. Unified metrics are evaluated one resource per call, so the wait covers single calls, not passes. Registry rebuilds are tracked rather than waited on. A refresh evaluates the registry it read back only if no rebuild from an ended epoch overlapped it, and the readers that open or remove alerts from the inventory use current state until a clean rebuild in the new epoch replaces the previous mode's registry. The mock supplemental fixture sources now serve nothing outside mock mode: the router swaps them for the real pollers only after the switch, refreshing between sources, so a single switch to live mode used to raise fixture vSphere incidents. The new tests park a real evaluation at a fixed point (a held lock or a blocking supplier), switch modes and release it. Against the unfenced code the mock pass left about 40 fixture alerts, the frontend refresh 5 vSphere incidents and the backup check 137 backup-age alerts in live mode, and a mock-mode connection observation counted toward a live connection's confirmation. The drain, republish and pruning tests fail against a fence without its wait, overlap check or registry tracking. Switching mock mode with several tenant monitors running remains uncovered and is noted in the monitoring contract.
7820 lines
256 KiB
Go
7820 lines
256 KiB
Go
package monitoring
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
stderrors "errors"
|
|
"fmt"
|
|
"math"
|
|
"math/rand"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/ai/memory"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/alerts"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/config"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/discovery"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/logging"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/mock"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/models"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/monitoring/errors"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/notifications"
|
|
recoverymanager "github.com/rcourtman/pulse-go-rewrite/internal/recovery/manager"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/storagehealth"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/system"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
|
|
"github.com/rcourtman/pulse-go-rewrite/internal/websocket"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/diskinventory"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/metrics"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/pbs"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/pmg"
|
|
"github.com/rcourtman/pulse-go-rewrite/pkg/proxmox"
|
|
"github.com/rs/zerolog"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
const (
|
|
defaultTaskTimeout = 90 * time.Second
|
|
minTaskTimeout = 30 * time.Second
|
|
maxTaskTimeout = 3 * time.Minute
|
|
)
|
|
|
|
const mockKeepRealPollingEnv = "PULSE_MOCK_KEEP_REAL_POLLING"
|
|
|
|
func keepRealPollingInMockMode() bool {
|
|
raw := strings.TrimSpace(strings.ToLower(os.Getenv(mockKeepRealPollingEnv)))
|
|
switch raw {
|
|
case "1", "true", "yes", "on":
|
|
return true
|
|
case "", "0", "false", "no", "off":
|
|
return false
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
// alertManagerRestoreOptions keeps mock mode a clean room across process
|
|
// restarts. SetMockMode clears active alerts when the toggle flips, but a
|
|
// process that boots with mock mode already enabled never runs that path, so
|
|
// alerts raised against real infrastructure are restored from disk and served
|
|
// beside fixture data until something clears them.
|
|
func alertManagerRestoreOptions() []alerts.ManagerOption {
|
|
options := []alerts.ManagerOption{alerts.WithDurableAlertStore()}
|
|
if mock.IsMockEnabled() {
|
|
options = append(options, alerts.WithoutPersistedAlertRestore())
|
|
}
|
|
return options
|
|
}
|
|
|
|
// newProxmoxClientFunc is a variable that holds the function to create a new Proxmox client.
|
|
// It is used to allow mocking the client creation in tests.
|
|
var newProxmoxClientFunc = func(cfg proxmox.ClientConfig) (PVEClientInterface, error) {
|
|
return proxmox.NewClient(cfg)
|
|
}
|
|
|
|
// PVEClientInterface defines the interface for PVE clients (both regular and cluster)
|
|
type PVEClientInterface interface {
|
|
GetNodes(ctx context.Context) ([]proxmox.Node, error)
|
|
GetNodeStatus(ctx context.Context, node string) (*proxmox.NodeStatus, error)
|
|
GetNodeRRDData(ctx context.Context, node string, timeframe string, cf string, ds []string) ([]proxmox.NodeRRDPoint, error)
|
|
GetVMs(ctx context.Context, node string) ([]proxmox.VM, error)
|
|
GetContainers(ctx context.Context, node string) ([]proxmox.Container, error)
|
|
GetStorage(ctx context.Context, node string) ([]proxmox.Storage, error)
|
|
GetAllStorage(ctx context.Context) ([]proxmox.Storage, error)
|
|
GetBackupTasks(ctx context.Context) ([]proxmox.Task, error)
|
|
GetTaskLog(ctx context.Context, node, upid string) ([]proxmox.TaskLogLine, error)
|
|
GetReplicationStatus(ctx context.Context) ([]proxmox.ReplicationJob, error)
|
|
GetStorageContent(ctx context.Context, node, storage string) ([]proxmox.StorageContent, error)
|
|
GetVMSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error)
|
|
GetContainerSnapshots(ctx context.Context, node string, vmid int) ([]proxmox.Snapshot, error)
|
|
GetVMStatus(ctx context.Context, node string, vmid int) (*proxmox.VMStatus, error)
|
|
GetContainerStatus(ctx context.Context, node string, vmid int) (*proxmox.Container, error)
|
|
GetContainerConfig(ctx context.Context, node string, vmid int) (map[string]interface{}, error)
|
|
GetContainerInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.ContainerInterface, error)
|
|
GetClusterResources(ctx context.Context, resourceType string) ([]proxmox.ClusterResource, error)
|
|
IsClusterMember(ctx context.Context) (bool, error)
|
|
GetVMFSInfo(ctx context.Context, node string, vmid int) ([]proxmox.VMFileSystem, error)
|
|
GetVMNetworkInterfaces(ctx context.Context, node string, vmid int) ([]proxmox.VMNetworkInterface, error)
|
|
GetVMAgentInfo(ctx context.Context, node string, vmid int) (map[string]interface{}, error)
|
|
GetVMAgentVersion(ctx context.Context, node string, vmid int) (string, error)
|
|
GetZFSPoolStatus(ctx context.Context, node string) ([]proxmox.ZFSPoolStatus, error)
|
|
GetZFSPoolsWithDetails(ctx context.Context, node string) ([]proxmox.ZFSPoolInfo, error)
|
|
GetDisks(ctx context.Context, node string) ([]proxmox.Disk, error)
|
|
GetNodePendingUpdates(ctx context.Context, node string) ([]proxmox.AptPackage, error)
|
|
GetCephStatus(ctx context.Context) (*proxmox.CephStatus, error)
|
|
GetCephDF(ctx context.Context) (*proxmox.CephDF, error)
|
|
}
|
|
|
|
// ResourceStoreInterface provides methods for polling optimization and resource access.
|
|
// When an agent is monitoring a node, we can reduce API polling for that node.
|
|
type ResourceStoreInterface interface {
|
|
// ShouldSkipAPIPolling returns true if API polling should be skipped for the hostname
|
|
// because an agent is providing richer data.
|
|
ShouldSkipAPIPolling(hostname string) bool
|
|
// GetPollingRecommendations returns a map of hostname -> polling multiplier.
|
|
// 0 = skip entirely, 0.5 = half frequency, 1 = normal
|
|
GetPollingRecommendations() map[string]float64
|
|
// GetAll returns all resources in the store (for WebSocket broadcasts)
|
|
GetAll() []unifiedresources.Resource
|
|
// PopulateFromSnapshot updates the store with data from a StateSnapshot
|
|
PopulateFromSnapshot(snapshot models.StateSnapshot)
|
|
}
|
|
|
|
// StaleThresholdResourceStore is implemented by stores that evaluate source
|
|
// freshness while ingesting monitor snapshots.
|
|
type StaleThresholdResourceStore interface {
|
|
SetStaleThresholds(thresholds map[unifiedresources.DataSource]time.Duration)
|
|
}
|
|
|
|
// SupplementalRecordStore is an optional extension for resource stores that can
|
|
// ingest source-native unified records in addition to legacy snapshots.
|
|
type SupplementalRecordStore interface {
|
|
PopulateSupplementalRecords(source unifiedresources.DataSource, records []unifiedresources.IngestRecord)
|
|
}
|
|
|
|
// AtomicSnapshotResourceStore is an optional extension for stores that can
|
|
// atomically replace their canonical registry from a snapshot plus
|
|
// supplemental records in a single swap.
|
|
type AtomicSnapshotResourceStore interface {
|
|
PopulateSnapshotAndSupplemental(snapshot models.StateSnapshot, recordsBySource map[unifiedresources.DataSource][]unifiedresources.IngestRecord)
|
|
}
|
|
|
|
// MetricsTargetResourceStore optionally resolves the history/metrics target for
|
|
// a canonical resource in the live unified store.
|
|
type MetricsTargetResourceStore interface {
|
|
MetricsTargetForResource(resourceID string) *unifiedresources.MetricsTarget
|
|
}
|
|
|
|
// UnifiedResourceFreshnessStore is an optional extension for stores that track
|
|
// their own canonical-resource freshness independent of state.LastUpdate.
|
|
type UnifiedResourceFreshnessStore interface {
|
|
UnifiedResourceFreshness() time.Time
|
|
}
|
|
|
|
// MonitorSupplementalRecordsProvider emits source-native records outside the
|
|
// poll-provider scheduling path (for example, dedicated background pollers).
|
|
type MonitorSupplementalRecordsProvider interface {
|
|
SupplementalRecords(m *Monitor, orgID string) []unifiedresources.IngestRecord
|
|
}
|
|
|
|
// MonitorSupplementalChangesProvider optionally emits canonical resource
|
|
// timeline changes alongside supplemental records.
|
|
type MonitorSupplementalChangesProvider interface {
|
|
SupplementalChanges(m *Monitor, orgID string) []unifiedresources.ResourceChange
|
|
}
|
|
|
|
// MonitorSupplementalInventoryReadinessProvider optionally reports when a
|
|
// supplemental provider's current org-scoped inventory is settled enough to be
|
|
// consumed by monitored-system grouping and support boundaries.
|
|
//
|
|
// Providers that suppress snapshot-owned sources must implement this contract
|
|
// so the monitor can fail closed until the canonical store has been rebuilt
|
|
// from a settled provider baseline.
|
|
type MonitorSupplementalInventoryReadinessProvider interface {
|
|
SupplementalInventoryReadyAt(m *Monitor, orgID string) (time.Time, bool)
|
|
}
|
|
|
|
// MonitorPhysicalDiskTemperatureHistoryProvider optionally exposes source-native
|
|
// physical-disk temperature history through the canonical monitoring chart
|
|
// boundary when Pulse's own stored history is shallow.
|
|
type MonitorPhysicalDiskTemperatureHistoryProvider interface {
|
|
PhysicalDiskTemperatureHistory(m *Monitor, orgID string, duration time.Duration) map[string][]MetricPoint
|
|
}
|
|
|
|
func getNodeDisplayName(instance *config.PVEInstance, nodeName string) string {
|
|
baseName := strings.TrimSpace(nodeName)
|
|
if baseName == "" {
|
|
baseName = "unknown-node"
|
|
}
|
|
|
|
if instance == nil {
|
|
return baseName
|
|
}
|
|
|
|
if instance.IsCluster {
|
|
_, displayName := config.PVEClusterNodePresentation(instance, baseName)
|
|
if displayName != "" {
|
|
return displayName
|
|
}
|
|
return baseName
|
|
}
|
|
|
|
friendly := strings.TrimSpace(instance.Name)
|
|
if friendly != "" {
|
|
return friendly
|
|
}
|
|
|
|
if baseName != "" && baseName != "unknown-node" {
|
|
return baseName
|
|
}
|
|
|
|
if label := normalizeEndpointHost(instance.Host); label != "" && !isLikelyIPAddress(label) {
|
|
return label
|
|
}
|
|
|
|
return baseName
|
|
}
|
|
|
|
func (m *Monitor) getInstanceConfig(instanceName string) *config.PVEInstance {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
if m.config == nil {
|
|
return nil
|
|
}
|
|
|
|
for i := range m.config.PVEInstances {
|
|
if strings.EqualFold(m.config.PVEInstances[i].Name, instanceName) {
|
|
instanceCopy := m.config.PVEInstances[i]
|
|
return &instanceCopy
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (m *Monitor) totalClientCount() int {
|
|
if m == nil {
|
|
return 0
|
|
}
|
|
|
|
total := 0
|
|
for _, provider := range m.pollProviderSnapshotWithBuiltins() {
|
|
if provider == nil {
|
|
continue
|
|
}
|
|
total += len(provider.ListInstances(m))
|
|
}
|
|
return total
|
|
}
|
|
|
|
func (m *Monitor) getPVEClient(name string) (PVEClientInterface, bool) {
|
|
if m == nil {
|
|
return nil, false
|
|
}
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
client, ok := m.pveClients[name]
|
|
return client, ok
|
|
}
|
|
|
|
func (m *Monitor) getPBSClient(name string) (*pbs.Client, bool) {
|
|
if m == nil {
|
|
return nil, false
|
|
}
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
client, ok := m.pbsClients[name]
|
|
return client, ok
|
|
}
|
|
|
|
func (m *Monitor) getPMGClient(name string) (*pmg.Client, bool) {
|
|
if m == nil {
|
|
return nil, false
|
|
}
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
client, ok := m.pmgClients[name]
|
|
return client, ok
|
|
}
|
|
|
|
// normalizeSMARTDeviceIdentifier strips a trailing type annotation (e.g.
|
|
// "sdd [scsi]" -> "sdd") and the /dev/ prefix so SMART temperatures match disks
|
|
// by device path even when smartctl reports a typed label.
|
|
func normalizeSMARTDeviceIdentifier(device string) string {
|
|
normalized := strings.TrimSpace(device)
|
|
if normalized == "" {
|
|
return ""
|
|
}
|
|
|
|
if base, suffix, found := strings.Cut(normalized, " ["); found && strings.HasSuffix(suffix, "]") {
|
|
normalized = base
|
|
}
|
|
|
|
normalized = strings.TrimSpace(normalized)
|
|
return strings.TrimPrefix(normalized, "/dev/")
|
|
}
|
|
|
|
func mergeNVMeTempsIntoDisks(disks []models.PhysicalDisk, nodes []models.Node) []models.PhysicalDisk {
|
|
if len(disks) == 0 || len(nodes) == 0 {
|
|
return disks
|
|
}
|
|
|
|
// Build temperature maps by node for both SMART and legacy NVMe data
|
|
smartTempsByNode := make(map[string][]models.DiskTemp)
|
|
nvmeTempsByNode := make(map[string][]models.NVMeTemp)
|
|
|
|
for _, node := range nodes {
|
|
log.Debug().
|
|
Str("nodeName", node.Name).
|
|
Bool("hasTemp", node.Temperature != nil).
|
|
Bool("tempAvailable", node.Temperature != nil && node.Temperature.Available).
|
|
Int("smartCount", func() int {
|
|
if node.Temperature != nil {
|
|
return len(node.Temperature.SMART)
|
|
}
|
|
return 0
|
|
}()).
|
|
Msg("mergeNVMeTempsIntoDisks: checking node temperature")
|
|
|
|
if node.Temperature == nil || !node.Temperature.Available {
|
|
continue
|
|
}
|
|
|
|
// Collect SMART temps (preferred source)
|
|
if len(node.Temperature.SMART) > 0 {
|
|
temps := make([]models.DiskTemp, len(node.Temperature.SMART))
|
|
copy(temps, node.Temperature.SMART)
|
|
smartTempsByNode[node.Name] = temps
|
|
log.Debug().
|
|
Str("nodeName", node.Name).
|
|
Int("smartTempCount", len(temps)).
|
|
Msg("mergeNVMeTempsIntoDisks: collected SMART temps for node")
|
|
}
|
|
|
|
// Collect legacy NVMe temps as fallback
|
|
if len(node.Temperature.NVMe) > 0 {
|
|
temps := make([]models.NVMeTemp, len(node.Temperature.NVMe))
|
|
copy(temps, node.Temperature.NVMe)
|
|
sort.Slice(temps, func(i, j int) bool {
|
|
return temps[i].Device < temps[j].Device
|
|
})
|
|
nvmeTempsByNode[node.Name] = temps
|
|
}
|
|
}
|
|
|
|
if len(smartTempsByNode) == 0 && len(nvmeTempsByNode) == 0 {
|
|
log.Debug().
|
|
Int("diskCount", len(disks)).
|
|
Msg("mergeNVMeTempsIntoDisks: no SMART or NVMe temperature data available")
|
|
return disks
|
|
}
|
|
|
|
log.Debug().
|
|
Int("smartNodeCount", len(smartTempsByNode)).
|
|
Int("nvmeNodeCount", len(nvmeTempsByNode)).
|
|
Int("diskCount", len(disks)).
|
|
Msg("mergeNVMeTempsIntoDisks: starting disk temperature merge")
|
|
|
|
updated := make([]models.PhysicalDisk, len(disks))
|
|
copy(updated, disks)
|
|
|
|
// Process SMART temperatures first (preferred method)
|
|
for i := range updated {
|
|
smartTemps, ok := smartTempsByNode[updated[i].Node]
|
|
log.Debug().
|
|
Str("diskDevPath", updated[i].DevPath).
|
|
Str("diskNode", updated[i].Node).
|
|
Bool("hasSMARTData", ok).
|
|
Int("smartTempCount", len(smartTemps)).
|
|
Msg("mergeNVMeTempsIntoDisks: checking disk for SMART temp match")
|
|
if !ok || len(smartTemps) == 0 {
|
|
continue
|
|
}
|
|
|
|
// Try to match by WWN (most reliable), but never guess when an
|
|
// identity appears more than once in the node snapshot.
|
|
if diskinventory.IsUsableHardwareID(updated[i].WWN) {
|
|
if temp := uniqueDiskTemperatureMatch(smartTemps, func(candidate models.DiskTemp) bool {
|
|
return diskinventory.IsUsableHardwareID(candidate.WWN) &&
|
|
strings.EqualFold(candidate.WWN, updated[i].WWN)
|
|
}); temp != nil && temp.Temperature > 0 && !temp.StandbySkipped {
|
|
setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart")
|
|
log.Debug().
|
|
Str("disk", updated[i].DevPath).
|
|
Str("wwn", updated[i].WWN).
|
|
Int("temp", temp.Temperature).
|
|
Msg("Matched SMART temperature by WWN")
|
|
}
|
|
}
|
|
|
|
// Fall back to serial number match (case-insensitive)
|
|
if diskinventory.IsUsableHardwareID(updated[i].Serial) && updated[i].Temperature == 0 {
|
|
if temp := uniqueDiskTemperatureMatch(smartTemps, func(candidate models.DiskTemp) bool {
|
|
return diskinventory.IsUsableHardwareID(candidate.Serial) &&
|
|
strings.EqualFold(candidate.Serial, updated[i].Serial)
|
|
}); temp != nil && temp.Temperature > 0 && !temp.StandbySkipped {
|
|
setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart")
|
|
log.Debug().
|
|
Str("disk", updated[i].DevPath).
|
|
Str("serial", updated[i].Serial).
|
|
Int("temp", temp.Temperature).
|
|
Msg("Matched SMART temperature by serial")
|
|
}
|
|
}
|
|
|
|
// Last resort: match by device path (normalized)
|
|
if updated[i].Temperature == 0 {
|
|
normalizedDevPath := normalizeSMARTDeviceIdentifier(updated[i].DevPath)
|
|
if temp := uniqueDiskTemperatureMatch(smartTemps, func(candidate models.DiskTemp) bool {
|
|
normalizedTempDev := normalizeSMARTDeviceIdentifier(candidate.Device)
|
|
return normalizedTempDev != "" && normalizedTempDev == normalizedDevPath
|
|
}); temp != nil && temp.Temperature > 0 && !temp.StandbySkipped {
|
|
setPhysicalDiskTemperature(&updated[i], temp.Temperature, "proxmox_node_smart")
|
|
log.Debug().
|
|
Str("disk", updated[i].DevPath).
|
|
Int("temp", temp.Temperature).
|
|
Msg("Matched SMART temperature by device path")
|
|
}
|
|
}
|
|
}
|
|
|
|
// Process legacy NVMe temperatures for disks that didn't get SMART data
|
|
disksByNode := make(map[string][]int)
|
|
for i := range updated {
|
|
if strings.EqualFold(updated[i].Type, "nvme") && updated[i].Temperature == 0 {
|
|
disksByNode[updated[i].Node] = append(disksByNode[updated[i].Node], i)
|
|
}
|
|
}
|
|
|
|
for nodeName, diskIndexes := range disksByNode {
|
|
temps, ok := nvmeTempsByNode[nodeName]
|
|
if !ok || len(temps) == 0 {
|
|
continue
|
|
}
|
|
|
|
sort.Slice(diskIndexes, func(i, j int) bool {
|
|
return updated[diskIndexes[i]].DevPath < updated[diskIndexes[j]].DevPath
|
|
})
|
|
|
|
for idx, diskIdx := range diskIndexes {
|
|
if idx >= len(temps) {
|
|
break
|
|
}
|
|
|
|
tempVal := temps[idx].Temp
|
|
if tempVal <= 0 || math.IsNaN(tempVal) {
|
|
continue
|
|
}
|
|
|
|
setPhysicalDiskTemperature(&updated[diskIdx], int(math.Round(tempVal)), "proxmox_node_nvme")
|
|
log.Debug().
|
|
Str("disk", updated[diskIdx].DevPath).
|
|
Int("temp", updated[diskIdx].Temperature).
|
|
Msg("Matched legacy NVMe temperature by index")
|
|
}
|
|
}
|
|
|
|
return updated
|
|
}
|
|
|
|
func uniqueDiskTemperatureMatch(temperatures []models.DiskTemp, predicate func(models.DiskTemp) bool) *models.DiskTemp {
|
|
matchIndex := -1
|
|
for index := range temperatures {
|
|
if !predicate(temperatures[index]) {
|
|
continue
|
|
}
|
|
if matchIndex >= 0 {
|
|
return nil
|
|
}
|
|
matchIndex = index
|
|
}
|
|
if matchIndex < 0 {
|
|
return nil
|
|
}
|
|
return &temperatures[matchIndex]
|
|
}
|
|
|
|
func setPhysicalDiskTemperature(disk *models.PhysicalDisk, temperature int, source string) {
|
|
if disk == nil || temperature <= 0 {
|
|
return
|
|
}
|
|
disk.Temperature = temperature
|
|
if disk.Collection == nil {
|
|
disk.Collection = &diskinventory.CollectionStatus{}
|
|
}
|
|
disk.Collection.Temperature = diskinventory.Available(source)
|
|
}
|
|
|
|
// mergeHostAgentSMARTIntoDisks merges SMART temperature data from linked host agents
|
|
// into physical disks for Proxmox nodes. This allows disk temps collected by the
|
|
// pulse-agent running on a PVE node to populate the Physical Disks view.
|
|
func mergeHostAgentSMARTIntoDisks(disks []models.PhysicalDisk, nodes []models.Node, hosts []models.Host) []models.PhysicalDisk {
|
|
if len(disks) == 0 || len(nodes) == 0 || len(hosts) == 0 {
|
|
return disks
|
|
}
|
|
|
|
// Build a map of host ID to host for quick lookup
|
|
hostByID := make(map[string]*models.Host, len(hosts))
|
|
for i := range hosts {
|
|
hostByID[hosts[i].ID] = &hosts[i]
|
|
}
|
|
|
|
// Build a map of node name to linked host's SMART data
|
|
smartByNodeName := make(map[string][]models.HostDiskSMART)
|
|
for _, node := range nodes {
|
|
if node.LinkedAgentID == "" {
|
|
continue
|
|
}
|
|
host, ok := hostByID[node.LinkedAgentID]
|
|
if !ok || len(host.Sensors.SMART) == 0 {
|
|
continue
|
|
}
|
|
smartByNodeName[node.Name] = host.Sensors.SMART
|
|
log.Debug().
|
|
Str("nodeName", node.Name).
|
|
Str("hostAgentID", node.LinkedAgentID).
|
|
Int("smartDiskCount", len(host.Sensors.SMART)).
|
|
Msg("mergeHostAgentSMARTIntoDisks: found linked host agent with SMART data")
|
|
}
|
|
|
|
if len(smartByNodeName) == 0 {
|
|
return disks
|
|
}
|
|
|
|
updated := make([]models.PhysicalDisk, len(disks))
|
|
copy(updated, disks)
|
|
|
|
for i := range updated {
|
|
smartData, ok := smartByNodeName[updated[i].Node]
|
|
if !ok || len(smartData) == 0 {
|
|
continue
|
|
}
|
|
|
|
// Find matching SMART entry by WWN, serial, or topology-scoped device path.
|
|
var matched *models.HostDiskSMART
|
|
uniqueMatch := func(predicate func(models.HostDiskSMART) bool) *models.HostDiskSMART {
|
|
matchIndex := -1
|
|
for index := range smartData {
|
|
if !predicate(smartData[index]) {
|
|
continue
|
|
}
|
|
if matchIndex >= 0 {
|
|
return nil
|
|
}
|
|
matchIndex = index
|
|
}
|
|
if matchIndex < 0 {
|
|
return nil
|
|
}
|
|
return &smartData[matchIndex]
|
|
}
|
|
|
|
// Try to match by WWN (most reliable)
|
|
if diskinventory.IsUsableHardwareID(updated[i].WWN) {
|
|
matched = uniqueMatch(func(candidate models.HostDiskSMART) bool {
|
|
return diskinventory.IsUsableHardwareID(candidate.WWN) &&
|
|
strings.EqualFold(candidate.WWN, updated[i].WWN) &&
|
|
diskTopologyCompatible(updated[i].Controller, updated[i].Target, candidate.Controller, candidate.Target)
|
|
})
|
|
}
|
|
|
|
// Fall back to serial number match
|
|
if matched == nil && diskinventory.IsUsableHardwareID(updated[i].Serial) {
|
|
matched = uniqueMatch(func(candidate models.HostDiskSMART) bool {
|
|
return diskinventory.IsUsableHardwareID(candidate.Serial) &&
|
|
strings.EqualFold(candidate.Serial, updated[i].Serial) &&
|
|
diskTopologyCompatible(updated[i].Controller, updated[i].Target, candidate.Controller, candidate.Target)
|
|
})
|
|
}
|
|
|
|
// Last resort: match by device path
|
|
if matched == nil {
|
|
normalizedDevPath := normalizeSMARTDeviceIdentifier(updated[i].DevPath)
|
|
matched = uniqueMatch(func(candidate models.HostDiskSMART) bool {
|
|
normalizedDiskDev := normalizeSMARTDeviceIdentifier(candidate.Device)
|
|
return normalizedDiskDev != "" &&
|
|
normalizedDiskDev == normalizedDevPath &&
|
|
diskTopologyCompatible(updated[i].Controller, updated[i].Target, candidate.Controller, candidate.Target)
|
|
})
|
|
}
|
|
|
|
if matched == nil {
|
|
continue
|
|
}
|
|
|
|
if strings.TrimSpace(updated[i].Model) == "" && strings.TrimSpace(matched.Model) != "" {
|
|
updated[i].Model = strings.TrimSpace(matched.Model)
|
|
}
|
|
if strings.TrimSpace(matched.Serial) != "" &&
|
|
(strings.TrimSpace(updated[i].Serial) == "" || shouldPromoteHostAgentSerial(updated[i], *matched)) {
|
|
updated[i].Serial = strings.TrimSpace(matched.Serial)
|
|
}
|
|
if strings.TrimSpace(updated[i].WWN) == "" && strings.TrimSpace(matched.WWN) != "" {
|
|
updated[i].WWN = strings.TrimSpace(matched.WWN)
|
|
}
|
|
if shouldPromoteHostAgentDiskType(updated[i].Type, matched.Type) {
|
|
updated[i].Type = strings.TrimSpace(matched.Type)
|
|
}
|
|
if strings.TrimSpace(updated[i].Controller) == "" && strings.TrimSpace(matched.Controller) != "" {
|
|
updated[i].Controller = strings.TrimSpace(matched.Controller)
|
|
}
|
|
if strings.TrimSpace(updated[i].Target) == "" && strings.TrimSpace(matched.Target) != "" {
|
|
updated[i].Target = strings.TrimSpace(matched.Target)
|
|
}
|
|
if updated[i].Size <= 0 && matched.SizeBytes > 0 {
|
|
updated[i].Size = matched.SizeBytes
|
|
}
|
|
if strings.TrimSpace(updated[i].StorageGroup) == "" && strings.TrimSpace(matched.Pool) != "" {
|
|
updated[i].StorageGroup = strings.TrimSpace(matched.Pool)
|
|
}
|
|
|
|
// Merge temperature if not already set
|
|
if !matched.Standby && updated[i].Temperature == 0 && matched.Temperature > 0 {
|
|
updated[i].Temperature = matched.Temperature
|
|
log.Debug().
|
|
Str("device", updated[i].DevPath).
|
|
Int("temp", matched.Temperature).
|
|
Msg("Matched host agent SMART temperature")
|
|
}
|
|
|
|
// Always merge SMART attributes from host agent. The agent's endurance
|
|
// counter is more precise than the Proxmox inventory, so it may correct
|
|
// a coarse value downward or fill an unreported one. It must not,
|
|
// however, make a disk look healthier than the Proxmox inventory already
|
|
// reports: an intermittent agent reading (for example an NVMe log page
|
|
// that briefly reports PercentageUsed 0) used to raise the remaining
|
|
// life, resolve a genuine low-life alert, and let it re-fire on the next
|
|
// poll with a fresh resolved notification (#2112).
|
|
if matched.Attributes != nil {
|
|
updated[i].SmartAttributes = smartAttributesCopy(matched.Attributes)
|
|
if derivedWearout := deriveWearoutFromSMARTAttributes(matched.Attributes); derivedWearout >= 0 {
|
|
if updated[i].Wearout < 0 || derivedWearout < updated[i].Wearout {
|
|
updated[i].Wearout = derivedWearout
|
|
}
|
|
}
|
|
}
|
|
if matched.IO != nil {
|
|
ioCopy := *matched.IO
|
|
updated[i].IO = &ioCopy
|
|
}
|
|
// The agent's later word on a field it supplied (its lease expired, or
|
|
// the disk went into standby) must win over the copy a skipped disk
|
|
// poll carries forward from the previous merge.
|
|
updated[i].Collection = diskinventory.MergeReportedStatus(updated[i].Collection, hostAgentSMARTCollection(*matched))
|
|
|
|
if shouldUseHostAgentPhysicalDiskHealth(updated[i].Health, matched.Health) {
|
|
updated[i].Health = matched.Health
|
|
}
|
|
}
|
|
|
|
return updated
|
|
}
|
|
|
|
// hostAgentSMARTCollection returns the agent's collection state for one SMART
|
|
// row, giving a provenance-less temperature the legacy agent source
|
|
// (diskinventory.LegacyHostAgentStatus) so that history and later lease expiry
|
|
// treat it like any other agent reading.
|
|
func hostAgentSMARTCollection(smart models.HostDiskSMART) *diskinventory.CollectionStatus {
|
|
collection := diskinventory.CloneStatus(smart.Collection)
|
|
if collection == nil {
|
|
collection = &diskinventory.CollectionStatus{}
|
|
}
|
|
collection.Temperature = diskinventory.LegacyHostAgentStatus(
|
|
collection.Temperature,
|
|
smart.Temperature > 0 && !smart.Standby,
|
|
)
|
|
return collection
|
|
}
|
|
|
|
func shouldUseHostAgentPhysicalDiskHealth(existing, incoming string) bool {
|
|
incoming = strings.ToUpper(strings.TrimSpace(incoming))
|
|
if incoming == "" || incoming == "UNKNOWN" {
|
|
return false
|
|
}
|
|
existing = strings.ToUpper(strings.TrimSpace(existing))
|
|
if incoming == "FAILED" {
|
|
return true
|
|
}
|
|
return existing == "" || existing == "UNKNOWN"
|
|
}
|
|
|
|
func diskTopologyCompatible(leftController, leftTarget, rightController, rightTarget string) bool {
|
|
leftController = strings.TrimSpace(leftController)
|
|
leftTarget = strings.TrimSpace(leftTarget)
|
|
rightController = strings.TrimSpace(rightController)
|
|
rightTarget = strings.TrimSpace(rightTarget)
|
|
if leftController != "" && rightController != "" && !strings.EqualFold(leftController, rightController) {
|
|
return false
|
|
}
|
|
if leftTarget != "" && rightTarget != "" && !strings.EqualFold(leftTarget, rightTarget) {
|
|
return false
|
|
}
|
|
return true
|
|
}
|
|
|
|
func shouldPromoteHostAgentSerial(disk models.PhysicalDisk, smart models.HostDiskSMART) bool {
|
|
if smart.Collection != nil &&
|
|
strings.EqualFold(strings.TrimSpace(smart.Type), "sas") &&
|
|
smart.Collection.Serial.State == diskinventory.FieldAvailable &&
|
|
strings.HasPrefix(strings.ToLower(strings.TrimSpace(smart.Collection.Serial.Source)), "smartctl") {
|
|
return true
|
|
}
|
|
// Older agents did not carry collection provenance. Proxmox commonly puts
|
|
// a 64-bit SAS address in its serial field; a non-address smartctl serial is
|
|
// the actual drive identity and must replace that transport address.
|
|
return strings.EqualFold(strings.TrimSpace(disk.Type), "sas") &&
|
|
strings.EqualFold(strings.TrimSpace(smart.Type), "sas") &&
|
|
looksLikeSASAddress(disk.Serial) &&
|
|
!looksLikeSASAddress(smart.Serial)
|
|
}
|
|
|
|
func looksLikeSASAddress(value string) bool {
|
|
value = strings.TrimPrefix(strings.ToLower(strings.TrimSpace(value)), "0x")
|
|
if len(value) != 16 {
|
|
return false
|
|
}
|
|
for _, char := range value {
|
|
if (char < '0' || char > '9') && (char < 'a' || char > 'f') {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func shouldPromoteHostAgentDiskType(existing, incoming string) bool {
|
|
incoming = strings.ToLower(strings.TrimSpace(incoming))
|
|
if incoming == "" {
|
|
return false
|
|
}
|
|
existing = strings.ToLower(strings.TrimSpace(existing))
|
|
if existing == "" || existing == "unknown" || existing == "scsi" {
|
|
return true
|
|
}
|
|
// Proxmox reports coarse form-factor types (hdd/ssd) or misreads SAS
|
|
// members as sata; the agent's smartctl transport evidence is
|
|
// authoritative for the link type.
|
|
if incoming == "sas" && (existing == "sata" || existing == "hdd" || existing == "ssd") {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func deriveWearoutFromSMARTAttributes(attrs *models.SMARTAttributes) int {
|
|
if attrs == nil || attrs.PercentageUsed == nil {
|
|
return -1
|
|
}
|
|
return storagehealth.RemainingLifeFromPercentageUsed(*attrs.PercentageUsed)
|
|
}
|
|
|
|
func physicalDiskFromReadStateView(view *unifiedresources.PhysicalDiskView) models.PhysicalDisk {
|
|
if view == nil {
|
|
// Wearout 0 means no endurance remaining, so an absent view has to
|
|
// carry the unreported sentinel rather than the struct zero value.
|
|
return models.PhysicalDisk{Wearout: unifiedresources.WearoutUnreported}
|
|
}
|
|
|
|
// Preserve the provider key across the canonical read-state round trip.
|
|
// Older/synthetic views without source metadata retain their existing fallback.
|
|
sourceID := view.SourceID()
|
|
if sourceID == "" {
|
|
sourceID = view.ID()
|
|
}
|
|
disk := models.PhysicalDisk{
|
|
ID: sourceID,
|
|
Node: view.Node(),
|
|
Instance: view.Instance(),
|
|
DevPath: view.DevPath(),
|
|
Model: view.Model(),
|
|
Serial: view.Serial(),
|
|
WWN: view.WWN(),
|
|
Type: view.DiskType(),
|
|
Controller: view.Controller(),
|
|
Target: view.Target(),
|
|
Size: view.SizeBytes(),
|
|
Health: view.Health(),
|
|
Wearout: view.Wearout(),
|
|
Temperature: view.Temperature(),
|
|
RPM: view.RPM(),
|
|
Used: view.Used(),
|
|
StorageGroup: view.StorageGroup(),
|
|
SmartAttributes: smartAttributesFromUnifiedMeta(view.SMART()),
|
|
IO: physicalDiskIOFromUnifiedMeta(view.IO()),
|
|
Collection: diskinventory.CloneStatus(view.Collection()),
|
|
LastChecked: view.LastSeen(),
|
|
}
|
|
if source, ok := view.SourceStatus(unifiedresources.SourceProxmox); ok {
|
|
disk.ExpectedUpdateInterval = time.Duration(source.ExpectedUpdateIntervalSeconds) * time.Second
|
|
}
|
|
return disk
|
|
|
|
}
|
|
|
|
func physicalDiskIOFromUnifiedMeta(in *unifiedresources.PhysicalDiskIOMeta) *models.DiskIO {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
return &models.DiskIO{
|
|
Device: in.Device,
|
|
ReadBytes: in.ReadBytes,
|
|
WriteBytes: in.WriteBytes,
|
|
ReadOps: in.ReadOps,
|
|
WriteOps: in.WriteOps,
|
|
ReadTime: in.ReadTimeMs,
|
|
WriteTime: in.WriteTimeMs,
|
|
IOTime: in.IOTimeMs,
|
|
}
|
|
}
|
|
|
|
func smartAttributesFromUnifiedMeta(in *unifiedresources.SMARTMeta) *models.SMARTAttributes {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
|
|
out := &models.SMARTAttributes{
|
|
PowerOnHours: cloneInt64Pointer(in.PowerOnHours),
|
|
PowerCycles: cloneInt64Pointer(in.PowerCycles),
|
|
ReallocatedSectors: cloneInt64Pointer(in.ReallocatedSectors),
|
|
PendingSectors: cloneInt64Pointer(in.PendingSectors),
|
|
OfflineUncorrectable: cloneInt64Pointer(in.OfflineUncorrectable),
|
|
UDMACRCErrors: cloneInt64Pointer(in.UDMACRCErrors),
|
|
PercentageUsed: cloneIntPointer(in.PercentageUsed),
|
|
AvailableSpare: cloneIntPointer(in.AvailableSpare),
|
|
MediaErrors: cloneInt64Pointer(in.MediaErrors),
|
|
UnsafeShutdowns: cloneInt64Pointer(in.UnsafeShutdowns),
|
|
}
|
|
if out.PowerOnHours == nil &&
|
|
out.PowerCycles == nil &&
|
|
out.ReallocatedSectors == nil &&
|
|
out.PendingSectors == nil &&
|
|
out.OfflineUncorrectable == nil &&
|
|
out.UDMACRCErrors == nil &&
|
|
out.PercentageUsed == nil &&
|
|
out.AvailableSpare == nil &&
|
|
out.MediaErrors == nil &&
|
|
out.UnsafeShutdowns == nil {
|
|
return nil
|
|
}
|
|
return out
|
|
}
|
|
|
|
func cloneIntPointer(in *int) *int {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
value := *in
|
|
return &value
|
|
}
|
|
|
|
func cloneInt64Pointer(in *int64) *int64 {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
value := *in
|
|
return &value
|
|
}
|
|
|
|
func physicalDisksForInstanceFromReadState(readState unifiedresources.ReadState, instance string) []models.PhysicalDisk {
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]models.PhysicalDisk, 0)
|
|
for _, disk := range readState.PhysicalDisks() {
|
|
if disk == nil || disk.Instance() != instance {
|
|
continue
|
|
}
|
|
// A linked host Agent's SMART disk inherits the PVE instance for
|
|
// presentation, but that does not make it a PVE inventory record.
|
|
// Feeding it back into State.PhysicalDisks invents a PVE source and
|
|
// alternates tags/history whenever the real disks/list is empty.
|
|
if _, observedByPVE := disk.SourceStatus(unifiedresources.SourceProxmox); !observedByPVE {
|
|
continue
|
|
}
|
|
out = append(out, physicalDiskFromReadStateView(disk))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func nodesForInstanceFromReadState(readState unifiedresources.ReadState, instance string) []models.Node {
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]models.Node, 0)
|
|
for _, node := range readState.Nodes() {
|
|
if node == nil || node.Instance() != instance {
|
|
continue
|
|
}
|
|
out = append(out, nodeFromReadStateView(node))
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostsFromReadState(readState unifiedresources.ReadState) []models.Host {
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
out := make([]models.Host, 0)
|
|
for _, host := range readState.Hosts() {
|
|
if host == nil {
|
|
continue
|
|
}
|
|
out = append(out, hostFromReadStateView(host))
|
|
}
|
|
return out
|
|
}
|
|
|
|
// writeSMARTMetrics writes SMART temperature history to the in-memory chart
|
|
// buffer and persists SMART attributes when the metrics store is enabled.
|
|
//
|
|
// now is the source observation time, not the rebuild wall clock: a read-side
|
|
// registry rebuild that re-issues one unchanged observation must map to a
|
|
// single sample. The store writes go through the unified replay guard so the
|
|
// same observation is not re-committed on every rebuild (#1966).
|
|
func (m *Monitor) writeSMARTMetrics(disk models.PhysicalDisk, now time.Time) {
|
|
if shouldSkipNativeMockStateMetricWrites() {
|
|
return
|
|
}
|
|
|
|
resourceID := unifiedresources.PhysicalDiskMetricID(disk)
|
|
if resourceID == "" {
|
|
return
|
|
}
|
|
|
|
if diskinventory.TemperatureCollected(disk.Temperature, disk.Collection) && m.metricsHistory != nil {
|
|
m.metricsHistory.AddDiskMetric(resourceID, "smart_temp", float64(disk.Temperature), now)
|
|
}
|
|
|
|
writes := m.smartMetricStoreWrites(disk, resourceID, now)
|
|
if len(writes) == 0 {
|
|
return
|
|
}
|
|
writes = m.dedupeUnifiedMetricWrites(writes)
|
|
if len(writes) > 0 {
|
|
m.metricsStore.WriteBatchBounded(writes)
|
|
}
|
|
}
|
|
|
|
// smartMetricStoreWrites builds the persisted SMART writes for one physical
|
|
// disk observation. The caller owns batching so the unified sync can anchor the
|
|
// observation time and drop exact replays (#1966).
|
|
func (m *Monitor) smartMetricStoreWrites(disk models.PhysicalDisk, resourceID string, observedAt time.Time) []metrics.WriteMetric {
|
|
if m.metricsStore == nil {
|
|
return nil
|
|
}
|
|
|
|
writes := make([]metrics.WriteMetric, 0, 12)
|
|
appendWrite := func(metricType string, value float64) {
|
|
writes = append(writes, metrics.WriteMetric{
|
|
ResourceType: "disk",
|
|
ResourceID: resourceID,
|
|
MetricType: metricType,
|
|
Value: value,
|
|
Timestamp: observedAt,
|
|
Tier: metrics.TierRaw,
|
|
})
|
|
}
|
|
|
|
if diskinventory.TemperatureCollected(disk.Temperature, disk.Collection) {
|
|
appendWrite("smart_temp", float64(disk.Temperature))
|
|
}
|
|
|
|
attrs := disk.SmartAttributes
|
|
if attrs == nil {
|
|
return writes
|
|
}
|
|
|
|
// Common
|
|
if attrs.PowerOnHours != nil {
|
|
appendWrite("smart_power_on_hours", float64(*attrs.PowerOnHours))
|
|
}
|
|
if attrs.PowerCycles != nil {
|
|
appendWrite("smart_power_cycles", float64(*attrs.PowerCycles))
|
|
}
|
|
|
|
// SATA-specific
|
|
if attrs.ReallocatedSectors != nil {
|
|
appendWrite("smart_reallocated_sectors", float64(*attrs.ReallocatedSectors))
|
|
}
|
|
if attrs.PendingSectors != nil {
|
|
appendWrite("smart_pending_sectors", float64(*attrs.PendingSectors))
|
|
}
|
|
if attrs.OfflineUncorrectable != nil {
|
|
appendWrite("smart_offline_uncorrectable", float64(*attrs.OfflineUncorrectable))
|
|
}
|
|
if attrs.UDMACRCErrors != nil {
|
|
appendWrite("smart_crc_errors", float64(*attrs.UDMACRCErrors))
|
|
}
|
|
|
|
// NVMe-specific
|
|
if attrs.PercentageUsed != nil {
|
|
appendWrite("smart_percentage_used", float64(*attrs.PercentageUsed))
|
|
}
|
|
if attrs.AvailableSpare != nil {
|
|
appendWrite("smart_available_spare", float64(*attrs.AvailableSpare))
|
|
}
|
|
if attrs.MediaErrors != nil {
|
|
appendWrite("smart_media_errors", float64(*attrs.MediaErrors))
|
|
}
|
|
if attrs.UnsafeShutdowns != nil {
|
|
appendWrite("smart_unsafe_shutdowns", float64(*attrs.UnsafeShutdowns))
|
|
}
|
|
|
|
return writes
|
|
}
|
|
|
|
// PollExecutor defines the contract for executing polling tasks.
|
|
type PollExecutor interface {
|
|
Execute(ctx context.Context, task PollTask)
|
|
}
|
|
|
|
type realExecutor struct {
|
|
monitor *Monitor
|
|
}
|
|
|
|
func newRealExecutor(m *Monitor) PollExecutor {
|
|
return &realExecutor{monitor: m}
|
|
}
|
|
|
|
func (r *realExecutor) Execute(ctx context.Context, task PollTask) {
|
|
if r == nil || r.monitor == nil {
|
|
return
|
|
}
|
|
|
|
if task.Run != nil {
|
|
task.Run(ctx)
|
|
return
|
|
}
|
|
|
|
switch strings.ToLower(task.InstanceType) {
|
|
case "pve":
|
|
if task.PVEClient == nil {
|
|
log.Warn().
|
|
Str("instance", task.InstanceName).
|
|
Msg("PollExecutor received nil PVE client")
|
|
return
|
|
}
|
|
r.monitor.pollPVEInstance(ctx, task.InstanceName, task.PVEClient)
|
|
case "pbs":
|
|
if task.PBSClient == nil {
|
|
log.Warn().
|
|
Str("instance", task.InstanceName).
|
|
Msg("PollExecutor received nil PBS client")
|
|
return
|
|
}
|
|
r.monitor.pollPBSInstance(ctx, task.InstanceName, task.PBSClient)
|
|
case "pmg":
|
|
if task.PMGClient == nil {
|
|
log.Warn().
|
|
Str("instance", task.InstanceName).
|
|
Msg("PollExecutor received nil PMG client")
|
|
return
|
|
}
|
|
r.monitor.pollPMGInstance(ctx, task.InstanceName, task.PMGClient)
|
|
default:
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", task.InstanceType).
|
|
Msg("PollExecutor received unsupported task type")
|
|
}
|
|
}
|
|
}
|
|
|
|
type instanceInfo struct {
|
|
Key string
|
|
Type InstanceType
|
|
DisplayName string
|
|
Connection string
|
|
Metadata map[string]string
|
|
}
|
|
|
|
type pollStatus struct {
|
|
LastSuccess time.Time
|
|
LastErrorAt time.Time
|
|
LastErrorMessage string
|
|
LastErrorCategory string
|
|
ConsecutiveFailures int
|
|
FirstFailureAt time.Time
|
|
}
|
|
|
|
type dlqInsight struct {
|
|
Reason string
|
|
FirstAttempt time.Time
|
|
LastAttempt time.Time
|
|
RetryCount int
|
|
NextRetry time.Time
|
|
}
|
|
|
|
type ErrorDetail struct {
|
|
At time.Time `json:"at"`
|
|
Message string `json:"message"`
|
|
Category string `json:"category"`
|
|
}
|
|
|
|
type InstancePollStatus struct {
|
|
LastSuccess *time.Time `json:"lastSuccess,omitempty"`
|
|
LastError *ErrorDetail `json:"lastError,omitempty"`
|
|
ConsecutiveFailures int `json:"consecutiveFailures"`
|
|
FirstFailureAt *time.Time `json:"firstFailureAt,omitempty"`
|
|
}
|
|
|
|
type InstanceBreaker struct {
|
|
State string `json:"state"`
|
|
Since *time.Time `json:"since,omitempty"`
|
|
LastTransition *time.Time `json:"lastTransition,omitempty"`
|
|
RetryAt *time.Time `json:"retryAt,omitempty"`
|
|
FailureCount int `json:"failureCount"`
|
|
}
|
|
|
|
type InstanceDLQ struct {
|
|
Present bool `json:"present"`
|
|
Reason string `json:"reason,omitempty"`
|
|
FirstAttempt *time.Time `json:"firstAttempt,omitempty"`
|
|
LastAttempt *time.Time `json:"lastAttempt,omitempty"`
|
|
RetryCount int `json:"retryCount,omitempty"`
|
|
NextRetry *time.Time `json:"nextRetry,omitempty"`
|
|
}
|
|
|
|
type InstanceHealth struct {
|
|
Key string `json:"key"`
|
|
Type string `json:"type"`
|
|
DisplayName string `json:"displayName"`
|
|
Instance string `json:"instance"`
|
|
Connection string `json:"connection"`
|
|
PollStatus InstancePollStatus `json:"pollStatus"`
|
|
Breaker InstanceBreaker `json:"breaker"`
|
|
DeadLetter InstanceDLQ `json:"deadLetter"`
|
|
Warnings []string `json:"warnings"`
|
|
}
|
|
|
|
func (h InstanceHealth) NormalizeCollections() InstanceHealth {
|
|
if h.Warnings == nil {
|
|
h.Warnings = []string{}
|
|
}
|
|
return h
|
|
}
|
|
|
|
// Monitor handles all monitoring operations
|
|
type Monitor struct {
|
|
config *config.Config
|
|
state *models.State
|
|
orgID string // Organization ID for tenant isolation (empty = default/legacy)
|
|
mockUnifiedViewMu sync.Mutex
|
|
mockUnifiedView monitorUnifiedStateView
|
|
mockUnifiedViewVersion uint64
|
|
mockUnifiedViewValid bool
|
|
pveClients map[string]PVEClientInterface
|
|
pbsClients map[string]*pbs.Client
|
|
pmgClients map[string]*pmg.Client
|
|
availabilityStatuses map[string]AvailabilityProbeStatus
|
|
availabilityByLocation map[string]map[string]AvailabilityProbeStatus
|
|
availabilityProbeTrackers map[string]availabilityProbeAssignmentTracker
|
|
pollProviders map[InstanceType]PollProvider
|
|
pollMetrics *PollMetrics
|
|
scheduler *AdaptiveScheduler
|
|
stalenessTracker *StalenessTracker
|
|
taskQueue *TaskQueue
|
|
pollTimeout time.Duration
|
|
circuitBreakers map[string]*circuitBreaker
|
|
deadLetterQueue *TaskQueue
|
|
failureCounts map[string]int
|
|
lastOutcome map[string]taskOutcome
|
|
backoffCfg backoffConfig
|
|
rng *rand.Rand
|
|
maxRetryAttempts int
|
|
tempCollector *TemperatureCollector // SSH-based temperature collector
|
|
guestMetadataStore *config.GuestMetadataStore
|
|
dockerMetadataStore *config.DockerMetadataStore
|
|
dockerReportOrderStore *config.DockerReportOrderStore
|
|
hostMetadataStore *config.HostMetadataStore
|
|
hostContinuityStore *config.HostContinuityStore
|
|
hostAgentLifecycleMu sync.RWMutex
|
|
mu sync.RWMutex
|
|
startTime time.Time
|
|
rateTracker *RateTracker
|
|
metricsHistory *MetricsHistory
|
|
metricsStore *metrics.Store // Persistent SQLite metrics storage
|
|
alertManager *alerts.Manager
|
|
alertResolvedAICallback func(*alerts.Alert)
|
|
alertTriggeredAICallback func(*alerts.Alert)
|
|
alertPushCallback func(*alerts.Alert)
|
|
connectionsSnapshotLister func() []alerts.ConnectionSnapshot // returns platform connection snapshots for the connection-degraded check
|
|
incidentStore *memory.IncidentStore
|
|
alertProjectionReplayMu sync.Mutex // serializes lifecycle projection replay passes
|
|
alertProjectionWG sync.WaitGroup // tracks scheduled background catch-up runs
|
|
notificationMgr *notifications.NotificationManager
|
|
deadMan *deadManRuntime
|
|
deadManProgressUnixNano atomic.Int64
|
|
deadManConfigMu sync.RWMutex
|
|
deadManConfig notifications.DeadManConfig
|
|
deadManConfigLoadErr error
|
|
deliveryHealthProjectionMu sync.Mutex // serializes delivery-health reads and alert projection
|
|
lastDeliveryHealthCheck time.Time // throttles the notification-delivery system alert evaluation; guarded by mu
|
|
configPersist *config.ConfigPersistence
|
|
discoveryService *discovery.Service // Background discovery service
|
|
activePollCount int32 // Number of active polling operations
|
|
pollCounter int64 // Counter for polling cycles
|
|
authFailures map[string]int // Track consecutive auth failures per node
|
|
lastAuthAttempt map[string]time.Time // Track last auth attempt time
|
|
lastClusterCheck map[string]time.Time // Track last cluster check for standalone nodes
|
|
lastPhysicalDiskPoll map[string]time.Time // Track last physical disk poll time per instance
|
|
lastPVEBackupPoll map[string]time.Time // Track last PVE backup poll per instance
|
|
vzdumpJobTaskCache map[string]vzdumpJobTaskCacheEntry // Cache synthesized per-guest tasks for multi-guest vzdump job runs, keyed by instance|UPID
|
|
lastPBSBackupPoll map[string]time.Time // Track last PBS backup poll per instance
|
|
pveBackupInventoryReady map[string]map[string]bool // Track PVE guest inventory readiness for backup orphan detection
|
|
pveBackupTemplateSubjects map[string]map[string]struct{} // Track template VMIDs excluded from runtime workloads but valid for backups
|
|
backupPermissionWarnings map[string]string // Track backup permission issues per instance (instance -> warning message)
|
|
backupAlertEvalMu sync.Mutex // Coalesce global rollup reads and order failure/recovery publication.
|
|
persistence *config.ConfigPersistence // Add persistence for saving updated configs
|
|
pbsBackupPollers map[string]bool // Track PBS backup polling goroutines per instance
|
|
pbsBackupCacheTime map[string]map[pbsBackupGroupKey]time.Time // Track when each PBS backup group was last fetched
|
|
runtimePollingMu sync.RWMutex // Guards runtimePollingOverride; polling goroutines read it every cycle
|
|
runtimePollingOverride runtimePollingOverrides // Runtime polling-cadence overrides pushed by the settings API (#1619)
|
|
runtimeCtx context.Context // Context used while monitor is running
|
|
wsHub *websocket.Hub // Hub used for broadcasting state
|
|
diagMu sync.RWMutex // Protects diagnostic snapshot maps
|
|
nodeSnapshots map[string]NodeMemorySnapshot
|
|
guestSnapshots map[string]GuestMemorySnapshot
|
|
rrdCacheMu sync.RWMutex // Protects short-lived guest memory caches.
|
|
nodeRRDMemCache map[string]rrdMemCacheEntry
|
|
vmAgentMemCache map[string]agentMemCacheEntry
|
|
removedDockerHosts map[string]time.Time // Track deliberately removed Docker hosts (ID -> removal time)
|
|
dockerTokenBindings map[string]string // Track token ID -> Docker host identity bindings to enforce uniqueness
|
|
dockerIdentityFlaps map[string]*identityFlapTracker // Track per-host identity flapping (cloned VMs sharing machine-id)
|
|
removedKubernetesClusters map[string]time.Time // Track deliberately removed Kubernetes clusters (ID -> removal time)
|
|
kubernetesTokenBindings map[string]string // Track token ID -> agent ID bindings to enforce uniqueness
|
|
removedHostAgents map[string]time.Time // Track deliberately removed host agents (ID -> removal time)
|
|
hostTokenBindings map[string]string // Track tokenID:hostname -> host identity bindings
|
|
hostIdentityFlaps map[string]*identityFlapTracker // Track per-host-agent identity flapping (cloned machines sharing machine-id)
|
|
hostReportApplyLocksMu sync.Mutex
|
|
hostReportApplyLocks map[string]*hostReportApplyLock
|
|
hostReportOrderMu sync.Mutex
|
|
hostReportOrders map[string]hostReportOrder
|
|
dockerCommands map[string]*dockerHostCommand
|
|
dockerCommandIndex map[string]string
|
|
guestMetadataMu sync.RWMutex
|
|
guestMetadataCache map[string]guestMetadataCacheEntry
|
|
guestMetadataLimiterMu sync.Mutex
|
|
guestMetadataLimiter map[string]time.Time
|
|
guestMetadataSlots chan struct{}
|
|
guestMetadataMinRefresh time.Duration
|
|
guestMetadataRefreshJitter time.Duration
|
|
guestMetadataRetryBackoff time.Duration
|
|
guestMetadataHoldDuration time.Duration
|
|
guestAgentWorkSlots chan struct{}
|
|
// Configurable guest agent timeouts (refs #592)
|
|
guestAgentFSInfoTimeout time.Duration
|
|
guestAgentNetworkTimeout time.Duration
|
|
guestAgentOSInfoTimeout time.Duration
|
|
guestAgentVersionTimeout time.Duration
|
|
guestAgentRetries int
|
|
guestAgentVMBudget time.Duration
|
|
executor PollExecutor
|
|
breakerBaseRetry time.Duration
|
|
breakerMaxDelay time.Duration
|
|
breakerHalfOpenWindow time.Duration
|
|
instanceInfoCache map[string]*instanceInfo
|
|
pollStatusMap map[string]*pollStatus
|
|
dlqInsightMap map[string]*dlqInsight
|
|
nodeLastOnline map[string]time.Time // Track last time each node was seen online (for grace period)
|
|
nodePendingUpdatesCache map[string]pendingUpdatesCache // Cache pending updates per node (checked every 30 min)
|
|
pveMembershipMisses map[string]int // Consecutive authoritative membership absences by instance/node
|
|
resourceStore ResourceStoreInterface // Optional unified resource store for polling optimization
|
|
supplementalProviders map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider
|
|
recoveryManager *recoverymanager.Manager // Optional recovery store manager for backup rollups
|
|
recoveryIngestMu sync.Mutex
|
|
recoveryIngestRunning bool
|
|
recoveryIngestPending []recoveryIngestBatch
|
|
mockMetricsCancel context.CancelFunc
|
|
mockMetricsWg sync.WaitGroup
|
|
mockHostAgentsMu sync.Mutex
|
|
mockHostAgents map[string]models.Host // Fixture agents evaluated by the last mock alert pass
|
|
mockModeFence mockModeFence // Keeps mode-dependent alert evaluations inside the epoch they read in
|
|
dockerChecker DockerChecker // Optional Docker checker for LXC containers
|
|
dockerCheckerConfiguredAt time.Time // Last time the Docker checker was configured
|
|
dockerCheckAllowedVMIDs map[int]struct{} // Optional VMID allowlist gating the LXC Docker socket probe; empty means all guests
|
|
dockerInventoryCollector DockerInventoryCollector // Optional Docker inventory collector for LXC containers
|
|
// Consecutive Docker socket probe failures per container ID, so persistent
|
|
// failures back off instead of re-running pct exec on every poll cycle.
|
|
dockerProbeFailureMu sync.Mutex
|
|
dockerProbeFailures map[string]*dockerProbeFailureState
|
|
// Dispatch time of the Docker probe/inventory command currently owning
|
|
// each guest. A guest with a live claim is never re-probed, so overlapping
|
|
// poll cycles (or cycles whose enrichment context expired while the agent
|
|
// was still running the previous pct exec) cannot stack identical commands
|
|
// on the Proxmox host (minipc probe-storm incident, 2026-08-20).
|
|
dockerProbesInFlight map[string]time.Time
|
|
// Consecutive Docker probe/inventory failures per Proxmox node. Once a
|
|
// node accumulates enough back-to-back failures the circuit breaker
|
|
// suspends all Docker command dispatch to it, because the node itself
|
|
// (not one guest) is the likely problem — e.g. pct exec crawling under
|
|
// NFS flapping.
|
|
dockerNodeProbeFailures map[string]*dockerProbeFailureState
|
|
// Agent profile cache to avoid disk I/O on every report (refs #1094)
|
|
agentProfileCacheMu sync.RWMutex
|
|
agentProfileCache *agentProfileCacheEntry
|
|
// Proxmox LXC filesystem cache: bounded pct df readings reported by an
|
|
// agent securely linked to the node that owns each running container.
|
|
proxmoxLXCFilesystemsMu sync.RWMutex
|
|
proxmoxLXCFilesystemsCache map[string]agentLXCFilesystemCacheEntry
|
|
|
|
// Cluster sensor cache: temperature data collected by an agent on one Proxmox
|
|
// cluster node via SSH to its siblings. Keyed by lowercase node name.
|
|
clusterSensorsMu sync.RWMutex
|
|
clusterSensorsCache map[string]clusterSensorsCacheEntry
|
|
mockChartCacheMu sync.RWMutex
|
|
mockChartMapCache map[mockChartMetricMapCacheKey]map[string][]MetricPoint
|
|
// Commercial feature gate for monitoring-owned Pro behaviour (external
|
|
// availability probes). Guarded by its own mutex so read-time derivations
|
|
// that already hold m.mu can consult it without lock re-entrancy.
|
|
licenseCheckerMu sync.RWMutex
|
|
licenseChecker func(feature string) bool
|
|
|
|
// Unified metric sync replay guard. The sync runs on both ingest
|
|
// boundaries and read-side registry rebuilds, so one observation is often
|
|
// replayed many times between polls. Without this every replay re-writes
|
|
// the same metrics row (an UPDATE in SQLite) and churns the WAL for no new
|
|
// data, which is the dominant write amplification in #1966.
|
|
unifiedMetricSyncMu sync.Mutex
|
|
unifiedMetricSyncLast map[unifiedMetricSampleKey]unifiedMetricSample
|
|
|
|
// agentReportRefreshWindow folds accepted agent reports into at most one
|
|
// canonical store refresh per window; zero refreshes on every report.
|
|
agentReportRefreshWindow time.Duration
|
|
agentReportRefresh agentReportRefreshState
|
|
}
|
|
|
|
func (m *Monitor) setRuntimeContext(ctx context.Context, hub *websocket.Hub) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.runtimeCtx = ctx
|
|
m.wsHub = hub
|
|
}
|
|
|
|
func (m *Monitor) getRuntimeContext() context.Context {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
return m.runtimeCtx
|
|
}
|
|
|
|
// clusterSensorsCacheEntry stores temperature data collected by a sibling agent via SSH.
|
|
type clusterSensorsCacheEntry struct {
|
|
reporterID string // host agent that collected the reading over SSH
|
|
nodeName string // lowercase Proxmox node name the reading describes
|
|
sensors models.HostSensorSummary
|
|
updatedAt time.Time
|
|
}
|
|
|
|
type rrdMemCacheEntry struct {
|
|
available uint64
|
|
used uint64
|
|
total uint64
|
|
hasAvail bool
|
|
hasUsed bool
|
|
hasTotal bool
|
|
netIn float64
|
|
netOut float64
|
|
hasNetIn bool
|
|
hasNetOut bool
|
|
fetchedAt time.Time
|
|
}
|
|
|
|
// pendingUpdatesCache caches apt pending updates count per node
|
|
type pendingUpdatesCache struct {
|
|
count int
|
|
checkedAt time.Time
|
|
}
|
|
|
|
// TTL for pending updates cache (30 minutes - balance between freshness and API load)
|
|
const pendingUpdatesCacheTTL = 30 * time.Minute
|
|
|
|
// agentProfileCacheEntry caches agent profiles and assignments to avoid disk I/O on every agent report.
|
|
// TTL is 60 seconds to balance freshness with performance.
|
|
type agentProfileCacheEntry struct {
|
|
profiles []models.AgentProfile
|
|
assignments []models.AgentProfileAssignment
|
|
loadedAt time.Time
|
|
}
|
|
|
|
const agentProfileCacheTTL = 60 * time.Second
|
|
|
|
// shouldRunBackupPoll determines whether a backup polling cycle should execute.
|
|
// Returns whether polling should run, a human-readable skip reason, and the timestamp to record.
|
|
func (m *Monitor) shouldRunBackupPoll(last time.Time, now time.Time) (bool, string, time.Time) {
|
|
if m == nil || m.config == nil {
|
|
return false, "configuration unavailable", last
|
|
}
|
|
|
|
if !m.backupPollingEnabledSetting() {
|
|
return false, "backup polling globally disabled", last
|
|
}
|
|
|
|
interval := m.backupPollingIntervalSetting()
|
|
if interval > 0 {
|
|
if !last.IsZero() && now.Sub(last) < interval {
|
|
next := last.Add(interval)
|
|
return false, fmt.Sprintf("next run scheduled for %s", next.Format(time.RFC3339)), last
|
|
}
|
|
return true, "", now
|
|
}
|
|
|
|
backupCycles := m.config.BackupPollingCycles
|
|
if backupCycles <= 0 {
|
|
backupCycles = 10
|
|
}
|
|
|
|
if m.pollCounter%int64(backupCycles) == 0 || m.pollCounter == 1 {
|
|
return true, "", now
|
|
}
|
|
|
|
remaining := int64(backupCycles) - (m.pollCounter % int64(backupCycles))
|
|
return false, fmt.Sprintf("next run in %d polling cycles", remaining), last
|
|
}
|
|
|
|
const (
|
|
dockerConnectionPrefix = "docker-"
|
|
kubernetesConnectionPrefix = "kubernetes-"
|
|
hostConnectionPrefix = "host-"
|
|
dockerOfflineGraceMultiplier = 4
|
|
dockerMinimumHealthWindow = 30 * time.Second
|
|
dockerMaximumHealthWindow = 10 * time.Minute
|
|
kubernetesOfflineGraceMultiplier = 4
|
|
kubernetesMinimumHealthWindow = 30 * time.Second
|
|
kubernetesMaximumHealthWindow = 10 * time.Minute
|
|
hostOfflineGraceMultiplier = 6
|
|
hostMinimumHealthWindow = 60 * time.Second
|
|
hostMaximumHealthWindow = 10 * time.Minute
|
|
nodeOfflineGracePeriod = 60 * time.Second // Grace period before marking Proxmox nodes offline
|
|
nodeRRDCacheTTL = 30 * time.Second
|
|
nodeRRDRequestTimeout = 2 * time.Second
|
|
)
|
|
|
|
type taskOutcome struct {
|
|
success bool
|
|
transient bool
|
|
err error
|
|
recordedAt time.Time
|
|
}
|
|
|
|
func nodeRRDCacheKey(instanceName, nodeName string) string {
|
|
return instanceName + "/" + nodeName
|
|
}
|
|
|
|
func (m *Monitor) getNodeRRDMetrics(ctx context.Context, client PVEClientInterface, instanceName, nodeName string) (rrdMemCacheEntry, error) {
|
|
if client == nil || nodeName == "" {
|
|
return rrdMemCacheEntry{}, fmt.Errorf("invalid arguments for RRD lookup")
|
|
}
|
|
|
|
now := time.Now()
|
|
cacheKey := nodeRRDCacheKey(instanceName, nodeName)
|
|
|
|
m.rrdCacheMu.RLock()
|
|
if entry, ok := m.nodeRRDMemCache[cacheKey]; ok && now.Sub(entry.fetchedAt) < nodeRRDCacheTTL {
|
|
m.rrdCacheMu.RUnlock()
|
|
return entry, nil
|
|
}
|
|
m.rrdCacheMu.RUnlock()
|
|
|
|
requestCtx, cancel := context.WithTimeout(ctx, nodeRRDRequestTimeout)
|
|
defer cancel()
|
|
|
|
points, err := client.GetNodeRRDData(requestCtx, nodeName, "hour", "AVERAGE", []string{"memavailable", "memused", "memtotal", "netin", "netout"})
|
|
if err != nil {
|
|
return rrdMemCacheEntry{}, err
|
|
}
|
|
|
|
var memAvailable uint64
|
|
var memUsed uint64
|
|
var memTotal uint64
|
|
var hasAvail bool
|
|
var hasUsed bool
|
|
var hasTotal bool
|
|
var netIn float64
|
|
var netOut float64
|
|
var hasNetIn bool
|
|
var hasNetOut bool
|
|
|
|
for i := len(points) - 1; i >= 0; i-- {
|
|
point := points[i]
|
|
|
|
if !hasTotal && point.MemTotal != nil && !math.IsNaN(*point.MemTotal) && !math.IsInf(*point.MemTotal, 0) && *point.MemTotal > 0 && *point.MemTotal <= math.MaxUint64 {
|
|
memTotal = uint64(math.Round(*point.MemTotal))
|
|
hasTotal = true
|
|
}
|
|
|
|
if !hasAvail && point.MemAvailable != nil && !math.IsNaN(*point.MemAvailable) && !math.IsInf(*point.MemAvailable, 0) && *point.MemAvailable >= 0 && *point.MemAvailable <= math.MaxUint64 {
|
|
memAvailable = uint64(math.Round(*point.MemAvailable))
|
|
hasAvail = true
|
|
}
|
|
|
|
if !hasUsed && point.MemUsed != nil && !math.IsNaN(*point.MemUsed) && !math.IsInf(*point.MemUsed, 0) && *point.MemUsed >= 0 && *point.MemUsed <= math.MaxUint64 {
|
|
memUsed = uint64(math.Round(*point.MemUsed))
|
|
hasUsed = true
|
|
}
|
|
|
|
if !hasNetIn && point.NetIn != nil && !math.IsNaN(*point.NetIn) && !math.IsInf(*point.NetIn, 0) {
|
|
netIn = *point.NetIn
|
|
hasNetIn = true
|
|
}
|
|
if !hasNetOut && point.NetOut != nil && !math.IsNaN(*point.NetOut) && !math.IsInf(*point.NetOut, 0) {
|
|
netOut = *point.NetOut
|
|
hasNetOut = true
|
|
}
|
|
}
|
|
|
|
if memTotal > 0 {
|
|
if memAvailable > memTotal {
|
|
memAvailable = 0
|
|
hasAvail = false
|
|
}
|
|
if memUsed > memTotal {
|
|
memUsed = 0
|
|
hasUsed = false
|
|
}
|
|
}
|
|
|
|
if !hasAvail && !hasUsed && !hasNetIn && !hasNetOut {
|
|
return rrdMemCacheEntry{}, fmt.Errorf("rrd node metrics not present")
|
|
}
|
|
|
|
entry := rrdMemCacheEntry{
|
|
available: memAvailable,
|
|
used: memUsed,
|
|
total: memTotal,
|
|
hasAvail: hasAvail,
|
|
hasUsed: hasUsed,
|
|
hasTotal: hasTotal,
|
|
netIn: netIn,
|
|
netOut: netOut,
|
|
hasNetIn: hasNetIn,
|
|
hasNetOut: hasNetOut,
|
|
fetchedAt: now,
|
|
}
|
|
|
|
m.rrdCacheMu.Lock()
|
|
if m.nodeRRDMemCache == nil {
|
|
m.nodeRRDMemCache = make(map[string]rrdMemCacheEntry)
|
|
}
|
|
m.nodeRRDMemCache[cacheKey] = entry
|
|
m.rrdCacheMu.Unlock()
|
|
|
|
return entry, nil
|
|
}
|
|
|
|
// RemoveDockerHost removes a docker host from the shared state and clears related alerts.
|
|
func (m *Monitor) GetConnectionStatuses() map[string]bool {
|
|
if m == nil {
|
|
return map[string]bool{}
|
|
}
|
|
|
|
if mock.IsMockEnabled() {
|
|
statuses := make(map[string]bool)
|
|
state := mock.CurrentFixtureGraph().State
|
|
for _, node := range state.Nodes {
|
|
key := "pve-" + node.Name
|
|
statuses[key] = strings.ToLower(node.Status) == "online"
|
|
if node.Host != "" {
|
|
statuses[node.Host] = strings.ToLower(node.Status) == "online"
|
|
}
|
|
}
|
|
for _, pbsInst := range state.PBSInstances {
|
|
key := "pbs-" + pbsInst.Name
|
|
statuses[key] = strings.ToLower(pbsInst.Status) != "offline"
|
|
if pbsInst.Host != "" {
|
|
statuses[pbsInst.Host] = strings.ToLower(pbsInst.Status) != "offline"
|
|
}
|
|
}
|
|
for _, pmgInst := range state.PMGInstances {
|
|
key := "pmg-" + pmgInst.Name
|
|
statuses[key] = strings.ToLower(pmgInst.Status) != "offline"
|
|
if pmgInst.Host != "" {
|
|
statuses[pmgInst.Host] = strings.ToLower(pmgInst.Status) != "offline"
|
|
}
|
|
}
|
|
|
|
for _, dockerHost := range state.DockerHosts {
|
|
key := dockerConnectionPrefix + dockerHost.ID
|
|
statuses[key] = strings.ToLower(dockerHost.Status) == "online"
|
|
}
|
|
return statuses
|
|
}
|
|
|
|
statuses := make(map[string]bool)
|
|
for _, provider := range m.pollProviderSnapshotWithBuiltins() {
|
|
for key, connected := range m.providerConnectionStatuses(provider) {
|
|
if strings.TrimSpace(key) == "" {
|
|
continue
|
|
}
|
|
statuses[key] = connected
|
|
}
|
|
}
|
|
return statuses
|
|
}
|
|
|
|
// checkContainerizedTempMonitoring logs a security warning if Pulse is running
|
|
// in a container with SSH-based temperature monitoring enabled
|
|
func checkContainerizedTempMonitoring() {
|
|
// Check if running in container
|
|
isContainer := os.Getenv("PULSE_DOCKER") == "true" || system.InContainer()
|
|
if !isContainer {
|
|
return
|
|
}
|
|
|
|
// Check if SSH keys exist (indicates temperature monitoring is configured)
|
|
homeDir := os.Getenv("HOME")
|
|
if homeDir == "" {
|
|
homeDir = "/home/pulse"
|
|
}
|
|
sshKeyPath := homeDir + "/.ssh/id_ed25519"
|
|
if _, err := os.Stat(sshKeyPath); err != nil {
|
|
// No SSH key found, temperature monitoring not configured
|
|
return
|
|
}
|
|
|
|
// Log warning
|
|
log.Warn().
|
|
Msg("SECURITY NOTICE: Pulse is running in a container with SSH-based temperature monitoring enabled. " +
|
|
"SSH private keys are stored inside the container, which could be a security risk if the container is compromised. " +
|
|
"Future versions will use agent-based architecture for better security. " +
|
|
"See documentation for hardening recommendations.")
|
|
}
|
|
|
|
// New creates a new Monitor instance
|
|
func New(cfg *config.Config) (*Monitor, error) {
|
|
if cfg == nil {
|
|
return nil, fmt.Errorf("config cannot be nil")
|
|
}
|
|
|
|
hostContinuityStore := config.NewHostContinuityStore(cfg.DataPath, nil)
|
|
if err := hostContinuityStore.LoadError(); err != nil {
|
|
return nil, fmt.Errorf("load host continuity lifecycle journal: %w", err)
|
|
}
|
|
dockerReportOrderStore := config.NewDockerReportOrderStore(cfg.DataPath, nil)
|
|
if err := dockerReportOrderStore.LoadError(); err != nil {
|
|
return nil, fmt.Errorf("load Docker report order journal: %w", err)
|
|
}
|
|
|
|
// Initialize temperature collector with sensors SSH key
|
|
// Will use root user for now - can be made configurable later
|
|
homeDir := os.Getenv("HOME")
|
|
if homeDir == "" {
|
|
homeDir = "/home/pulse"
|
|
}
|
|
sshKeyPath := filepath.Join(homeDir, ".ssh/id_ed25519_sensors")
|
|
tempCollector := NewTemperatureCollectorWithPort("root", sshKeyPath, cfg.SSHPort)
|
|
|
|
// Security warning if running in container with SSH temperature monitoring
|
|
checkContainerizedTempMonitoring()
|
|
|
|
stalenessTracker := NewStalenessTracker(getPollMetrics())
|
|
stalenessTracker.SetBounds(cfg.AdaptivePollingBaseInterval, cfg.AdaptivePollingMaxInterval)
|
|
taskQueue := NewTaskQueue()
|
|
deadLetterQueue := NewTaskQueue()
|
|
breakers := make(map[string]*circuitBreaker)
|
|
failureCounts := make(map[string]int)
|
|
lastOutcome := make(map[string]taskOutcome)
|
|
backoff := backoffConfig{
|
|
Initial: 5 * time.Second,
|
|
Multiplier: 2,
|
|
Jitter: 0.2,
|
|
Max: 5 * time.Minute,
|
|
}
|
|
|
|
if cfg.AdaptivePollingEnabled && cfg.AdaptivePollingMaxInterval > 0 && cfg.AdaptivePollingMaxInterval <= 15*time.Second {
|
|
backoff.Initial = 750 * time.Millisecond
|
|
backoff.Max = 6 * time.Second
|
|
}
|
|
|
|
var scheduler *AdaptiveScheduler
|
|
if cfg.AdaptivePollingEnabled {
|
|
scheduler = NewAdaptiveScheduler(SchedulerConfig{
|
|
BaseInterval: cfg.AdaptivePollingBaseInterval,
|
|
MinInterval: cfg.AdaptivePollingMinInterval,
|
|
MaxInterval: cfg.AdaptivePollingMaxInterval,
|
|
}, stalenessTracker, nil, nil)
|
|
}
|
|
|
|
minRefresh := cfg.GuestMetadataMinRefreshInterval
|
|
if minRefresh <= 0 {
|
|
minRefresh = config.DefaultGuestMetadataMinRefresh
|
|
}
|
|
jitter := cfg.GuestMetadataRefreshJitter
|
|
if jitter < 0 {
|
|
jitter = 0
|
|
}
|
|
retryBackoff := cfg.GuestMetadataRetryBackoff
|
|
if retryBackoff <= 0 {
|
|
retryBackoff = config.DefaultGuestMetadataRetryBackoff
|
|
}
|
|
concurrency := cfg.GuestMetadataMaxConcurrent
|
|
if concurrency <= 0 {
|
|
concurrency = config.DefaultGuestMetadataMaxConcurrent
|
|
}
|
|
holdDuration := defaultGuestMetadataHold
|
|
|
|
// Load guest agent timeout configuration from environment variables (refs #592)
|
|
guestAgentFSInfoTimeout := parsePositiveDurationEnv("GUEST_AGENT_FSINFO_TIMEOUT", defaultGuestAgentFSInfoTimeout)
|
|
guestAgentNetworkTimeout := parsePositiveDurationEnv("GUEST_AGENT_NETWORK_TIMEOUT", defaultGuestAgentNetworkTimeout)
|
|
guestAgentOSInfoTimeout := parsePositiveDurationEnv("GUEST_AGENT_OSINFO_TIMEOUT", defaultGuestAgentOSInfoTimeout)
|
|
guestAgentVersionTimeout := parsePositiveDurationEnv("GUEST_AGENT_VERSION_TIMEOUT", defaultGuestAgentVersionTimeout)
|
|
guestAgentRetries := parseNonNegativeIntEnv("GUEST_AGENT_RETRIES", defaultGuestAgentRetries)
|
|
guestAgentVMBudget := parseDurationEnv("GUEST_AGENT_VM_BUDGET", 0)
|
|
guestAgentVMMaxConcurrent := parseNonNegativeIntEnv("GUEST_AGENT_VM_MAX_CONCURRENT", defaultGuestAgentVMMaxConcurrent)
|
|
|
|
// Initialize persistent metrics store (SQLite) with configurable retention
|
|
var metricsStore *metrics.Store
|
|
metricsStoreConfig := metrics.DefaultConfig(cfg.DataPath)
|
|
if strings.TrimSpace(cfg.MetricsDBPath) != "" {
|
|
metricsStoreConfig.DBPath = cfg.MetricsDBPath
|
|
}
|
|
if cfg.MetricsRollupInterval > 0 {
|
|
metricsStoreConfig.RollupInterval = cfg.MetricsRollupInterval
|
|
}
|
|
// Override retention settings from config (allows tier-based pricing in future)
|
|
if cfg.MetricsRetentionRawHours > 0 {
|
|
metricsStoreConfig.RetentionRaw = time.Duration(cfg.MetricsRetentionRawHours) * time.Hour
|
|
}
|
|
if cfg.MetricsRetentionMinuteHours > 0 {
|
|
metricsStoreConfig.RetentionMinute = time.Duration(cfg.MetricsRetentionMinuteHours) * time.Hour
|
|
}
|
|
if cfg.MetricsRetentionHourlyDays > 0 {
|
|
metricsStoreConfig.RetentionHourly = time.Duration(cfg.MetricsRetentionHourlyDays) * 24 * time.Hour
|
|
}
|
|
if cfg.MetricsRetentionDailyDays > 0 {
|
|
metricsStoreConfig.RetentionDaily = time.Duration(cfg.MetricsRetentionDailyDays) * 24 * time.Hour
|
|
}
|
|
|
|
// In mock mode, extend hourly/daily retention to 90 days so seeded mock
|
|
// backfill (daily tier 30d, hourly tier 7d — see mockStoreSeedPlans)
|
|
// outlives its windows. Raw and minute tiers keep production defaults —
|
|
// the minute seed window (24h) matches the default retention, and short
|
|
// raw retention keeps live mock ticks from bloating the DB (the old
|
|
// 90-day raw retention caused metrics.db to grow to ~2 GB).
|
|
if mock.IsMockEnabled() {
|
|
metricsStoreConfig.WriteBufferSize = 2000
|
|
metricsStoreConfig.RetentionHourly = 90 * 24 * time.Hour
|
|
metricsStoreConfig.RetentionDaily = 90 * 24 * time.Hour
|
|
}
|
|
ms, err := metrics.NewStore(metricsStoreConfig)
|
|
if err != nil {
|
|
// Do not automatically delete the DB on error, as it causes data loss on transient errors (e.g. locks).
|
|
// If the DB is truly corrupted, the user should manually remove it.
|
|
log.Error().Err(err).Msg("failed to initialize persistent metrics store - continuing without metrics persistence")
|
|
} else {
|
|
if mock.IsMockEnabled() {
|
|
ms.SetMaxOpenConns(10)
|
|
}
|
|
metricsStore = ms
|
|
log.Info().
|
|
Str("path", metricsStoreConfig.DBPath).
|
|
Dur("retentionRaw", metricsStoreConfig.RetentionRaw).
|
|
Dur("retentionMinute", metricsStoreConfig.RetentionMinute).
|
|
Dur("retentionHourly", metricsStoreConfig.RetentionHourly).
|
|
Dur("retentionDaily", metricsStoreConfig.RetentionDaily).
|
|
Msg("Persistent metrics store initialized with configurable retention")
|
|
}
|
|
|
|
incidentStore := memory.NewIncidentStore(memory.IncidentStoreConfig{
|
|
DataDir: cfg.DataPath,
|
|
})
|
|
|
|
m := &Monitor{
|
|
config: cfg,
|
|
state: models.NewState(),
|
|
agentReportRefreshWindow: agentReportRefreshInterval,
|
|
pveClients: make(map[string]PVEClientInterface),
|
|
pbsClients: make(map[string]*pbs.Client),
|
|
pmgClients: make(map[string]*pmg.Client),
|
|
availabilityStatuses: make(map[string]AvailabilityProbeStatus),
|
|
availabilityByLocation: make(map[string]map[string]AvailabilityProbeStatus),
|
|
availabilityProbeTrackers: make(map[string]availabilityProbeAssignmentTracker),
|
|
pollProviders: make(map[InstanceType]PollProvider),
|
|
pollMetrics: getPollMetrics(),
|
|
scheduler: scheduler,
|
|
stalenessTracker: stalenessTracker,
|
|
taskQueue: taskQueue,
|
|
pollTimeout: derivePollTimeout(cfg),
|
|
deadLetterQueue: deadLetterQueue,
|
|
circuitBreakers: breakers,
|
|
failureCounts: failureCounts,
|
|
lastOutcome: lastOutcome,
|
|
backoffCfg: backoff,
|
|
rng: rand.New(rand.NewSource(time.Now().UnixNano())),
|
|
maxRetryAttempts: 5,
|
|
tempCollector: tempCollector,
|
|
guestMetadataStore: config.NewGuestMetadataStore(cfg.DataPath, nil),
|
|
dockerMetadataStore: config.NewDockerMetadataStore(cfg.DataPath, nil),
|
|
dockerReportOrderStore: dockerReportOrderStore,
|
|
hostMetadataStore: config.NewHostMetadataStore(cfg.DataPath, nil),
|
|
hostContinuityStore: hostContinuityStore,
|
|
startTime: time.Now(),
|
|
rateTracker: NewRateTracker(),
|
|
metricsHistory: NewMetricsHistory(1000, 24*time.Hour), // Keep up to 1000 points (~8h @ 30s)
|
|
metricsStore: metricsStore, // Persistent SQLite storage
|
|
alertManager: alerts.NewManagerWithDataDir(cfg.DataPath, alertManagerRestoreOptions()...),
|
|
incidentStore: incidentStore,
|
|
notificationMgr: notifications.NewNotificationManagerWithDeferredQueue(cfg.PublicURL, cfg.DataPath),
|
|
deadMan: newDeadManRuntime(config.ResolveRuntimeDataDir(cfg.DataPath)),
|
|
configPersist: config.NewConfigPersistence(cfg.DataPath),
|
|
discoveryService: nil, // Will be initialized in Start()
|
|
authFailures: make(map[string]int),
|
|
lastAuthAttempt: make(map[string]time.Time),
|
|
lastClusterCheck: make(map[string]time.Time),
|
|
lastPhysicalDiskPoll: make(map[string]time.Time),
|
|
lastPVEBackupPoll: make(map[string]time.Time),
|
|
vzdumpJobTaskCache: make(map[string]vzdumpJobTaskCacheEntry),
|
|
lastPBSBackupPoll: make(map[string]time.Time),
|
|
pveBackupInventoryReady: make(map[string]map[string]bool),
|
|
pveBackupTemplateSubjects: make(map[string]map[string]struct{}),
|
|
backupPermissionWarnings: make(map[string]string),
|
|
persistence: config.NewConfigPersistence(cfg.DataPath),
|
|
pbsBackupPollers: make(map[string]bool),
|
|
pbsBackupCacheTime: make(map[string]map[pbsBackupGroupKey]time.Time),
|
|
nodeSnapshots: make(map[string]NodeMemorySnapshot),
|
|
guestSnapshots: make(map[string]GuestMemorySnapshot),
|
|
nodeRRDMemCache: make(map[string]rrdMemCacheEntry),
|
|
vmAgentMemCache: make(map[string]agentMemCacheEntry),
|
|
removedDockerHosts: make(map[string]time.Time),
|
|
dockerTokenBindings: make(map[string]string),
|
|
dockerIdentityFlaps: make(map[string]*identityFlapTracker),
|
|
removedKubernetesClusters: make(map[string]time.Time),
|
|
kubernetesTokenBindings: make(map[string]string),
|
|
removedHostAgents: make(map[string]time.Time),
|
|
hostTokenBindings: make(map[string]string),
|
|
hostIdentityFlaps: make(map[string]*identityFlapTracker),
|
|
hostReportApplyLocks: make(map[string]*hostReportApplyLock),
|
|
hostReportOrders: make(map[string]hostReportOrder),
|
|
clusterSensorsCache: make(map[string]clusterSensorsCacheEntry),
|
|
proxmoxLXCFilesystemsCache: make(map[string]agentLXCFilesystemCacheEntry),
|
|
dockerCommands: make(map[string]*dockerHostCommand),
|
|
dockerCommandIndex: make(map[string]string),
|
|
guestMetadataCache: make(map[string]guestMetadataCacheEntry),
|
|
guestMetadataLimiter: make(map[string]time.Time),
|
|
guestMetadataMinRefresh: minRefresh,
|
|
guestMetadataRefreshJitter: jitter,
|
|
guestMetadataRetryBackoff: retryBackoff,
|
|
guestMetadataHoldDuration: holdDuration,
|
|
guestAgentFSInfoTimeout: guestAgentFSInfoTimeout,
|
|
guestAgentNetworkTimeout: guestAgentNetworkTimeout,
|
|
guestAgentOSInfoTimeout: guestAgentOSInfoTimeout,
|
|
guestAgentVersionTimeout: guestAgentVersionTimeout,
|
|
guestAgentRetries: guestAgentRetries,
|
|
guestAgentVMBudget: guestAgentVMBudget,
|
|
instanceInfoCache: make(map[string]*instanceInfo),
|
|
pollStatusMap: make(map[string]*pollStatus),
|
|
dlqInsightMap: make(map[string]*dlqInsight),
|
|
nodeLastOnline: make(map[string]time.Time),
|
|
nodePendingUpdatesCache: make(map[string]pendingUpdatesCache),
|
|
pveMembershipMisses: make(map[string]int),
|
|
supplementalProviders: make(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider),
|
|
}
|
|
|
|
m.breakerBaseRetry = 5 * time.Second
|
|
m.breakerMaxDelay = 5 * time.Minute
|
|
m.breakerHalfOpenWindow = 30 * time.Second
|
|
|
|
if cfg.AdaptivePollingEnabled && cfg.AdaptivePollingMaxInterval > 0 && cfg.AdaptivePollingMaxInterval <= 15*time.Second {
|
|
m.breakerBaseRetry = 2 * time.Second
|
|
m.breakerMaxDelay = 10 * time.Second
|
|
m.breakerHalfOpenWindow = 2 * time.Second
|
|
}
|
|
|
|
m.executor = newRealExecutor(m)
|
|
if queue := m.notificationMgr.GetQueue(); queue != nil {
|
|
queue.SetDeliveryHealthChangedCallback(m.reconcileNotificationDelivery)
|
|
}
|
|
m.hydrateRemovedHostAgents(time.Now().UTC())
|
|
m.alertManager.SetBackupIntentContextResolver(m.resolveBackupIntentContext)
|
|
m.alertManager.SetMetricWindowProvider(m.metricWindowPoints)
|
|
m.alertManager.EnableShadowFeed()
|
|
m.registerBuiltInPollProviders()
|
|
m.buildInstanceInfoCache(cfg)
|
|
|
|
// Initialize state with config values
|
|
m.state.TemperatureMonitoringEnabled = cfg.TemperatureMonitoringEnabled
|
|
|
|
if m.pollMetrics != nil {
|
|
m.pollMetrics.ResetQueueDepth(0)
|
|
}
|
|
|
|
// Load saved configurations
|
|
if alertConfig, err := m.configPersist.LoadAlertConfig(); err == nil {
|
|
m.alertManager.UpdateConfig(*alertConfig)
|
|
// Apply schedule settings to notification manager
|
|
m.notificationMgr.SetEnabled(alertConfig.Enabled && alertConfig.ActivationState == alerts.ActivationActive)
|
|
m.notificationMgr.SetCooldown(alertConfig.Schedule.Cooldown)
|
|
m.notificationMgr.SetGroupingConfig(
|
|
alertConfig.Schedule.Grouping.Enabled,
|
|
alertConfig.Schedule.Grouping.Window,
|
|
alertConfig.Schedule.Grouping.ByNode,
|
|
alertConfig.Schedule.Grouping.ByGuest,
|
|
)
|
|
m.notificationMgr.SetInitialNotifyTarget(alertConfig.Schedule.InitialNotify)
|
|
m.notificationMgr.SetNotifyOnResolve(alertConfig.Schedule.NotifyOnResolve)
|
|
} else {
|
|
log.Warn().Err(err).Msg("failed to load alert configuration")
|
|
}
|
|
if intentPolicies, err := m.configPersist.LoadAlertIntentPolicies(); err == nil {
|
|
if err := m.alertManager.LoadIntentPolicies(*intentPolicies); err != nil {
|
|
log.Warn().Err(err).Msg("failed to install alert intent policies")
|
|
}
|
|
} else {
|
|
log.Warn().Err(err).Msg("failed to load alert intent policies")
|
|
}
|
|
|
|
if emailConfig, err := m.configPersist.LoadEmailConfig(); err == nil {
|
|
m.notificationMgr.SetEmailConfig(*emailConfig)
|
|
} else {
|
|
log.Warn().Err(err).Msg("failed to load email configuration")
|
|
}
|
|
|
|
if concurrency > 0 {
|
|
m.guestMetadataSlots = make(chan struct{}, concurrency)
|
|
}
|
|
if guestAgentVMMaxConcurrent > 0 {
|
|
m.guestAgentWorkSlots = make(chan struct{}, guestAgentVMMaxConcurrent)
|
|
}
|
|
|
|
if appriseConfig, err := m.configPersist.LoadAppriseConfig(); err == nil {
|
|
m.notificationMgr.SetAppriseConfig(*appriseConfig)
|
|
} else {
|
|
log.Warn().Err(err).Msg("failed to load Apprise configuration")
|
|
}
|
|
if deadManConfig, err := m.configPersist.LoadDeadManConfig(); err == nil {
|
|
m.deadManConfig = *deadManConfig
|
|
} else {
|
|
m.deadManConfigLoadErr = err
|
|
log.Warn().Err(err).Msg("failed to load dead-man configuration")
|
|
}
|
|
|
|
// Migrate webhooks if needed (from unencrypted to encrypted)
|
|
if err := m.configPersist.MigrateWebhooksIfNeeded(); err != nil {
|
|
log.Warn().Err(err).Msg("failed to migrate webhooks")
|
|
}
|
|
|
|
if webhooks, err := m.configPersist.LoadWebhooks(); err == nil {
|
|
for _, webhook := range webhooks {
|
|
m.notificationMgr.AddWebhook(webhook)
|
|
}
|
|
} else {
|
|
log.Warn().Err(err).Msg("failed to load webhook configuration")
|
|
}
|
|
// The queue worker starts with the manager but has no processor until all
|
|
// saved destinations are installed. Activate it only after the last load;
|
|
// a slow migration or config read must not cancel due persisted work.
|
|
m.notificationMgr.SetQuietHoursPolicyProvider(m.alertManager.QuietHoursNotificationPolicy)
|
|
m.notificationMgr.StartQueueProcessing()
|
|
|
|
// In mock mode the canonical sampler owns demo chart history by default.
|
|
// Support-only hybrid runs can opt back into real client initialization.
|
|
mockEnabled := mock.IsMockEnabled()
|
|
if mockEnabled && !keepRealPollingInMockMode() {
|
|
log.Info().Msg("mock mode enabled - real client initialization disabled")
|
|
} else {
|
|
m.initPVEClients(cfg)
|
|
m.initPBSClients(cfg)
|
|
m.initPMGClients(cfg)
|
|
}
|
|
|
|
// Initialize state stats
|
|
m.state.Stats = models.Stats{
|
|
StartTime: m.startTime,
|
|
Version: "2.0.0-go",
|
|
}
|
|
|
|
// Agent endpoints can receive reports as soon as New returns, before the
|
|
// monitoring loop goroutine reaches Start. Wire external alert delivery now
|
|
// so a first startup observation cannot create an active alert without also
|
|
// reaching notifications. Start rewires the escalation callback with its
|
|
// WebSocket hub and installs lifecycle projection handling.
|
|
m.wireExternalAlertCallbacks(nil)
|
|
|
|
return m, nil
|
|
}
|
|
|
|
// SetExecutor allows tests to override the poll executor; passing nil restores the default executor.
|
|
func (m *Monitor) SetExecutor(exec PollExecutor) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if exec == nil {
|
|
m.executor = newRealExecutor(m)
|
|
return
|
|
}
|
|
|
|
m.executor = exec
|
|
}
|
|
|
|
func (m *Monitor) buildInstanceInfoCache(cfg *config.Config) {
|
|
_ = cfg
|
|
m.refreshInstanceInfoCacheFromProviders()
|
|
}
|
|
|
|
func (m *Monitor) getExecutor() PollExecutor {
|
|
m.mu.RLock()
|
|
exec := m.executor
|
|
m.mu.RUnlock()
|
|
return exec
|
|
}
|
|
|
|
func clampInterval(value, min, max time.Duration) time.Duration {
|
|
if value <= 0 {
|
|
return min
|
|
}
|
|
if min > 0 && value < min {
|
|
return min
|
|
}
|
|
if max > 0 && value > max {
|
|
return max
|
|
}
|
|
return value
|
|
}
|
|
|
|
func (m *Monitor) effectivePVEPollingInterval() time.Duration {
|
|
if m == nil {
|
|
return effectivePVEPollingIntervalForConfig(nil)
|
|
}
|
|
return effectivePVEPollingIntervalForConfig(m.config)
|
|
}
|
|
|
|
func (m *Monitor) fixedIntervalForInstance(instanceType InstanceType, instanceName string) time.Duration {
|
|
if provider, ok := m.getPollProvider(instanceType).(FixedIntervalPollProvider); ok && provider != nil {
|
|
return provider.FixedInstanceInterval(m, instanceName)
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (m *Monitor) baseIntervalForInstanceType(instanceType InstanceType) time.Duration {
|
|
if provider := m.getPollProvider(instanceType); provider != nil {
|
|
if interval := provider.BaseInterval(m); interval > 0 {
|
|
return interval
|
|
}
|
|
}
|
|
|
|
if m == nil || m.config == nil {
|
|
return DefaultSchedulerConfig().BaseInterval
|
|
}
|
|
|
|
switch instanceType {
|
|
case InstanceTypePVE:
|
|
return m.effectivePVEPollingInterval()
|
|
case InstanceTypePBS:
|
|
return clampInterval(m.pbsPollingIntervalSetting(), 10*time.Second, time.Hour)
|
|
case InstanceTypePMG:
|
|
return clampInterval(m.pmgPollingIntervalSetting(), 10*time.Second, time.Hour)
|
|
default:
|
|
base := m.config.AdaptivePollingBaseInterval
|
|
if base <= 0 {
|
|
base = DefaultSchedulerConfig().BaseInterval
|
|
}
|
|
return clampInterval(base, time.Second, 0)
|
|
}
|
|
}
|
|
|
|
// Start begins the monitoring loop
|
|
func (m *Monitor) Start(ctx context.Context, wsHub *websocket.Hub) {
|
|
// Consolidate any duplicate cluster instances before starting
|
|
// This fixes the case where multiple agents registered from the same cluster
|
|
m.consolidateDuplicateClusters()
|
|
|
|
pollingInterval := m.effectivePVEPollingInterval()
|
|
log.Info().
|
|
Dur("pollingInterval", pollingInterval).
|
|
Msg("Starting monitoring loop")
|
|
|
|
m.setRuntimeContext(ctx, wsHub)
|
|
defer m.stopMockMetricsSampler()
|
|
|
|
// Best-effort startup cleanup: when direct PBS is configured, remove legacy
|
|
// PVE-proxied PBS backup points to prevent duplicate recovery entries.
|
|
m.purgeStalePVEPBSBackupsBestEffort(ctx)
|
|
|
|
if mock.IsMockEnabled() {
|
|
m.startMockMetricsSampler(ctx)
|
|
}
|
|
|
|
// Initialize and start discovery service if enabled
|
|
if mock.IsMockEnabled() {
|
|
log.Info().Msg("mock mode enabled - skipping discovery service")
|
|
m.discoveryService = nil
|
|
} else if m.config.DiscoveryEnabled {
|
|
discoverySubnet := m.config.DiscoverySubnet
|
|
if discoverySubnet == "" {
|
|
discoverySubnet = "auto"
|
|
}
|
|
cfgProvider := func() config.DiscoveryConfig {
|
|
return m.discoveryConfigSnapshot()
|
|
}
|
|
m.discoveryService = discovery.NewService(wsHub, 5*time.Minute, discoverySubnet, cfgProvider)
|
|
if m.discoveryService != nil {
|
|
m.discoveryService.Start(ctx)
|
|
log.Info().Msg("discovery service initialized and started")
|
|
} else {
|
|
log.Error().Msg("failed to initialize discovery service")
|
|
}
|
|
} else {
|
|
log.Info().Msg("discovery service disabled by configuration")
|
|
m.discoveryService = nil
|
|
}
|
|
|
|
// Set up alert callbacks. Projection replay is deliberately absent here:
|
|
// the canonical resource store is not attached yet, so a replay now could
|
|
// not advance the durable watermark and would walk the same events again
|
|
// at the SetResourceStore boundary, which owns the single scheduled
|
|
// catch-up pass.
|
|
m.alertManager.SubscribeLifecycleCallback(m.handleAlertLifecycleEvent)
|
|
m.wireExternalAlertCallbacks(wsHub)
|
|
m.markDeadManMonitoringProgress(time.Now().UTC())
|
|
if err := m.deadManConfigurationLoadError(); err != nil {
|
|
m.alertManager.RaiseSystemAlert(alerts.SystemAlertInput{
|
|
Type: alerts.DeadManStateAlertType,
|
|
Level: alerts.AlertLevelWarning,
|
|
Message: "Pulse could not read the encrypted external watchdog configuration. Watchdog monitoring is unavailable until the destination is saved again.",
|
|
Fingerprint: "configuration-load-failed",
|
|
})
|
|
}
|
|
if m.deadMan != nil {
|
|
go m.deadMan.run(
|
|
ctx,
|
|
func() string {
|
|
if m.alertManager == nil {
|
|
return ""
|
|
}
|
|
return m.deadManConfigSnapshot().PingURL
|
|
},
|
|
m.deadManMonitoringProgress,
|
|
m.alertManager,
|
|
)
|
|
}
|
|
|
|
// Create separate tickers for polling and broadcasting using the configured cadence
|
|
|
|
workerCount := m.totalClientCount()
|
|
m.startTaskWorkers(ctx, workerCount)
|
|
|
|
pollTicker := time.NewTicker(pollingInterval)
|
|
defer pollTicker.Stop()
|
|
|
|
broadcastTicker := time.NewTicker(pollingInterval)
|
|
defer broadcastTicker.Stop()
|
|
deadManProgressTicker := time.NewTicker(15 * time.Second)
|
|
defer deadManProgressTicker.Stop()
|
|
|
|
keepRealPolling := keepRealPollingInMockMode()
|
|
|
|
// Start connection retry mechanism for failed clients
|
|
// This handles cases where network/Proxmox isn't ready on initial startup
|
|
if !mock.IsMockEnabled() || keepRealPolling {
|
|
go m.retryFailedConnections(ctx)
|
|
}
|
|
|
|
// Do an immediate poll on start.
|
|
if mock.IsMockEnabled() {
|
|
if keepRealPolling {
|
|
log.Info().Msg("mock mode enabled - running mock alerts and real metric polling")
|
|
go m.checkMockAlerts()
|
|
go m.poll(ctx, wsHub)
|
|
} else {
|
|
log.Info().Msg("mock mode enabled - skipping real node polling")
|
|
go m.checkMockAlerts()
|
|
}
|
|
} else {
|
|
go m.poll(ctx, wsHub)
|
|
}
|
|
|
|
for {
|
|
select {
|
|
case now := <-deadManProgressTicker.C:
|
|
// This tick runs on the canonical monitor loop itself. A separate
|
|
// heartbeat goroutine can therefore prove that scheduling remains
|
|
// responsive instead of merely proving its own timer is alive.
|
|
m.markDeadManMonitoringProgress(now.UTC())
|
|
case <-pollTicker.C:
|
|
now := time.Now()
|
|
m.evaluateDockerAgents(now)
|
|
m.evaluateKubernetesAgents(now)
|
|
m.evaluateHostAgents(now)
|
|
m.cleanupRemovedDockerHosts(now)
|
|
m.cleanupRemovedKubernetesClusters(now)
|
|
m.cleanupRemovedHostAgents(now)
|
|
m.cleanupGuestMetadataCache(now)
|
|
m.cleanupDiagnosticSnapshots(now)
|
|
m.cleanupRRDCache(now)
|
|
m.cleanupTrackingMaps(now)
|
|
m.cleanupMetricsHistory()
|
|
m.cleanupRateTracker(now)
|
|
m.evaluateNotificationDelivery(now)
|
|
if mock.IsMockEnabled() {
|
|
// In mock mode, keep synthetic alerts fresh
|
|
go m.checkMockAlerts()
|
|
if keepRealPolling {
|
|
// Keep real metrics flowing while mock UI mode is active.
|
|
go m.poll(ctx, wsHub)
|
|
}
|
|
} else {
|
|
// Poll real infrastructure
|
|
go m.poll(ctx, wsHub)
|
|
}
|
|
// Connection-degraded alerts derive from the unified connections
|
|
// ledger, which lives behind the api layer. We invoke the
|
|
// registered lister here so a wedged PVE/PBS/PMG/VMware/TrueNAS
|
|
// connection escalates into the top-nav alert stream.
|
|
go m.checkConnectionAlerts()
|
|
|
|
case <-broadcastTicker.C:
|
|
// Broadcast current state regardless of polling status
|
|
// GetState lazily initializes the mock alert snapshot. Preserve that
|
|
// fixture maintenance even when no browser is subscribed; production
|
|
// monitors still avoid the full snapshot build on this fast path.
|
|
if mock.IsMockEnabled() {
|
|
_ = m.GetState()
|
|
}
|
|
if !currentStateBroadcasterHasSubscribers(wsHub, m.GetOrgID()) {
|
|
continue
|
|
}
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
// Use GetState() instead of m.state.GetSnapshot() to respect mock mode
|
|
state := m.GetState()
|
|
log.Debug().
|
|
Int("nodes", len(state.Nodes)).
|
|
Int("vms", len(state.VMs)).
|
|
Int("containers", len(state.Containers)).
|
|
Int("hosts", len(state.Hosts)).
|
|
Int("pbs", len(state.PBSInstances)).
|
|
Int("pbsBackups", len(state.Backups.PBS)).
|
|
Int("physicalDisks", len(state.PhysicalDisks)).
|
|
Msg("Broadcasting state update (ticker)")
|
|
}
|
|
m.broadcastCurrentState(wsHub)
|
|
|
|
case <-ctx.Done():
|
|
log.Info().Msg("monitoring loop stopped")
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// wireExternalAlertCallbacks installs callbacks whose effects leave the alert
|
|
// manager. It is safe to call again: these are single callback slots, so Start
|
|
// replaces the constructor-time escalation closure with one carrying its hub.
|
|
func (m *Monitor) wireExternalAlertCallbacks(wsHub *websocket.Hub) {
|
|
if m == nil || m.alertManager == nil {
|
|
return
|
|
}
|
|
m.alertManager.SetAlertCallback(func(alert *alerts.Alert) {
|
|
m.handleAlertFired(alert)
|
|
})
|
|
// AI analysis bypasses activation and notification suppression so findings
|
|
// can be prepared while alert delivery is still pending review.
|
|
m.alertManager.SetAlertForAICallback(func(alert *alerts.Alert) {
|
|
log.Debug().Str("alertID", alert.ID).Msg("AI alert callback invoked (bypassing notification suppression)")
|
|
m.mu.RLock()
|
|
callback := m.alertTriggeredAICallback
|
|
m.mu.RUnlock()
|
|
if callback != nil {
|
|
callback(alert)
|
|
}
|
|
})
|
|
m.alertManager.SetResolvedAlertCallback(func(resolved *alerts.ResolvedAlert) {
|
|
m.handleResolvedAlert(resolved)
|
|
// Don't broadcast full state here - it causes a cascade with many guests.
|
|
// The frontend gets updated alerts through the regular broadcast ticker.
|
|
})
|
|
m.alertManager.SetEscalateCallback(func(alert *alerts.Alert, level int) {
|
|
m.handleAlertEscalated(wsHub, alert, level)
|
|
})
|
|
}
|
|
|
|
// poll fetches data from all configured instances
|
|
func (m *Monitor) poll(_ context.Context, wsHub *websocket.Hub) {
|
|
defer recoverFromPanic("poll")
|
|
|
|
// Limit concurrent polls to 2 to prevent resource exhaustion
|
|
currentCount := atomic.AddInt32(&m.activePollCount, 1)
|
|
if currentCount > 2 {
|
|
atomic.AddInt32(&m.activePollCount, -1)
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Int32("activePolls", currentCount-1).Msg("too many concurrent polls, skipping")
|
|
}
|
|
return
|
|
}
|
|
defer atomic.AddInt32(&m.activePollCount, -1)
|
|
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Msg("starting polling cycle")
|
|
}
|
|
startTime := time.Now()
|
|
now := startTime
|
|
|
|
plannedTasks := m.buildScheduledTasks(now)
|
|
for _, task := range plannedTasks {
|
|
m.taskQueue.Upsert(task)
|
|
}
|
|
m.updateQueueDepthMetric()
|
|
|
|
// Update performance metrics atomically to prevent data races when
|
|
// multiple poll() goroutines run concurrently (e.g. mock mode transitions).
|
|
wsClients := 0
|
|
if wsHub != nil {
|
|
wsClients = wsHub.GetClientCount()
|
|
}
|
|
m.state.UpdatePollStats(
|
|
time.Since(startTime).Seconds(),
|
|
int64(time.Since(m.startTime).Seconds()),
|
|
wsClients,
|
|
)
|
|
|
|
// Sync alert state so broadcasts include the latest acknowledgement data
|
|
m.syncAlertsToState()
|
|
|
|
// Increment poll counter
|
|
m.mu.Lock()
|
|
m.pollCounter++
|
|
m.mu.Unlock()
|
|
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Dur("duration", time.Since(startTime)).Msg("polling cycle completed")
|
|
}
|
|
|
|
// Broadcasting is now handled by the timer in Start()
|
|
}
|
|
|
|
func (m *Monitor) startTaskWorkers(ctx context.Context, workers int) {
|
|
if m.taskQueue == nil {
|
|
return
|
|
}
|
|
if limiter := processPollTaskWorkerLimiter(); limiter != nil {
|
|
// A monitor owns one dispatcher, while the process-wide limiter owns
|
|
// actual task concurrency. This prevents the override from multiplying
|
|
// into a full worker pool for every tenant.
|
|
go m.dispatchTaskWorkers(ctx, limiter)
|
|
return
|
|
}
|
|
workers = resolveTaskWorkerCount(workers)
|
|
for i := 0; i < workers; i++ {
|
|
go m.taskWorker(ctx, i)
|
|
}
|
|
}
|
|
|
|
const maxPollTaskWorkers = 128
|
|
|
|
type pollTaskWorkerLimiter struct {
|
|
slots chan struct{}
|
|
}
|
|
|
|
func newPollTaskWorkerLimiter(limit int) *pollTaskWorkerLimiter {
|
|
if limit < 1 {
|
|
limit = 1
|
|
}
|
|
return &pollTaskWorkerLimiter{slots: make(chan struct{}, limit)}
|
|
}
|
|
|
|
func (l *pollTaskWorkerLimiter) acquire(ctx context.Context) bool {
|
|
select {
|
|
case l.slots <- struct{}{}:
|
|
return true
|
|
case <-ctx.Done():
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (l *pollTaskWorkerLimiter) release() {
|
|
<-l.slots
|
|
}
|
|
|
|
var (
|
|
processPollTaskWorkerLimiterOnce sync.Once
|
|
processPollTaskWorkerLimit *pollTaskWorkerLimiter
|
|
)
|
|
|
|
func processPollTaskWorkerLimiter() *pollTaskWorkerLimiter {
|
|
processPollTaskWorkerLimiterOnce.Do(func() {
|
|
override := pollTaskWorkerOverride()
|
|
if override == 0 {
|
|
return
|
|
}
|
|
processPollTaskWorkerLimit = newPollTaskWorkerLimiter(override)
|
|
log.Info().
|
|
Int("workers", override).
|
|
Msg("Poll task worker count overridden process-wide by POLL_TASK_WORKERS")
|
|
})
|
|
return processPollTaskWorkerLimit
|
|
}
|
|
|
|
func pollTaskWorkerOverride() int {
|
|
override := parseNonNegativeIntEnv("POLL_TASK_WORKERS", 0)
|
|
if override > maxPollTaskWorkers {
|
|
return maxPollTaskWorkers
|
|
}
|
|
return override
|
|
}
|
|
|
|
// resolveTaskWorkerCount applies the default clamp of [1, 10] to the
|
|
// client-derived worker count. POLL_TASK_WORKERS is reported here for
|
|
// diagnostics and tests; startTaskWorkers routes that override through the
|
|
// process-wide limiter instead of starting this many workers per monitor.
|
|
func resolveTaskWorkerCount(workers int) int {
|
|
if override := pollTaskWorkerOverride(); override > 0 {
|
|
return override
|
|
}
|
|
if workers < 1 {
|
|
workers = 1
|
|
}
|
|
if workers > 10 {
|
|
workers = 10
|
|
}
|
|
return workers
|
|
}
|
|
|
|
func (m *Monitor) dispatchTaskWorkers(ctx context.Context, limiter *pollTaskWorkerLimiter) {
|
|
defer recoverFromPanic("taskWorkerDispatcher")
|
|
|
|
for {
|
|
task, ok := m.taskQueue.WaitNext(ctx)
|
|
if !ok {
|
|
return
|
|
}
|
|
if !limiter.acquire(ctx) {
|
|
return
|
|
}
|
|
|
|
go func() {
|
|
defer limiter.release()
|
|
defer recoverFromPanic(fmt.Sprintf("taskWorker-%s-%s", task.InstanceType, task.InstanceName))
|
|
m.executeAndRescheduleTask(ctx, task)
|
|
}()
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) taskWorker(ctx context.Context, id int) {
|
|
defer recoverFromPanic(fmt.Sprintf("taskWorker-%d", id))
|
|
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Int("worker", id).Msg("task worker started")
|
|
}
|
|
for {
|
|
task, ok := m.taskQueue.WaitNext(ctx)
|
|
if !ok {
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().Int("worker", id).Msg("task worker stopping")
|
|
}
|
|
return
|
|
}
|
|
|
|
m.executeAndRescheduleTask(ctx, task)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) executeAndRescheduleTask(ctx context.Context, task ScheduledTask) {
|
|
m.executeScheduledTask(ctx, task)
|
|
m.rescheduleTask(task)
|
|
m.updateQueueDepthMetric()
|
|
}
|
|
|
|
func derivePollTimeout(cfg *config.Config) time.Duration {
|
|
timeout := defaultTaskTimeout
|
|
if cfg != nil && cfg.ConnectionTimeout > 0 {
|
|
// ConnectionTimeout is the budget for one provider request, while a
|
|
// poll cycle may enumerate hundreds of guests across many cluster
|
|
// nodes. A short per-request timeout must not shrink the whole-cycle
|
|
// budget below the default; that cancelled otherwise healthy large
|
|
// PVE generations at exactly 30 seconds (#1437).
|
|
if requestScaledTimeout := cfg.ConnectionTimeout * 2; requestScaledTimeout > timeout {
|
|
timeout = requestScaledTimeout
|
|
}
|
|
}
|
|
if timeout < minTaskTimeout {
|
|
timeout = minTaskTimeout
|
|
}
|
|
// Use configurable max timeout from config (set via MAX_POLL_TIMEOUT env var)
|
|
// Falls back to hardcoded maxTaskTimeout if config is nil or MaxPollTimeout not set
|
|
maxTimeout := maxTaskTimeout
|
|
if cfg != nil && cfg.MaxPollTimeout > 0 {
|
|
maxTimeout = cfg.MaxPollTimeout
|
|
}
|
|
if timeout > maxTimeout {
|
|
timeout = maxTimeout
|
|
}
|
|
return timeout
|
|
}
|
|
|
|
func (m *Monitor) taskExecutionTimeout(_ InstanceType) time.Duration {
|
|
if m == nil {
|
|
return defaultTaskTimeout
|
|
}
|
|
timeout := m.pollTimeout
|
|
if timeout <= 0 {
|
|
timeout = defaultTaskTimeout
|
|
}
|
|
return timeout
|
|
}
|
|
|
|
func (m *Monitor) executeScheduledTask(ctx context.Context, task ScheduledTask) {
|
|
if !m.allowExecution(task) {
|
|
if logging.IsLevelEnabled(zerolog.DebugLevel) {
|
|
log.Debug().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Msg("Task blocked by circuit breaker")
|
|
}
|
|
return
|
|
}
|
|
|
|
if m.pollMetrics != nil {
|
|
wait := time.Duration(0)
|
|
if !task.NextRun.IsZero() {
|
|
wait = time.Since(task.NextRun)
|
|
if wait < 0 {
|
|
wait = 0
|
|
}
|
|
}
|
|
instanceType := string(task.InstanceType)
|
|
if strings.TrimSpace(instanceType) == "" {
|
|
instanceType = "unknown"
|
|
}
|
|
m.pollMetrics.RecordQueueWait(instanceType, wait)
|
|
}
|
|
|
|
executor := m.getExecutor()
|
|
if executor == nil {
|
|
log.Error().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Msg("No poll executor configured; skipping task")
|
|
return
|
|
}
|
|
|
|
pollTask, ok := m.buildPollTask(task)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
taskCtx := ctx
|
|
var cancel context.CancelFunc
|
|
timeout := m.taskExecutionTimeout(task.InstanceType)
|
|
if timeout > 0 {
|
|
taskCtx, cancel = context.WithTimeout(ctx, timeout)
|
|
defer cancel()
|
|
}
|
|
|
|
executor.Execute(taskCtx, pollTask)
|
|
|
|
if timeout > 0 && stderrors.Is(taskCtx.Err(), context.DeadlineExceeded) {
|
|
log.Warn().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Dur("timeout", timeout).
|
|
Msg("Polling task timed out; rescheduling with fresh worker")
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) buildPollTask(task ScheduledTask) (PollTask, bool) {
|
|
provider := m.getPollProvider(task.InstanceType)
|
|
if provider == nil {
|
|
log.Debug().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Msg("Skipping unsupported task type")
|
|
return PollTask{}, false
|
|
}
|
|
|
|
pollTask, err := provider.BuildPollTask(m, task.InstanceName)
|
|
if err != nil {
|
|
log.Warn().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Err(err).
|
|
Msg("Skipping scheduled task")
|
|
return PollTask{}, false
|
|
}
|
|
|
|
if strings.TrimSpace(pollTask.InstanceName) == "" {
|
|
pollTask.InstanceName = task.InstanceName
|
|
}
|
|
if strings.TrimSpace(pollTask.InstanceType) == "" {
|
|
pollTask.InstanceType = string(task.InstanceType)
|
|
}
|
|
return pollTask, true
|
|
}
|
|
|
|
func (m *Monitor) rescheduleTask(task ScheduledTask) {
|
|
if m.taskQueue == nil {
|
|
return
|
|
}
|
|
|
|
key := schedulerKey(task.InstanceType, task.InstanceName)
|
|
m.mu.Lock()
|
|
outcome, hasOutcome := m.lastOutcome[key]
|
|
failureCount := m.failureCounts[key]
|
|
m.mu.Unlock()
|
|
|
|
if hasOutcome && !outcome.success {
|
|
if !outcome.transient || failureCount >= m.maxRetryAttempts {
|
|
m.sendToDeadLetter(task, outcome.err)
|
|
return
|
|
}
|
|
delay := m.backoffCfg.nextDelay(failureCount-1, m.randomFloat())
|
|
if delay <= 0 {
|
|
delay = 5 * time.Second
|
|
}
|
|
if m.config != nil && m.config.AdaptivePollingEnabled && m.config.AdaptivePollingMaxInterval > 0 && m.config.AdaptivePollingMaxInterval <= 15*time.Second {
|
|
maxDelay := 4 * time.Second
|
|
if delay > maxDelay {
|
|
delay = maxDelay
|
|
}
|
|
}
|
|
next := task
|
|
next.Interval = delay
|
|
next.NextRun = time.Now().Add(delay)
|
|
m.taskQueue.Upsert(next)
|
|
return
|
|
}
|
|
|
|
fixedInterval := m.fixedIntervalForInstance(task.InstanceType, task.InstanceName)
|
|
|
|
if m.scheduler == nil {
|
|
nextInterval := fixedInterval
|
|
if nextInterval <= 0 {
|
|
nextInterval = task.Interval
|
|
}
|
|
if nextInterval <= 0 {
|
|
nextInterval = m.baseIntervalForInstanceType(task.InstanceType)
|
|
}
|
|
if nextInterval <= 0 {
|
|
nextInterval = DefaultSchedulerConfig().BaseInterval
|
|
}
|
|
next := task
|
|
next.NextRun = time.Now().Add(nextInterval)
|
|
next.Interval = nextInterval
|
|
m.taskQueue.Upsert(next)
|
|
return
|
|
}
|
|
|
|
desc := InstanceDescriptor{
|
|
Name: task.InstanceName,
|
|
Type: task.InstanceType,
|
|
LastInterval: task.Interval,
|
|
LastScheduled: task.NextRun,
|
|
FixedInterval: fixedInterval,
|
|
}
|
|
if m.stalenessTracker != nil {
|
|
if snap, ok := m.stalenessTracker.snapshot(task.InstanceType, task.InstanceName); ok {
|
|
desc.LastSuccess = snap.LastSuccess
|
|
desc.LastFailure = snap.LastError
|
|
if snap.ChangeHash != "" {
|
|
desc.Metadata = TaskMetadata{ChangeHash: snap.ChangeHash}
|
|
}
|
|
}
|
|
}
|
|
|
|
tasks := m.scheduler.BuildPlan(time.Now(), []InstanceDescriptor{desc}, m.taskQueue.Size())
|
|
if len(tasks) == 0 {
|
|
next := task
|
|
nextInterval := task.Interval
|
|
if nextInterval <= 0 && m.config != nil {
|
|
nextInterval = m.config.AdaptivePollingBaseInterval
|
|
}
|
|
if nextInterval <= 0 {
|
|
nextInterval = DefaultSchedulerConfig().BaseInterval
|
|
}
|
|
next.Interval = nextInterval
|
|
next.NextRun = time.Now().Add(nextInterval)
|
|
m.taskQueue.Upsert(next)
|
|
return
|
|
}
|
|
for _, next := range tasks {
|
|
m.taskQueue.Upsert(next)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) sendToDeadLetter(task ScheduledTask, err error) {
|
|
if m.deadLetterQueue == nil {
|
|
log.Error().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Err(err).
|
|
Msg("Dead-letter queue unavailable; dropping task")
|
|
return
|
|
}
|
|
|
|
log.Error().
|
|
Str("instance", task.InstanceName).
|
|
Str("type", string(task.InstanceType)).
|
|
Err(err).
|
|
Msg("Routing task to dead-letter queue after repeated failures")
|
|
|
|
next := task
|
|
next.Interval = 30 * time.Minute
|
|
next.NextRun = time.Now().Add(next.Interval)
|
|
m.deadLetterQueue.Upsert(next)
|
|
m.updateDeadLetterMetrics()
|
|
|
|
key := schedulerKey(task.InstanceType, task.InstanceName)
|
|
now := time.Now()
|
|
|
|
m.mu.Lock()
|
|
if m.dlqInsightMap == nil {
|
|
m.dlqInsightMap = make(map[string]*dlqInsight)
|
|
}
|
|
info, ok := m.dlqInsightMap[key]
|
|
if !ok {
|
|
info = &dlqInsight{}
|
|
m.dlqInsightMap[key] = info
|
|
}
|
|
if info.FirstAttempt.IsZero() {
|
|
info.FirstAttempt = now
|
|
}
|
|
info.LastAttempt = now
|
|
info.RetryCount++
|
|
info.NextRetry = next.NextRun
|
|
if err != nil {
|
|
info.Reason = classifyDLQReason(err)
|
|
}
|
|
m.mu.Unlock()
|
|
}
|
|
|
|
func classifyDLQReason(err error) string {
|
|
if err == nil {
|
|
return ""
|
|
}
|
|
if errors.IsRetryableError(err) {
|
|
return "max_retry_attempts"
|
|
}
|
|
return "permanent_failure"
|
|
}
|
|
|
|
func (m *Monitor) updateDeadLetterMetrics() {
|
|
if m.pollMetrics == nil || m.deadLetterQueue == nil {
|
|
return
|
|
}
|
|
|
|
size := m.deadLetterQueue.Size()
|
|
if size <= 0 {
|
|
m.pollMetrics.UpdateDeadLetterCounts(nil)
|
|
return
|
|
}
|
|
|
|
tasks := m.deadLetterQueue.PeekAll(size)
|
|
m.pollMetrics.UpdateDeadLetterCounts(tasks)
|
|
}
|
|
|
|
func (m *Monitor) updateBreakerMetric(instanceType InstanceType, instance string, breaker *circuitBreaker) {
|
|
if m.pollMetrics == nil || breaker == nil {
|
|
return
|
|
}
|
|
|
|
state, failures, retryAt, _, _ := breaker.stateDetails()
|
|
m.pollMetrics.SetBreakerState(string(instanceType), instance, state, failures, retryAt)
|
|
}
|
|
|
|
func (m *Monitor) randomFloat() float64 {
|
|
if m.rng == nil {
|
|
m.rng = rand.New(rand.NewSource(time.Now().UnixNano()))
|
|
}
|
|
return m.rng.Float64()
|
|
}
|
|
|
|
func (m *Monitor) updateQueueDepthMetric() {
|
|
if m.pollMetrics == nil || m.taskQueue == nil {
|
|
return
|
|
}
|
|
snapshot := m.taskQueue.Snapshot()
|
|
m.pollMetrics.SetQueueDepth(snapshot.Depth)
|
|
m.pollMetrics.UpdateQueueSnapshot(snapshot)
|
|
}
|
|
|
|
func (m *Monitor) allowExecution(task ScheduledTask) bool {
|
|
if m.circuitBreakers == nil {
|
|
return true
|
|
}
|
|
key := schedulerKey(task.InstanceType, task.InstanceName)
|
|
breaker := m.ensureBreaker(key)
|
|
allowed := breaker.allow(time.Now())
|
|
m.updateBreakerMetric(task.InstanceType, task.InstanceName, breaker)
|
|
return allowed
|
|
}
|
|
|
|
func (m *Monitor) ensureBreaker(key string) *circuitBreaker {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if m.circuitBreakers == nil {
|
|
m.circuitBreakers = make(map[string]*circuitBreaker)
|
|
}
|
|
if breaker, ok := m.circuitBreakers[key]; ok {
|
|
return breaker
|
|
}
|
|
baseRetry := m.breakerBaseRetry
|
|
if baseRetry <= 0 {
|
|
baseRetry = 5 * time.Second
|
|
}
|
|
maxDelay := m.breakerMaxDelay
|
|
if maxDelay <= 0 {
|
|
maxDelay = 5 * time.Minute
|
|
}
|
|
halfOpen := m.breakerHalfOpenWindow
|
|
if halfOpen <= 0 {
|
|
halfOpen = 30 * time.Second
|
|
}
|
|
breaker := newCircuitBreaker(3, baseRetry, maxDelay, halfOpen)
|
|
m.circuitBreakers[key] = breaker
|
|
return breaker
|
|
}
|
|
|
|
func (m *Monitor) recordTaskResult(instanceType InstanceType, instance string, pollErr error) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
|
|
key := schedulerKey(instanceType, instance)
|
|
now := time.Now()
|
|
|
|
breaker := m.ensureBreaker(key)
|
|
|
|
m.mu.Lock()
|
|
status, ok := m.pollStatusMap[key]
|
|
if !ok {
|
|
status = &pollStatus{}
|
|
m.pollStatusMap[key] = status
|
|
}
|
|
|
|
if pollErr == nil {
|
|
if m.failureCounts != nil {
|
|
m.failureCounts[key] = 0
|
|
}
|
|
if m.lastOutcome != nil {
|
|
m.lastOutcome[key] = taskOutcome{
|
|
success: true,
|
|
transient: true,
|
|
err: nil,
|
|
recordedAt: now,
|
|
}
|
|
}
|
|
status.LastSuccess = now
|
|
status.ConsecutiveFailures = 0
|
|
status.FirstFailureAt = time.Time{}
|
|
// A recorded error means "current outstanding failure", not "last error
|
|
// ever". Clearing it on success keeps downstream consumers honest: the
|
|
// connections aggregator surfaces LastError as a live error banner and
|
|
// derives Unauthorized state from it, so a stale entry would show a
|
|
// red error on a healthy connection forever (#1493).
|
|
status.LastErrorAt = time.Time{}
|
|
status.LastErrorMessage = ""
|
|
status.LastErrorCategory = ""
|
|
m.mu.Unlock()
|
|
if breaker != nil {
|
|
breaker.recordSuccess()
|
|
m.updateBreakerMetric(instanceType, instance, breaker)
|
|
}
|
|
return
|
|
}
|
|
|
|
transient := isTransientError(pollErr)
|
|
category := "permanent"
|
|
if transient {
|
|
category = "transient"
|
|
}
|
|
if m.failureCounts != nil {
|
|
m.failureCounts[key] = m.failureCounts[key] + 1
|
|
}
|
|
if m.lastOutcome != nil {
|
|
m.lastOutcome[key] = taskOutcome{
|
|
success: false,
|
|
transient: transient,
|
|
err: pollErr,
|
|
recordedAt: now,
|
|
}
|
|
}
|
|
status.LastErrorAt = now
|
|
status.LastErrorMessage = pollErr.Error()
|
|
status.LastErrorCategory = category
|
|
status.ConsecutiveFailures++
|
|
if status.ConsecutiveFailures == 1 {
|
|
status.FirstFailureAt = now
|
|
}
|
|
m.mu.Unlock()
|
|
if breaker != nil {
|
|
breaker.recordFailure(now)
|
|
m.updateBreakerMetric(instanceType, instance, breaker)
|
|
}
|
|
}
|
|
|
|
// SchedulerHealthResponse contains complete scheduler health data for API exposure.
|
|
type SchedulerHealthResponse struct {
|
|
UpdatedAt time.Time `json:"updatedAt"`
|
|
Enabled bool `json:"enabled"`
|
|
Queue QueueSnapshot `json:"queue"`
|
|
DeadLetter DeadLetterSnapshot `json:"deadLetter"`
|
|
Breakers []BreakerSnapshot `json:"breakers"`
|
|
Staleness []StalenessSnapshot `json:"staleness"`
|
|
Instances []InstanceHealth `json:"instances"`
|
|
}
|
|
|
|
// DeadLetterSnapshot contains dead-letter queue data.
|
|
type DeadLetterSnapshot struct {
|
|
Count int `json:"count"`
|
|
Tasks []DeadLetterTask `json:"tasks"`
|
|
}
|
|
|
|
func emptyDeadLetterSnapshot() DeadLetterSnapshot {
|
|
return DeadLetterSnapshot{
|
|
Tasks: []DeadLetterTask{},
|
|
}
|
|
}
|
|
|
|
func emptySchedulerHealthResponse(enabled bool) SchedulerHealthResponse {
|
|
return SchedulerHealthResponse{
|
|
UpdatedAt: time.Now(),
|
|
Enabled: enabled,
|
|
Queue: emptyQueueSnapshot(),
|
|
DeadLetter: emptyDeadLetterSnapshot(),
|
|
Breakers: []BreakerSnapshot{},
|
|
Staleness: []StalenessSnapshot{},
|
|
Instances: []InstanceHealth{},
|
|
}
|
|
}
|
|
|
|
// SchedulerHealth returns a complete snapshot of scheduler health for API exposure.
|
|
// DeadLetterCount reports the number of dead-lettered poll tasks using only
|
|
// the queue's own short-lived mutex. Liveness probes (/api/health) use this
|
|
// instead of SchedulerHealth, which refreshes provider caches under the
|
|
// monitor's write lock on every call — far more work and lock surface than a
|
|
// watchdog-polled liveness check should pay for.
|
|
func (m *Monitor) DeadLetterCount() int {
|
|
if m == nil || m.deadLetterQueue == nil {
|
|
return 0
|
|
}
|
|
return m.deadLetterQueue.Size()
|
|
}
|
|
|
|
// PlannedPollInterval reports the adaptive scheduler's currently planned
|
|
// interval for one instance, or zero when no plan exists (adaptive polling
|
|
// disabled, unknown instance, or startup before the first plan). Consumers
|
|
// use it to judge poll freshness against the schedule actually in force
|
|
// rather than the configured cadence, which adaptive polling deliberately
|
|
// stretches while data is fresh.
|
|
func (m *Monitor) PlannedPollInterval(instanceType InstanceType, instanceName string) time.Duration {
|
|
if m == nil || m.scheduler == nil {
|
|
return 0
|
|
}
|
|
if task, ok := m.scheduler.LastScheduled(instanceType, instanceName); ok {
|
|
return task.Interval
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (m *Monitor) SchedulerHealth() SchedulerHealthResponse {
|
|
response := emptySchedulerHealthResponse(m.config != nil && m.config.AdaptivePollingEnabled)
|
|
|
|
m.refreshInstanceInfoCacheFromProviders()
|
|
|
|
// Queue snapshot
|
|
if m.taskQueue != nil {
|
|
response.Queue = m.taskQueue.Snapshot()
|
|
if m.pollMetrics != nil {
|
|
m.pollMetrics.UpdateQueueSnapshot(response.Queue)
|
|
}
|
|
}
|
|
|
|
// Dead-letter queue snapshot
|
|
if m.deadLetterQueue != nil {
|
|
deadLetterTasks := m.deadLetterQueue.PeekAll(25) // limit to top 25
|
|
m.mu.RLock()
|
|
for i := range deadLetterTasks {
|
|
key := schedulerKey(InstanceType(deadLetterTasks[i].Type), deadLetterTasks[i].Instance)
|
|
if outcome, ok := m.lastOutcome[key]; ok && outcome.err != nil {
|
|
deadLetterTasks[i].LastError = outcome.err.Error()
|
|
}
|
|
if count, ok := m.failureCounts[key]; ok {
|
|
deadLetterTasks[i].Failures = count
|
|
}
|
|
}
|
|
m.mu.RUnlock()
|
|
response.DeadLetter = DeadLetterSnapshot{
|
|
Count: m.deadLetterQueue.Size(),
|
|
Tasks: deadLetterTasks,
|
|
}
|
|
m.updateDeadLetterMetrics()
|
|
}
|
|
|
|
// Circuit breaker snapshots
|
|
m.mu.RLock()
|
|
breakerSnapshots := make([]BreakerSnapshot, 0, len(m.circuitBreakers))
|
|
for key, breaker := range m.circuitBreakers {
|
|
state, failures, retryAt := breaker.State()
|
|
// Only include breakers that are not in default closed state with 0 failures
|
|
if state != "closed" || failures > 0 {
|
|
// Parse instance type and name from key
|
|
parts := strings.SplitN(key, "::", 2)
|
|
instanceType, instanceName := "unknown", key
|
|
if len(parts) == 2 {
|
|
instanceType, instanceName = parts[0], parts[1]
|
|
}
|
|
breakerSnapshots = append(breakerSnapshots, BreakerSnapshot{
|
|
Instance: instanceName,
|
|
Type: instanceType,
|
|
State: state,
|
|
Failures: failures,
|
|
RetryAt: retryAt,
|
|
})
|
|
}
|
|
}
|
|
m.mu.RUnlock()
|
|
response.Breakers = breakerSnapshots
|
|
|
|
// Staleness snapshots
|
|
if m.stalenessTracker != nil {
|
|
response.Staleness = m.stalenessTracker.Snapshot()
|
|
}
|
|
|
|
instanceInfos := make(map[string]*instanceInfo)
|
|
pollStatuses := make(map[string]pollStatus)
|
|
dlqInsights := make(map[string]dlqInsight)
|
|
breakerRefs := make(map[string]*circuitBreaker)
|
|
|
|
m.mu.RLock()
|
|
for k, v := range m.instanceInfoCache {
|
|
if v == nil {
|
|
continue
|
|
}
|
|
copyVal := *v
|
|
instanceInfos[k] = ©Val
|
|
}
|
|
for k, v := range m.pollStatusMap {
|
|
if v == nil {
|
|
continue
|
|
}
|
|
pollStatuses[k] = *v
|
|
}
|
|
for k, v := range m.dlqInsightMap {
|
|
if v == nil {
|
|
continue
|
|
}
|
|
dlqInsights[k] = *v
|
|
}
|
|
for k, v := range m.circuitBreakers {
|
|
if v != nil {
|
|
breakerRefs[k] = v
|
|
}
|
|
}
|
|
m.mu.RUnlock()
|
|
for key, breaker := range breakerRefs {
|
|
instanceType := InstanceType("unknown")
|
|
instanceName := key
|
|
if parts := strings.SplitN(key, "::", 2); len(parts) == 2 {
|
|
if parts[0] != "" {
|
|
instanceType = InstanceType(parts[0])
|
|
}
|
|
if parts[1] != "" {
|
|
instanceName = parts[1]
|
|
}
|
|
}
|
|
m.updateBreakerMetric(instanceType, instanceName, breaker)
|
|
}
|
|
|
|
keySet := make(map[string]struct{})
|
|
for k := range instanceInfos {
|
|
if k != "" {
|
|
keySet[k] = struct{}{}
|
|
}
|
|
}
|
|
for k := range pollStatuses {
|
|
if k != "" {
|
|
keySet[k] = struct{}{}
|
|
}
|
|
}
|
|
for k := range dlqInsights {
|
|
if k != "" {
|
|
keySet[k] = struct{}{}
|
|
}
|
|
}
|
|
for k := range breakerRefs {
|
|
if k != "" {
|
|
keySet[k] = struct{}{}
|
|
}
|
|
}
|
|
for _, task := range response.DeadLetter.Tasks {
|
|
if task.Instance == "" {
|
|
continue
|
|
}
|
|
keySet[schedulerKey(InstanceType(task.Type), task.Instance)] = struct{}{}
|
|
}
|
|
for _, snap := range response.Staleness {
|
|
if snap.Instance == "" {
|
|
continue
|
|
}
|
|
keySet[schedulerKey(InstanceType(snap.Type), snap.Instance)] = struct{}{}
|
|
}
|
|
|
|
if len(keySet) > 0 {
|
|
keys := make([]string, 0, len(keySet))
|
|
for k := range keySet {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
|
|
instances := make([]InstanceHealth, 0, len(keys))
|
|
for _, key := range keys {
|
|
instType := "unknown"
|
|
instName := key
|
|
if parts := strings.SplitN(key, "::", 2); len(parts) == 2 {
|
|
if parts[0] != "" {
|
|
instType = parts[0]
|
|
}
|
|
if parts[1] != "" {
|
|
instName = parts[1]
|
|
}
|
|
}
|
|
instType = strings.TrimSpace(instType)
|
|
instName = strings.TrimSpace(instName)
|
|
|
|
info := instanceInfos[key]
|
|
display := instName
|
|
connection := ""
|
|
if info != nil {
|
|
if instType == "unknown" || instType == "" {
|
|
if info.Type != "" {
|
|
instType = string(info.Type)
|
|
}
|
|
}
|
|
if strings.Contains(info.Key, "::") {
|
|
if parts := strings.SplitN(info.Key, "::", 2); len(parts) == 2 {
|
|
if instName == key {
|
|
instName = parts[1]
|
|
}
|
|
if (instType == "" || instType == "unknown") && parts[0] != "" {
|
|
instType = parts[0]
|
|
}
|
|
}
|
|
}
|
|
if info.DisplayName != "" {
|
|
display = info.DisplayName
|
|
}
|
|
if info.Connection != "" {
|
|
connection = info.Connection
|
|
}
|
|
}
|
|
display = strings.TrimSpace(display)
|
|
connection = strings.TrimSpace(connection)
|
|
if display == "" {
|
|
display = instName
|
|
}
|
|
if display == "" {
|
|
display = connection
|
|
}
|
|
if instType == "" {
|
|
instType = "unknown"
|
|
}
|
|
if instName == "" {
|
|
instName = key
|
|
}
|
|
|
|
status, hasStatus := pollStatuses[key]
|
|
instanceStatus := InstancePollStatus{}
|
|
if hasStatus {
|
|
instanceStatus.ConsecutiveFailures = status.ConsecutiveFailures
|
|
instanceStatus.LastSuccess = timePtr(status.LastSuccess)
|
|
if !status.FirstFailureAt.IsZero() {
|
|
instanceStatus.FirstFailureAt = timePtr(status.FirstFailureAt)
|
|
}
|
|
if !status.LastErrorAt.IsZero() && status.LastErrorMessage != "" {
|
|
instanceStatus.LastError = &ErrorDetail{
|
|
At: status.LastErrorAt,
|
|
Message: status.LastErrorMessage,
|
|
Category: status.LastErrorCategory,
|
|
}
|
|
}
|
|
}
|
|
|
|
breakerInfo := InstanceBreaker{
|
|
State: "closed",
|
|
FailureCount: 0,
|
|
}
|
|
if br, ok := breakerRefs[key]; ok && br != nil {
|
|
state, failures, retryAt, since, lastTransition := br.stateDetails()
|
|
if state != "" {
|
|
breakerInfo.State = state
|
|
}
|
|
breakerInfo.FailureCount = failures
|
|
breakerInfo.RetryAt = timePtr(retryAt)
|
|
breakerInfo.Since = timePtr(since)
|
|
breakerInfo.LastTransition = timePtr(lastTransition)
|
|
}
|
|
|
|
dlqInfo := InstanceDLQ{Present: false}
|
|
if dlq, ok := dlqInsights[key]; ok {
|
|
dlqInfo.Present = true
|
|
dlqInfo.Reason = dlq.Reason
|
|
dlqInfo.FirstAttempt = timePtr(dlq.FirstAttempt)
|
|
dlqInfo.LastAttempt = timePtr(dlq.LastAttempt)
|
|
dlqInfo.RetryCount = dlq.RetryCount
|
|
dlqInfo.NextRetry = timePtr(dlq.NextRetry)
|
|
}
|
|
|
|
// Collect any warnings for this instance
|
|
var warnings []string
|
|
if instType == "pve" {
|
|
if warning, ok := m.backupPermissionWarnings[instName]; ok {
|
|
warnings = append(warnings, warning)
|
|
}
|
|
}
|
|
|
|
instances = append(instances, InstanceHealth{
|
|
Key: key,
|
|
Type: instType,
|
|
DisplayName: display,
|
|
Instance: instName,
|
|
Connection: connection,
|
|
PollStatus: instanceStatus,
|
|
Breaker: breakerInfo,
|
|
DeadLetter: dlqInfo,
|
|
Warnings: warnings,
|
|
}.NormalizeCollections())
|
|
}
|
|
|
|
response.Instances = instances
|
|
} else {
|
|
response.Instances = []InstanceHealth{}
|
|
}
|
|
|
|
return response
|
|
}
|
|
|
|
func isTransientError(err error) bool {
|
|
if err == nil {
|
|
return true
|
|
}
|
|
if errors.IsRetryableError(err) {
|
|
return true
|
|
}
|
|
if stderrors.Is(err, context.Canceled) || stderrors.Is(err, context.DeadlineExceeded) {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (m *Monitor) GetState() models.StateSnapshot {
|
|
if m == nil {
|
|
return models.StateSnapshot{}
|
|
}
|
|
|
|
// Check if mock mode is enabled
|
|
if mock.IsMockEnabled() {
|
|
state := mock.CurrentFixtureGraph().State
|
|
if state.ActiveAlerts == nil && m.alertManager != nil {
|
|
// Populate snapshot lazily if the cache hasn't been filled yet.
|
|
mock.UpdateAlertSnapshots(m.alertManager.GetActiveAlerts(), m.alertManager.GetRecentlyResolved())
|
|
state = mock.CurrentFixtureGraph().State
|
|
}
|
|
return state
|
|
}
|
|
// Mock-mode transitions replace the state under m.mu. Capture the fully
|
|
// initialized pointer under the same lock, then let State protect its own
|
|
// snapshot without holding the monitor lock across downstream readers.
|
|
m.mu.RLock()
|
|
currentState := m.state
|
|
m.mu.RUnlock()
|
|
if currentState == nil {
|
|
return models.StateSnapshot{}
|
|
}
|
|
|
|
state := currentState.GetSnapshot()
|
|
// Keep externally served alert arrays aligned with the live alert manager
|
|
// even between explicit sync points, so APIs do not expose stale alert
|
|
// counts or recently resolved incidents from cached state.
|
|
if m.alertManager != nil {
|
|
state.ActiveAlerts = m.activeAlertsSnapshot()
|
|
state.RecentlyResolved = m.recentlyResolvedAlertsSnapshot()
|
|
}
|
|
// Without an alert manager, retain alerts from this same captured state;
|
|
// the fallback helpers would read the replaceable pointer again.
|
|
// Surface filesystems reported by a unified pulse-agent inside a guest
|
|
// (for example ZFS mounts that qemu-guest-agent's get-fsinfo cannot see
|
|
// on PBS, #1438) in the guest overview disk listing.
|
|
state.MergeLinkedHostDisksIntoGuests()
|
|
return state
|
|
}
|
|
|
|
// ReadSnapshot returns a snapshot of the current infrastructure state,
|
|
// respecting mock mode when enabled.
|
|
//
|
|
// This is the preferred accessor for consumer code that needs the full
|
|
// StateSnapshot (e.g., chart rendering, reporting, AI state queries).
|
|
// This method satisfies models.SnapshotProvider — the single canonical
|
|
// interface that all consumer packages depend on. Fields available
|
|
// via ReadState should be accessed there instead when practical.
|
|
func (m *Monitor) ReadSnapshot() models.StateSnapshot {
|
|
return m.GetState()
|
|
}
|
|
|
|
// BackupsSnapshot returns the current backup state.
|
|
func (m *Monitor) BackupsSnapshot() models.Backups {
|
|
return m.GetState().Backups
|
|
}
|
|
|
|
// PBSInstancesSnapshot returns the current PBS instances.
|
|
func (m *Monitor) PBSInstancesSnapshot() []models.PBSInstance {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
pbsViews := readState.PBSInstances()
|
|
if len(pbsViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
instances := make([]models.PBSInstance, 0, len(pbsViews))
|
|
for _, instance := range pbsViews {
|
|
if instance == nil {
|
|
continue
|
|
}
|
|
instances = append(instances, pbsInstanceFromReadStateView(instance))
|
|
}
|
|
return instances
|
|
}
|
|
|
|
// ReplicationJobsSnapshot returns the current replication jobs.
|
|
func (m *Monitor) ReplicationJobsSnapshot() []models.ReplicationJob {
|
|
return m.GetState().ReplicationJobs
|
|
}
|
|
|
|
// ConnectionHealthSnapshot returns the current connection health map.
|
|
func (m *Monitor) ConnectionHealthSnapshot() map[string]bool {
|
|
return m.GetState().ConnectionHealth
|
|
}
|
|
|
|
// HostsSnapshot returns the current hosts.
|
|
func (m *Monitor) HostsSnapshot() []models.Host {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
readState = m.readStateWithStandaloneHostContinuity(readState)
|
|
|
|
hostViews := readState.Hosts()
|
|
if len(hostViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
hosts := make([]models.Host, 0, len(hostViews))
|
|
for _, host := range hostViews {
|
|
if host == nil {
|
|
continue
|
|
}
|
|
hosts = append(hosts, hostFromReadStateView(host))
|
|
}
|
|
return hosts
|
|
}
|
|
|
|
// VMsSnapshot returns the current VMs.
|
|
func (m *Monitor) VMsSnapshot() []models.VM {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
vmViews := readState.VMs()
|
|
if len(vmViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
vms := make([]models.VM, 0, len(vmViews))
|
|
for _, vm := range vmViews {
|
|
if vm == nil {
|
|
continue
|
|
}
|
|
vms = append(vms, vmFromReadStateView(vm))
|
|
}
|
|
return vms
|
|
}
|
|
|
|
// ContainersSnapshot returns the current system containers.
|
|
func (m *Monitor) ContainersSnapshot() []models.Container {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
containerViews := readState.Containers()
|
|
if len(containerViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
containers := make([]models.Container, 0, len(containerViews))
|
|
for _, container := range containerViews {
|
|
if container == nil {
|
|
continue
|
|
}
|
|
containers = append(containers, containerFromReadStateView(container))
|
|
}
|
|
return containers
|
|
}
|
|
|
|
// NodesSnapshot returns the current Proxmox nodes.
|
|
func (m *Monitor) NodesSnapshot() []models.Node {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
nodeViews := readState.Nodes()
|
|
if len(nodeViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
nodes := make([]models.Node, 0, len(nodeViews))
|
|
for _, node := range nodeViews {
|
|
if node == nil {
|
|
continue
|
|
}
|
|
nodes = append(nodes, nodeFromReadStateView(node))
|
|
}
|
|
return nodes
|
|
}
|
|
|
|
// DockerHostsSnapshot returns the current Docker hosts.
|
|
func (m *Monitor) DockerHostsSnapshot() []models.DockerHost {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
dockerViews := readState.DockerHosts()
|
|
if len(dockerViews) == 0 {
|
|
return nil
|
|
}
|
|
|
|
hosts := make([]models.DockerHost, 0, len(dockerViews))
|
|
for _, host := range dockerViews {
|
|
if host == nil {
|
|
continue
|
|
}
|
|
hosts = append(hosts, dockerHostFromReadStateView(host))
|
|
}
|
|
return hosts
|
|
}
|
|
|
|
// StorageSnapshot returns the current storage pools.
|
|
func (m *Monitor) StorageSnapshot() []models.Storage {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
readState := m.GetUnifiedReadStateOrSnapshot()
|
|
if readState == nil {
|
|
return nil
|
|
}
|
|
|
|
storagePools := readState.StoragePools()
|
|
if len(storagePools) == 0 {
|
|
return nil
|
|
}
|
|
|
|
storage := make([]models.Storage, 0, len(storagePools))
|
|
for _, pool := range storagePools {
|
|
if pool == nil {
|
|
continue
|
|
}
|
|
storage = append(storage, storageFromReadStateView(pool))
|
|
}
|
|
return storage
|
|
}
|
|
|
|
func storageFromReadStateView(view *unifiedresources.StoragePoolView) models.Storage {
|
|
if view == nil {
|
|
return models.Storage{}
|
|
}
|
|
|
|
storageID := strings.TrimSpace(view.SourceID())
|
|
if storageID == "" {
|
|
storageID = strings.TrimSpace(view.ID())
|
|
}
|
|
|
|
nodes := view.AccessibleNodes()
|
|
nodeIDs := storageNodeIDsFromReadState(view.Instance(), nodes)
|
|
total := view.DiskTotal()
|
|
used := view.DiskUsed()
|
|
free := total - used
|
|
if free < 0 {
|
|
free = 0
|
|
}
|
|
|
|
return models.Storage{
|
|
ID: storageID,
|
|
Name: view.Name(),
|
|
Node: view.Node(),
|
|
Instance: view.Instance(),
|
|
Nodes: nodes,
|
|
NodeIDs: nodeIDs,
|
|
NodeCount: len(nodes),
|
|
Type: view.StorageType(),
|
|
Status: string(view.Status()),
|
|
Path: view.Path(),
|
|
Total: total,
|
|
Used: used,
|
|
Free: free,
|
|
Usage: view.DiskPercent(),
|
|
Content: view.Content(),
|
|
Shared: view.Shared(),
|
|
Enabled: view.Enabled(),
|
|
Active: view.Active(),
|
|
LastSeen: view.LastSeen(),
|
|
ZFSPool: storageZFSPoolFromReadStateView(view),
|
|
}
|
|
}
|
|
|
|
func proxmoxRatioFromPercent(percent float64) float64 {
|
|
if math.IsNaN(percent) || math.IsInf(percent, 0) || percent <= 0 {
|
|
return 0
|
|
}
|
|
return percent / 100
|
|
}
|
|
|
|
func nodeFromReadStateView(view *unifiedresources.NodeView) models.Node {
|
|
if view == nil {
|
|
return models.Node{}
|
|
}
|
|
|
|
name := view.NodeName()
|
|
displayName := ""
|
|
if trimmed := strings.TrimSpace(view.Name()); trimmed != "" && trimmed != name {
|
|
displayName = trimmed
|
|
}
|
|
|
|
return models.Node{
|
|
ID: firstNonEmptyString(view.SourceID(), view.ID()),
|
|
Name: name,
|
|
DisplayName: displayName,
|
|
Instance: view.Instance(),
|
|
Host: view.HostURL(),
|
|
GuestURL: view.GuestURL(),
|
|
Status: string(view.Status()),
|
|
Type: "node",
|
|
CPU: proxmoxRatioFromPercent(view.CPUPercent()),
|
|
Memory: models.Memory{Used: view.MemoryUsed(), Total: view.MemoryTotal(), Free: maxInt64(0, view.MemoryTotal()-view.MemoryUsed()), Usage: view.MemoryPercent()},
|
|
Disk: models.Disk{Used: view.DiskUsed(), Total: view.DiskTotal(), Free: maxInt64(0, view.DiskTotal()-view.DiskUsed()), Usage: view.DiskPercent()},
|
|
Uptime: view.Uptime(),
|
|
LoadAverage: view.LoadAverage(),
|
|
KernelVersion: view.KernelVersion(),
|
|
PVEVersion: view.PVEVersion(),
|
|
CPUInfo: view.CPUInfo(),
|
|
NetworkInterfaces: hostNetworkInterfacesFromReadStateView(view.NetworkInterfaces()),
|
|
Temperature: view.TemperatureDetails(),
|
|
TemperatureMonitoringEnabled: view.TemperatureMonitoringEnabled(),
|
|
LastSeen: view.LastSeen(),
|
|
ConnectionHealth: view.ConnectionHealth(),
|
|
IsClusterMember: view.IsClusterMember(),
|
|
ClusterName: view.ClusterName(),
|
|
PendingUpdates: view.PendingUpdates(),
|
|
PendingUpdatesCheckedAt: view.PendingUpdatesCheckedAt(),
|
|
PendingUpdatesStatus: view.PendingUpdatesStatus(),
|
|
PendingUpdatesReason: view.PendingUpdatesReason(),
|
|
LinkedAgentID: view.LinkedAgentID(),
|
|
}
|
|
}
|
|
|
|
func hostFromReadStateView(view *unifiedresources.HostView) models.Host {
|
|
if view == nil {
|
|
return models.Host{}
|
|
}
|
|
|
|
displayName := ""
|
|
if trimmed := strings.TrimSpace(view.Name()); trimmed != "" && trimmed != view.Hostname() {
|
|
displayName = trimmed
|
|
}
|
|
status, lastSeen := hostAgentLivenessFromReadStateView(view, time.Now())
|
|
|
|
return models.Host{
|
|
ID: firstNonEmptyString(view.AgentID(), view.ID()),
|
|
Hostname: view.Hostname(),
|
|
DisplayName: displayName,
|
|
Platform: view.Platform(),
|
|
OSName: view.OSName(),
|
|
OSVersion: view.OSVersion(),
|
|
KernelVersion: view.KernelVersion(),
|
|
Architecture: view.Architecture(),
|
|
CPUCount: view.CPUCount(),
|
|
CPUUsage: view.CPUPercent(),
|
|
Memory: hostMemoryFromReadStateView(view),
|
|
LoadAverage: view.LoadAverage(),
|
|
Disks: hostDisksFromReadStateView(view.Disks()),
|
|
DiskIO: hostDiskIOFromReadStateView(view.DiskIO()),
|
|
NetworkInterfaces: hostNetworkInterfacesFromReadStateView(view.NetworkInterfaces()),
|
|
Sensors: hostSensorsFromReadStateView(view.Sensors()),
|
|
RAID: hostRAIDFromReadStateView(view.RAID()),
|
|
Unraid: hostUnraidFromReadStateView(view.Unraid()),
|
|
Ceph: hostCephFromReadStateView(view.Ceph()),
|
|
Status: status,
|
|
UptimeSeconds: view.UptimeSeconds(),
|
|
IntervalSeconds: view.IntervalSeconds(),
|
|
LastSeen: lastSeen,
|
|
AgentVersion: view.AgentVersion(),
|
|
IntegrationSource: view.IntegrationSource(),
|
|
MachineID: view.MachineID(),
|
|
CommandsEnabled: view.CommandsEnabled(),
|
|
ReportIP: view.ReportIP(),
|
|
TokenID: view.TokenID(),
|
|
TokenName: view.TokenName(),
|
|
TokenHint: view.TokenHint(),
|
|
TokenLastUsedAt: view.TokenLastUsedAt(),
|
|
Tags: view.Tags(),
|
|
DiskExclude: view.DiskExclude(),
|
|
IsLegacy: view.IsLegacy(),
|
|
NetInRate: view.NetInRate(),
|
|
NetOutRate: view.NetOutRate(),
|
|
DiskReadRate: view.DiskReadRate(),
|
|
DiskWriteRate: view.DiskWriteRate(),
|
|
LinkedNodeID: view.LinkedNodeID(),
|
|
LinkedVMID: view.LinkedVMID(),
|
|
LinkedContainerID: view.LinkedContainerID(),
|
|
}
|
|
}
|
|
|
|
// hostAgentLivenessFromReadStateView returns the status and last report of the
|
|
// agent behind a read-state host row. A row merged with a Proxmox node takes
|
|
// LastSeen from its freshest source and stays online while PVE polling
|
|
// continues, so a silent agent would look current: its retained disk summary
|
|
// and datasets kept feeding the node, and its connection read as active. A
|
|
// models.Host describes the agent, so read the agent source's own sighting
|
|
// and lease, as evaluateHostAgents does for the host in state.
|
|
func hostAgentLivenessFromReadStateView(view *unifiedresources.HostView, now time.Time) (string, time.Time) {
|
|
status, lastSeen := string(view.Status()), view.LastSeen()
|
|
agentSighting, ok := view.SourceStatus(unifiedresources.SourceAgent)
|
|
if !ok || agentSighting.LastSeen.IsZero() {
|
|
return status, lastSeen
|
|
}
|
|
if !hostAgentReportCurrent(agentSighting.LastSeen, view.IntervalSeconds(), now) {
|
|
status = "offline"
|
|
}
|
|
return status, agentSighting.LastSeen
|
|
}
|
|
|
|
func vmFromReadStateView(view *unifiedresources.VMView) models.VM {
|
|
if view == nil {
|
|
return models.VM{}
|
|
}
|
|
|
|
totalMemory := view.MemoryTotal()
|
|
usedMemory := view.MemoryUsed()
|
|
totalDisk := view.DiskTotal()
|
|
usedDisk := view.DiskUsed()
|
|
|
|
return models.VM{
|
|
ID: firstNonEmptyString(view.SourceID(), view.ID()),
|
|
VMID: view.VMID(),
|
|
Name: view.Name(),
|
|
Node: view.Node(),
|
|
Instance: view.Instance(),
|
|
Status: string(view.Status()),
|
|
Type: "qemu",
|
|
CPU: proxmoxRatioFromPercent(view.CPUPercent()),
|
|
CPUs: view.CPUs(),
|
|
Memory: models.Memory{Total: totalMemory, Used: usedMemory, Free: maxInt64(0, totalMemory-usedMemory), Usage: view.MemoryPercent()},
|
|
Disk: models.Disk{Used: usedDisk, Total: totalDisk, Free: maxInt64(0, totalDisk-usedDisk), Usage: view.DiskPercent()},
|
|
Disks: guestDisksFromReadStateView(view.Disks()),
|
|
DiskStatusReason: view.DiskStatusReason(),
|
|
IPAddresses: view.IPAddresses(),
|
|
OSName: view.OSName(),
|
|
OSVersion: view.OSVersion(),
|
|
AgentVersion: view.AgentVersion(),
|
|
NetworkInterfaces: guestNetworkInterfacesFromReadStateView(view.NetworkInterfaces()),
|
|
NetworkIn: maxInt64(0, int64(view.NetIn())),
|
|
NetworkOut: maxInt64(0, int64(view.NetOut())),
|
|
DiskRead: maxInt64(0, int64(view.DiskRead())),
|
|
DiskWrite: maxInt64(0, int64(view.DiskWrite())),
|
|
Uptime: view.Uptime(),
|
|
Template: view.Template(),
|
|
LastBackup: view.LastBackup(),
|
|
BackupInProgress: view.BackupInProgress(),
|
|
Tags: view.Tags(),
|
|
Lock: view.Lock(),
|
|
LastSeen: view.LastSeen(),
|
|
}
|
|
}
|
|
|
|
func containerFromReadStateView(view *unifiedresources.ContainerView) models.Container {
|
|
if view == nil {
|
|
return models.Container{}
|
|
}
|
|
|
|
totalMemory := view.MemoryTotal()
|
|
usedMemory := view.MemoryUsed()
|
|
totalDisk := view.DiskTotal()
|
|
usedDisk := view.DiskUsed()
|
|
|
|
return models.Container{
|
|
ID: firstNonEmptyString(view.SourceID(), view.ID()),
|
|
VMID: view.VMID(),
|
|
Name: view.Name(),
|
|
Node: view.Node(),
|
|
Instance: view.Instance(),
|
|
Status: string(view.Status()),
|
|
Type: firstNonEmptyString(view.ContainerType(), "lxc"),
|
|
CPU: proxmoxRatioFromPercent(view.CPUPercent()),
|
|
CPUs: view.CPUs(),
|
|
Memory: models.Memory{Total: totalMemory, Used: usedMemory, Free: maxInt64(0, totalMemory-usedMemory), Usage: view.MemoryPercent()},
|
|
Disk: models.Disk{Used: usedDisk, Total: totalDisk, Free: maxInt64(0, totalDisk-usedDisk), Usage: view.DiskPercent()},
|
|
Disks: guestDisksFromReadStateView(view.Disks()),
|
|
NetworkIn: maxInt64(0, int64(view.NetIn())),
|
|
NetworkOut: maxInt64(0, int64(view.NetOut())),
|
|
DiskRead: maxInt64(0, int64(view.DiskRead())),
|
|
DiskWrite: maxInt64(0, int64(view.DiskWrite())),
|
|
Uptime: view.Uptime(),
|
|
Template: view.Template(),
|
|
LastBackup: view.LastBackup(),
|
|
BackupInProgress: view.BackupInProgress(),
|
|
Tags: view.Tags(),
|
|
Lock: view.Lock(),
|
|
LastSeen: view.LastSeen(),
|
|
IPAddresses: view.IPAddresses(),
|
|
NetworkInterfaces: guestNetworkInterfacesFromReadStateView(view.NetworkInterfaces()),
|
|
OSName: view.OSName(),
|
|
IsOCI: view.IsOCI(),
|
|
OSTemplate: view.OSTemplate(),
|
|
HasDocker: view.HasDocker(),
|
|
DockerCheckedAt: view.DockerCheckedAt(),
|
|
}
|
|
}
|
|
|
|
func guestDisksFromReadStateView(disks []unifiedresources.DiskInfo) []models.Disk {
|
|
if len(disks) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.Disk, 0, len(disks))
|
|
for _, disk := range disks {
|
|
out = append(out, models.Disk{
|
|
Total: disk.Total,
|
|
Used: disk.Used,
|
|
Free: disk.Free,
|
|
Usage: disk.Usage,
|
|
Mountpoint: disk.Mountpoint,
|
|
Type: disk.Filesystem,
|
|
Device: disk.Device,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func guestNetworkInterfacesFromReadStateView(interfaces []unifiedresources.NetworkInterface) []models.GuestNetworkInterface {
|
|
if len(interfaces) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.GuestNetworkInterface, 0, len(interfaces))
|
|
for _, iface := range interfaces {
|
|
out = append(out, models.GuestNetworkInterface{
|
|
Name: iface.Name,
|
|
MAC: iface.MAC,
|
|
Addresses: append([]string(nil), iface.Addresses...),
|
|
RXBytes: maxInt64(0, int64(iface.RXBytes)),
|
|
TXBytes: maxInt64(0, int64(iface.TXBytes)),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func pbsInstanceFromReadStateView(view *unifiedresources.PBSInstanceView) models.PBSInstance {
|
|
if view == nil {
|
|
return models.PBSInstance{}
|
|
}
|
|
|
|
return models.PBSInstance{
|
|
ID: firstNonEmptyString(view.InstanceID(), view.ID()),
|
|
Name: view.Name(),
|
|
Host: view.HostURL(),
|
|
GuestURL: view.GuestURL(),
|
|
Status: string(view.Status()),
|
|
Version: view.Version(),
|
|
CPU: view.CPUPercent(),
|
|
Memory: view.MemoryPercent(),
|
|
MemoryUsed: view.MemoryUsed(),
|
|
MemoryTotal: view.MemoryTotal(),
|
|
Uptime: view.UptimeSeconds(),
|
|
Datastores: view.DatastoreDetails(),
|
|
BackupJobs: view.BackupJobs(),
|
|
SyncJobs: view.SyncJobs(),
|
|
VerifyJobs: view.VerifyJobs(),
|
|
PruneJobs: view.PruneJobs(),
|
|
GarbageJobs: view.GarbageJobs(),
|
|
ConnectionHealth: view.ConnectionHealth(),
|
|
LastSeen: view.LastSeen(),
|
|
}
|
|
}
|
|
|
|
func hostMemoryFromReadStateView(view *unifiedresources.HostView) models.Memory {
|
|
if view == nil {
|
|
return models.Memory{}
|
|
}
|
|
total := view.MemoryTotal()
|
|
used := view.MemoryUsed()
|
|
return models.Memory{
|
|
Total: total,
|
|
Used: used,
|
|
Free: maxInt64(0, total-used),
|
|
Usage: view.MemoryPercent(),
|
|
SwapUsed: view.SwapUsed(),
|
|
SwapTotal: view.SwapTotal(),
|
|
}
|
|
}
|
|
|
|
func hostDisksFromReadStateView(disks []unifiedresources.DiskInfo) []models.Disk {
|
|
if len(disks) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.Disk, 0, len(disks))
|
|
for _, disk := range disks {
|
|
out = append(out, models.Disk{
|
|
Total: disk.Total,
|
|
Used: disk.Used,
|
|
Free: disk.Free,
|
|
Usage: disk.Usage,
|
|
Mountpoint: disk.Mountpoint,
|
|
Type: disk.Filesystem,
|
|
Device: disk.Device,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostDiskIOFromReadStateView(diskIO []unifiedresources.HostDiskIOMeta) []models.DiskIO {
|
|
if len(diskIO) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.DiskIO, 0, len(diskIO))
|
|
for _, entry := range diskIO {
|
|
out = append(out, models.DiskIO{
|
|
Device: entry.Device,
|
|
ReadBytes: entry.ReadBytes,
|
|
WriteBytes: entry.WriteBytes,
|
|
ReadOps: entry.ReadOps,
|
|
WriteOps: entry.WriteOps,
|
|
IOTime: entry.IOTimeMs,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostNetworkInterfacesFromReadStateView(interfaces []unifiedresources.NetworkInterface) []models.HostNetworkInterface {
|
|
if len(interfaces) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.HostNetworkInterface, 0, len(interfaces))
|
|
for _, iface := range interfaces {
|
|
out = append(out, models.HostNetworkInterface{
|
|
Name: iface.Name,
|
|
MAC: iface.MAC,
|
|
Addresses: append([]string(nil), iface.Addresses...),
|
|
RXBytes: iface.RXBytes,
|
|
TXBytes: iface.TXBytes,
|
|
SpeedMbps: int64PtrCopy(iface.SpeedMbps),
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostSensorsFromReadStateView(sensors *unifiedresources.HostSensorMeta) models.HostSensorSummary {
|
|
if sensors == nil {
|
|
return models.HostSensorSummary{}
|
|
}
|
|
out := models.HostSensorSummary{}
|
|
if len(sensors.TemperatureCelsius) > 0 {
|
|
out.TemperatureCelsius = make(map[string]float64, len(sensors.TemperatureCelsius))
|
|
for k, v := range sensors.TemperatureCelsius {
|
|
out.TemperatureCelsius[k] = v
|
|
}
|
|
}
|
|
if len(sensors.FanRPM) > 0 {
|
|
out.FanRPM = make(map[string]float64, len(sensors.FanRPM))
|
|
for k, v := range sensors.FanRPM {
|
|
out.FanRPM[k] = v
|
|
}
|
|
}
|
|
if len(sensors.PowerWatts) > 0 {
|
|
out.PowerWatts = make(map[string]float64, len(sensors.PowerWatts))
|
|
for k, v := range sensors.PowerWatts {
|
|
out.PowerWatts[k] = v
|
|
}
|
|
}
|
|
if len(sensors.Additional) > 0 {
|
|
out.Additional = make(map[string]float64, len(sensors.Additional))
|
|
for k, v := range sensors.Additional {
|
|
out.Additional[k] = v
|
|
}
|
|
}
|
|
if len(sensors.Custom) > 0 {
|
|
out.Custom = make([]models.HostCustomSensorMetric, len(sensors.Custom))
|
|
for i, metric := range sensors.Custom {
|
|
out.Custom[i] = models.HostCustomSensorMetric{
|
|
ID: metric.ID,
|
|
Name: metric.Name,
|
|
Group: metric.Group,
|
|
Subgroup: metric.Subgroup,
|
|
Kind: metric.Kind,
|
|
Unit: metric.Unit,
|
|
Value: cloneFloat64Ptr(metric.Value),
|
|
Status: metric.Status,
|
|
ObservedAt: metric.ObservedAt,
|
|
EventAt: cloneReportTimePtr(metric.EventAt),
|
|
Error: metric.Error,
|
|
AlertOnError: metric.AlertOnError,
|
|
Stale: metric.Stale,
|
|
}
|
|
}
|
|
}
|
|
if len(sensors.GPU) > 0 {
|
|
out.GPU = make([]models.HostGPUSensor, len(sensors.GPU))
|
|
for i, gpu := range sensors.GPU {
|
|
out.GPU[i] = models.HostGPUSensor{
|
|
ID: gpu.ID,
|
|
Name: gpu.Name,
|
|
TemperatureCelsius: cloneFloat64Ptr(gpu.TemperatureCelsius),
|
|
UtilizationPercent: cloneFloat64Ptr(gpu.UtilizationPercent),
|
|
MemoryUsedBytes: cloneInt64Ptr(gpu.MemoryUsedBytes),
|
|
MemoryTotalBytes: cloneInt64Ptr(gpu.MemoryTotalBytes),
|
|
}
|
|
}
|
|
}
|
|
if sensors.ThermalState != nil {
|
|
out.ThermalState = hostThermalStateFromReadStateView(sensors.ThermalState)
|
|
}
|
|
if len(sensors.SMART) > 0 {
|
|
out.SMART = make([]models.HostDiskSMART, 0, len(sensors.SMART))
|
|
for _, smart := range sensors.SMART {
|
|
out.SMART = append(out.SMART, models.HostDiskSMART{
|
|
Device: smart.Device,
|
|
Model: smart.Model,
|
|
Serial: smart.Serial,
|
|
WWN: smart.WWN,
|
|
Type: smart.Type,
|
|
Controller: smart.Controller,
|
|
Target: smart.Target,
|
|
SizeBytes: smart.SizeBytes,
|
|
Temperature: smart.Temperature,
|
|
Health: smart.Health,
|
|
Standby: smart.Standby,
|
|
Pool: smart.Pool,
|
|
IO: physicalDiskIOFromUnifiedMeta(smart.IO),
|
|
Collection: diskinventory.CloneStatus(smart.Collection),
|
|
Attributes: smartAttributesCopy(smart.Attributes),
|
|
})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostThermalStateFromReadStateView(in *unifiedresources.HostThermalState) *models.HostThermalState {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
return &models.HostThermalState{
|
|
Source: in.Source,
|
|
Pressure: in.Pressure,
|
|
ThermalWarningLevel: cloneIntPtr(in.ThermalWarningLevel),
|
|
PerformanceWarningLevel: cloneIntPtr(in.PerformanceWarningLevel),
|
|
CPUPowerStatus: cloneIntPtr(in.CPUPowerStatus),
|
|
LimitsPercent: cloneStringIntMap(in.LimitsPercent),
|
|
}
|
|
}
|
|
|
|
func hostRAIDFromReadStateView(raid []unifiedresources.HostRAIDMeta) []models.HostRAIDArray {
|
|
if len(raid) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]models.HostRAIDArray, 0, len(raid))
|
|
for _, entry := range raid {
|
|
devices := make([]models.HostRAIDDevice, 0, len(entry.Devices))
|
|
for _, device := range entry.Devices {
|
|
devices = append(devices, models.HostRAIDDevice{
|
|
Device: device.Device,
|
|
State: device.State,
|
|
Slot: device.Slot,
|
|
})
|
|
}
|
|
out = append(out, models.HostRAIDArray{
|
|
Device: entry.Device,
|
|
Name: entry.Name,
|
|
Level: entry.Level,
|
|
State: entry.State,
|
|
RequiredDevices: entry.RequiredDevices,
|
|
TotalDevices: entry.TotalDevices,
|
|
ActiveDevices: entry.ActiveDevices,
|
|
WorkingDevices: entry.WorkingDevices,
|
|
FailedDevices: entry.FailedDevices,
|
|
SpareDevices: entry.SpareDevices,
|
|
UUID: entry.UUID,
|
|
Devices: devices,
|
|
RebuildPercent: entry.RebuildPercent,
|
|
RebuildSpeed: entry.RebuildSpeed,
|
|
Operation: entry.Operation,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostUnraidFromReadStateView(unraid *unifiedresources.HostUnraidMeta) *models.HostUnraidStorage {
|
|
if unraid == nil {
|
|
return nil
|
|
}
|
|
out := &models.HostUnraidStorage{
|
|
ArrayStarted: unraid.ArrayStarted,
|
|
ArrayState: unraid.ArrayState,
|
|
SyncAction: unraid.SyncAction,
|
|
SyncProgress: unraid.SyncProgress,
|
|
SyncErrors: unraid.SyncErrors,
|
|
NumDisks: unraid.NumDisks,
|
|
NumProtected: unraid.NumProtected,
|
|
NumDisabled: unraid.NumDisabled,
|
|
NumInvalid: unraid.NumInvalid,
|
|
NumMissing: unraid.NumMissing,
|
|
}
|
|
if len(unraid.Disks) > 0 {
|
|
out.Disks = make([]models.HostUnraidDisk, 0, len(unraid.Disks))
|
|
for _, disk := range unraid.Disks {
|
|
out.Disks = append(out.Disks, models.HostUnraidDisk{
|
|
Name: disk.Name,
|
|
Device: disk.Device,
|
|
Role: disk.Role,
|
|
Status: disk.Status,
|
|
RawStatus: disk.RawStatus,
|
|
Model: disk.Model,
|
|
Serial: disk.Serial,
|
|
Filesystem: disk.Filesystem,
|
|
Transport: disk.Transport,
|
|
SizeBytes: disk.SizeBytes,
|
|
UsedBytes: disk.UsedBytes,
|
|
FreeBytes: disk.FreeBytes,
|
|
Temperature: disk.Temperature,
|
|
SpunDown: disk.SpunDown,
|
|
ReadCount: disk.ReadCount,
|
|
WriteCount: disk.WriteCount,
|
|
ErrorCount: disk.ErrorCount,
|
|
Slot: disk.Slot,
|
|
})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func hostCephFromReadStateView(ceph *unifiedresources.HostCephMeta) *models.HostCephCluster {
|
|
if ceph == nil {
|
|
return nil
|
|
}
|
|
out := &models.HostCephCluster{
|
|
FSID: ceph.FSID,
|
|
Health: models.HostCephHealth{
|
|
Status: ceph.Health.Status,
|
|
},
|
|
MonMap: models.HostCephMonitorMap{
|
|
Epoch: ceph.MonMap.Epoch,
|
|
NumMons: ceph.MonMap.NumMons,
|
|
},
|
|
MgrMap: models.HostCephManagerMap{
|
|
Available: ceph.MgrMap.Available,
|
|
NumMgrs: ceph.MgrMap.NumMgrs,
|
|
ActiveMgr: ceph.MgrMap.ActiveMgr,
|
|
Standbys: ceph.MgrMap.Standbys,
|
|
},
|
|
OSDMap: models.HostCephOSDMap{
|
|
Epoch: ceph.OSDMap.Epoch,
|
|
NumOSDs: ceph.OSDMap.NumOSDs,
|
|
NumUp: ceph.OSDMap.NumUp,
|
|
NumIn: ceph.OSDMap.NumIn,
|
|
NumDown: ceph.OSDMap.NumDown,
|
|
NumOut: ceph.OSDMap.NumOut,
|
|
},
|
|
PGMap: models.HostCephPGMap{
|
|
NumPGs: ceph.PGMap.NumPGs,
|
|
BytesTotal: ceph.PGMap.BytesTotal,
|
|
BytesUsed: ceph.PGMap.BytesUsed,
|
|
BytesAvailable: ceph.PGMap.BytesAvailable,
|
|
DataBytes: ceph.PGMap.DataBytes,
|
|
UsagePercent: ceph.PGMap.UsagePercent,
|
|
DegradedRatio: ceph.PGMap.DegradedRatio,
|
|
MisplacedRatio: ceph.PGMap.MisplacedRatio,
|
|
ReadBytesPerSec: ceph.PGMap.ReadBytesPerSec,
|
|
WriteBytesPerSec: ceph.PGMap.WriteBytesPerSec,
|
|
ReadOpsPerSec: ceph.PGMap.ReadOpsPerSec,
|
|
WriteOpsPerSec: ceph.PGMap.WriteOpsPerSec,
|
|
},
|
|
CollectedAt: ceph.CollectedAt,
|
|
}
|
|
if len(ceph.Health.Summary) > 0 {
|
|
out.Health.Summary = make([]models.HostCephHealthSummary, 0, len(ceph.Health.Summary))
|
|
for _, summary := range ceph.Health.Summary {
|
|
out.Health.Summary = append(out.Health.Summary, models.HostCephHealthSummary{
|
|
Severity: summary.Severity,
|
|
Message: summary.Message,
|
|
})
|
|
}
|
|
}
|
|
if len(ceph.Health.Checks) > 0 {
|
|
out.Health.Checks = make(map[string]models.HostCephCheck, len(ceph.Health.Checks))
|
|
for name, check := range ceph.Health.Checks {
|
|
out.Health.Checks[name] = models.HostCephCheck{
|
|
Severity: check.Severity,
|
|
Message: check.Message,
|
|
Detail: append([]string(nil), check.Detail...),
|
|
}
|
|
}
|
|
}
|
|
if len(ceph.MonMap.Monitors) > 0 {
|
|
out.MonMap.Monitors = make([]models.HostCephMonitor, 0, len(ceph.MonMap.Monitors))
|
|
for _, monitor := range ceph.MonMap.Monitors {
|
|
out.MonMap.Monitors = append(out.MonMap.Monitors, models.HostCephMonitor{
|
|
Name: monitor.Name,
|
|
Rank: monitor.Rank,
|
|
Addr: monitor.Addr,
|
|
Status: monitor.Status,
|
|
})
|
|
}
|
|
}
|
|
if len(ceph.Pools) > 0 {
|
|
out.Pools = make([]models.HostCephPool, 0, len(ceph.Pools))
|
|
for _, pool := range ceph.Pools {
|
|
out.Pools = append(out.Pools, models.HostCephPool{
|
|
ID: pool.ID,
|
|
Name: pool.Name,
|
|
BytesUsed: pool.BytesUsed,
|
|
BytesAvailable: pool.BytesAvailable,
|
|
Objects: pool.Objects,
|
|
PercentUsed: pool.PercentUsed,
|
|
})
|
|
}
|
|
}
|
|
if len(ceph.Services) > 0 {
|
|
out.Services = make([]models.HostCephService, 0, len(ceph.Services))
|
|
for _, service := range ceph.Services {
|
|
out.Services = append(out.Services, models.HostCephService{
|
|
Type: service.Type,
|
|
Running: service.Running,
|
|
Total: service.Total,
|
|
Daemons: append([]string(nil), service.Daemons...),
|
|
})
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func dockerHostFromReadStateView(view *unifiedresources.DockerHostView) models.DockerHost {
|
|
if view == nil {
|
|
return models.DockerHost{}
|
|
}
|
|
|
|
totalMemory := view.TotalMemoryBytes()
|
|
if totalMemory == 0 {
|
|
totalMemory = view.MemoryTotal()
|
|
}
|
|
usedMemory := view.MemoryUsed()
|
|
freeMemory := maxInt64(0, totalMemory-usedMemory)
|
|
|
|
return models.DockerHost{
|
|
ID: firstNonEmptyString(view.HostSourceID(), view.ID()),
|
|
AgentID: view.AgentID(),
|
|
Hostname: view.Hostname(),
|
|
DisplayName: view.DisplayName(),
|
|
CustomDisplayName: view.CustomDisplayName(),
|
|
MachineID: view.MachineID(),
|
|
OS: view.OS(),
|
|
KernelVersion: view.KernelVersion(),
|
|
Architecture: view.Architecture(),
|
|
Runtime: view.Runtime(),
|
|
RuntimeVersion: view.RuntimeVersion(),
|
|
DockerVersion: view.DockerVersion(),
|
|
CollectionMode: view.CollectionMode(),
|
|
CPUs: view.CPUs(),
|
|
TotalMemoryBytes: totalMemory,
|
|
UptimeSeconds: view.UptimeSeconds(),
|
|
CPUUsage: view.CPUPercent(),
|
|
LoadAverage: view.LoadAverage(),
|
|
Memory: models.Memory{
|
|
Total: totalMemory,
|
|
Used: usedMemory,
|
|
Free: freeMemory,
|
|
Usage: view.MemoryPercent(),
|
|
},
|
|
Disks: hostDisksFromReadStateView(view.Disks()),
|
|
NetworkInterfaces: hostNetworkInterfacesFromReadStateView(view.NetworkInterfaces()),
|
|
Status: string(view.Status()),
|
|
LastSeen: view.LastSeen(),
|
|
IntervalSeconds: view.IntervalSeconds(),
|
|
AgentVersion: view.AgentVersion(),
|
|
Containers: view.Containers(),
|
|
Services: view.Services(),
|
|
Tasks: view.Tasks(),
|
|
Swarm: dockerSwarmFromReadStateView(view.Swarm()),
|
|
TokenID: view.TokenID(),
|
|
TokenName: view.TokenName(),
|
|
TokenHint: view.TokenHint(),
|
|
TokenLastUsedAt: view.TokenLastUsedAt(),
|
|
Hidden: view.Hidden(),
|
|
PendingUninstall: view.PendingUninstall(),
|
|
Command: view.Command(),
|
|
IsLegacy: view.IsLegacy(),
|
|
IdentityConflict: view.IdentityConflict(),
|
|
NetInRate: view.NetInRate(),
|
|
NetOutRate: view.NetOutRate(),
|
|
DiskReadRate: view.DiskReadRate(),
|
|
DiskWriteRate: view.DiskWriteRate(),
|
|
}
|
|
}
|
|
|
|
func dockerSwarmFromReadStateView(in *unifiedresources.DockerSwarmInfo) *models.DockerSwarmInfo {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
return &models.DockerSwarmInfo{
|
|
NodeID: in.NodeID,
|
|
NodeRole: in.NodeRole,
|
|
LocalState: in.LocalState,
|
|
ControlAvailable: in.ControlAvailable,
|
|
ClusterID: in.ClusterID,
|
|
ClusterName: in.ClusterName,
|
|
Scope: in.Scope,
|
|
Error: in.Error,
|
|
}
|
|
}
|
|
|
|
func maxInt64(a, b int64) int64 {
|
|
if a > b {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func int64PtrCopy(in *int64) *int64 {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
out := *in
|
|
return &out
|
|
}
|
|
|
|
func smartAttributesCopy(in *models.SMARTAttributes) *models.SMARTAttributes {
|
|
if in == nil {
|
|
return nil
|
|
}
|
|
out := *in
|
|
out.PowerOnHours = int64PtrCopy(in.PowerOnHours)
|
|
out.PowerCycles = int64PtrCopy(in.PowerCycles)
|
|
out.ReallocatedSectors = int64PtrCopy(in.ReallocatedSectors)
|
|
out.PendingSectors = int64PtrCopy(in.PendingSectors)
|
|
out.OfflineUncorrectable = int64PtrCopy(in.OfflineUncorrectable)
|
|
out.UDMACRCErrors = int64PtrCopy(in.UDMACRCErrors)
|
|
if in.PercentageUsed != nil {
|
|
value := *in.PercentageUsed
|
|
out.PercentageUsed = &value
|
|
}
|
|
if in.AvailableSpare != nil {
|
|
value := *in.AvailableSpare
|
|
out.AvailableSpare = &value
|
|
}
|
|
out.MediaErrors = int64PtrCopy(in.MediaErrors)
|
|
out.UnsafeShutdowns = int64PtrCopy(in.UnsafeShutdowns)
|
|
return &out
|
|
}
|
|
|
|
func firstNonEmptyString(values ...string) string {
|
|
for _, value := range values {
|
|
if trimmed := strings.TrimSpace(value); trimmed != "" {
|
|
return trimmed
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func storageZFSPoolFromReadStateView(view *unifiedresources.StoragePoolView) *models.ZFSPool {
|
|
if view == nil {
|
|
return nil
|
|
}
|
|
|
|
state := strings.TrimSpace(view.ZFSPoolState())
|
|
if !view.IsZFS() && state == "" && view.ZFSReadErrors() == 0 && view.ZFSWriteErrors() == 0 && view.ZFSChecksumErrors() == 0 {
|
|
return nil
|
|
}
|
|
|
|
return &models.ZFSPool{
|
|
Name: view.Name(),
|
|
State: state,
|
|
ReadErrors: view.ZFSReadErrors(),
|
|
WriteErrors: view.ZFSWriteErrors(),
|
|
ChecksumErrors: view.ZFSChecksumErrors(),
|
|
}
|
|
}
|
|
|
|
func storageNodeIDsFromReadState(instance string, nodes []string) []string {
|
|
if len(nodes) == 0 {
|
|
return nil
|
|
}
|
|
|
|
nodeIDs := make([]string, 0, len(nodes))
|
|
for _, node := range nodes {
|
|
node = strings.TrimSpace(node)
|
|
if node == "" {
|
|
continue
|
|
}
|
|
if instance == "" {
|
|
nodeIDs = append(nodeIDs, node)
|
|
continue
|
|
}
|
|
nodeIDs = append(nodeIDs, instance+"-"+node)
|
|
}
|
|
if len(nodeIDs) == 0 {
|
|
return nil
|
|
}
|
|
return nodeIDs
|
|
}
|
|
|
|
// ActiveAlertsSnapshot returns the current active alerts.
|
|
func (m *Monitor) ActiveAlertsSnapshot() []models.Alert {
|
|
return m.activeAlertsSnapshot()
|
|
}
|
|
|
|
// RecentlyResolvedSnapshot returns the recently resolved alerts.
|
|
func (m *Monitor) RecentlyResolvedSnapshot() []models.ResolvedAlert {
|
|
return m.recentlyResolvedAlertsSnapshot()
|
|
}
|
|
|
|
// PVEBackupsSnapshot returns the current PVE backups.
|
|
func (m *Monitor) PVEBackupsSnapshot() models.PVEBackups {
|
|
return m.GetState().PVEBackups
|
|
}
|
|
|
|
// PBSBackupsSnapshot returns the current PBS recovery artifacts.
|
|
func (m *Monitor) PBSBackupsSnapshot() []models.PBSBackup {
|
|
return m.GetState().PBSBackups
|
|
}
|
|
|
|
// BuildFrontendState returns the current state converted to frontend format.
|
|
// This replaces the GetState().ToFrontend() pattern in consumer code.
|
|
func (m *Monitor) BuildFrontendState() models.StateFrontend {
|
|
return m.buildBroadcastFrontendStateFromSnapshot(m.currentStateWithScope())
|
|
}
|
|
|
|
// BuildBroadcastFrontendState returns frontend state ready for websocket
|
|
// broadcasts, including the unified resource payload when a resource store is
|
|
// configured.
|
|
func (m *Monitor) BuildBroadcastFrontendState() models.StateFrontend {
|
|
return m.buildBroadcastFrontendStateFromSnapshot(m.currentStateWithScope())
|
|
}
|
|
|
|
func buildFrontendStateFromSnapshot(snapshot models.StateSnapshot) models.StateFrontend {
|
|
return snapshot.ToFrontend()
|
|
}
|
|
|
|
func (m *Monitor) buildBroadcastFrontendStateFromSnapshot(snapshot models.StateSnapshot, scope mockModeScope) models.StateFrontend {
|
|
return m.buildBroadcastFrontendStateFromSnapshotWithClock(snapshot, scope, time.Now)
|
|
}
|
|
|
|
// The clock is sampled at the same post-read health-evaluation boundary as
|
|
// ordinary broadcasts. Tests can compare complete projections at one instant
|
|
// without mistaking a naturally advancing health-age label for lost content.
|
|
func (m *Monitor) buildBroadcastFrontendStateFromSnapshotWithClock(snapshot models.StateSnapshot, scope mockModeScope, healthClock func() time.Time) models.StateFrontend {
|
|
frontendState := buildFrontendStateFromSnapshot(snapshot)
|
|
m.updateResourceStoreForRead(snapshot, scope)
|
|
if m != nil && m.alertManager != nil {
|
|
if liveAlerts := m.activeAlertsSnapshot(); len(liveAlerts) > 0 || len(frontendState.ActiveAlerts) > 0 {
|
|
frontendState.ActiveAlerts = liveAlerts
|
|
}
|
|
}
|
|
unifiedView := m.currentUnifiedStateView()
|
|
metricsTargetResolver := unifiedView.metricsTargets
|
|
if metricsTargetResolver == nil {
|
|
metricsTargetResolver = broadcastMetricsTargetResolver(unifiedView.readState)
|
|
}
|
|
broadcastResources := unifiedresources.CoalescePresentationHostResources(unifiedView.resources)
|
|
// Coalescing owns the outer slice. Decorate that one projection in place,
|
|
// not three full-resource copies; nested store data is still read-only.
|
|
healthAlerts := resourceHealthAlerts(frontendState.ActiveAlerts)
|
|
now := healthClock().UTC()
|
|
for i := range broadcastResources {
|
|
m.applyPersistedMetadataToUnifiedResource(&broadcastResources[i])
|
|
health := unifiedresources.EvaluateResourceHealth(broadcastResources[i], healthAlerts, now)
|
|
broadcastResources[i].Health = &health
|
|
}
|
|
attachBroadcastMetricsTargetsInPlace(broadcastResources, metricsTargetResolver)
|
|
broadcastFrontendResources, broadcastCatalogs := convertPresentationResourcesForBroadcast(broadcastResources)
|
|
frontendState.Resources = broadcastFrontendResources
|
|
frontendState.CapabilityCatalog = broadcastCatalogs.capabilities
|
|
frontendState.PolicyCatalog = broadcastCatalogs.policies
|
|
frontendState.AISafeSummaryCatalog = broadcastCatalogs.aiSafeSummaries
|
|
frontendState.ConnectedInfrastructure = buildConnectedInfrastructure(broadcastResources, snapshot)
|
|
if !unifiedView.freshness.IsZero() {
|
|
frontendState.LastUpdate = unifiedView.freshness.UnixMilli()
|
|
}
|
|
return frontendState
|
|
}
|
|
|
|
// GetLiveStateSnapshot returns the underlying monitor state snapshot without
|
|
// applying global mock mode overrides.
|
|
//
|
|
// This is useful for agent management endpoints that need to reflect actual
|
|
// registrations even when mock mode is enabled for the UI/demo experience.
|
|
func (m *Monitor) GetLiveStateSnapshot() models.StateSnapshot {
|
|
if m == nil || m.state == nil {
|
|
return models.EmptyStateSnapshot()
|
|
}
|
|
return m.state.GetSnapshot()
|
|
}
|
|
|
|
// GetLiveHostsSnapshot returns the underlying registered host agents without
|
|
// applying global mock mode overrides.
|
|
func (m *Monitor) GetLiveHostsSnapshot() []models.Host {
|
|
if m == nil || m.state == nil {
|
|
return nil
|
|
}
|
|
// Copy only the hosts. GetSnapshot deep-copies every guest as well, and
|
|
// this runs on each agent report, config fetch, and continuity lookup.
|
|
return m.state.GetHosts()
|
|
}
|
|
|
|
// SetOrgID sets the organization ID for this monitor instance.
|
|
// This is used for tenant isolation in multi-tenant deployments.
|
|
func (m *Monitor) SetOrgID(orgID string) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
m.orgID = strings.TrimSpace(orgID)
|
|
}
|
|
|
|
// GetOrgID returns the organization ID for this monitor instance.
|
|
// Returns empty string for default/legacy monitors.
|
|
func (m *Monitor) GetOrgID() string {
|
|
m.mu.RLock()
|
|
defer m.mu.RUnlock()
|
|
return m.orgID
|
|
}
|
|
|
|
type stateBroadcaster interface {
|
|
BroadcastState(state interface{})
|
|
BroadcastStateToTenant(orgID string, state interface{})
|
|
}
|
|
|
|
type currentStateBroadcaster interface {
|
|
BroadcastCurrentState()
|
|
BroadcastCurrentStateToTenant(orgID string)
|
|
}
|
|
|
|
type stateSubscriberCounter interface {
|
|
GetClientCount() int
|
|
GetTenantClientCount(orgID string) int
|
|
}
|
|
|
|
func stateBroadcasterHasSubscribers(hub stateBroadcaster, orgID string) bool {
|
|
if isNilStateBroadcaster(hub) {
|
|
return false
|
|
}
|
|
|
|
counter, ok := hub.(stateSubscriberCounter)
|
|
if !ok {
|
|
return true
|
|
}
|
|
|
|
orgID = strings.TrimSpace(orgID)
|
|
if orgID != "" {
|
|
return counter.GetTenantClientCount(orgID) > 0
|
|
}
|
|
return counter.GetClientCount() > 0
|
|
}
|
|
|
|
func currentStateBroadcasterHasSubscribers(hub currentStateBroadcaster, orgID string) bool {
|
|
if isNilCurrentStateBroadcaster(hub) {
|
|
return false
|
|
}
|
|
|
|
counter, ok := hub.(stateSubscriberCounter)
|
|
if !ok {
|
|
return true
|
|
}
|
|
|
|
orgID = strings.TrimSpace(orgID)
|
|
if orgID != "" {
|
|
return counter.GetTenantClientCount(orgID) > 0
|
|
}
|
|
return counter.GetClientCount() > 0
|
|
}
|
|
|
|
// broadcastState broadcasts state to WebSocket clients.
|
|
// Monitors with an explicit org ID (including "default") are tenant-scoped.
|
|
// Legacy monitors without an org ID broadcast globally.
|
|
func (m *Monitor) broadcastState(hub stateBroadcaster, frontendState interface{}) {
|
|
if isNilStateBroadcaster(hub) {
|
|
return
|
|
}
|
|
|
|
orgID := strings.TrimSpace(m.GetOrgID())
|
|
if !stateBroadcasterHasSubscribers(hub, orgID) {
|
|
return
|
|
}
|
|
if orgID != "" {
|
|
hub.BroadcastStateToTenant(orgID, frontendState)
|
|
} else {
|
|
hub.BroadcastState(frontendState)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) broadcastCurrentState(hub currentStateBroadcaster) {
|
|
if isNilCurrentStateBroadcaster(hub) {
|
|
return
|
|
}
|
|
|
|
orgID := strings.TrimSpace(m.GetOrgID())
|
|
if !currentStateBroadcasterHasSubscribers(hub, orgID) {
|
|
return
|
|
}
|
|
if orgID != "" {
|
|
hub.BroadcastCurrentStateToTenant(orgID)
|
|
} else {
|
|
hub.BroadcastCurrentState()
|
|
}
|
|
}
|
|
|
|
func isNilStateBroadcaster(hub stateBroadcaster) bool {
|
|
if hub == nil {
|
|
return true
|
|
}
|
|
|
|
value := reflect.ValueOf(hub)
|
|
switch value.Kind() {
|
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
|
|
return value.IsNil()
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func isNilCurrentStateBroadcaster(hub currentStateBroadcaster) bool {
|
|
if hub == nil {
|
|
return true
|
|
}
|
|
|
|
value := reflect.ValueOf(hub)
|
|
switch value.Kind() {
|
|
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
|
|
return value.IsNil()
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) broadcastEscalatedAlert(hub *websocket.Hub, alert *alerts.Alert) {
|
|
if hub == nil || alert == nil {
|
|
return
|
|
}
|
|
|
|
hub.BroadcastAlertToTenant(m.GetOrgID(), alert)
|
|
}
|
|
|
|
// mockModeSwitchMu serializes mock-mode switches. The mode is process-wide,
|
|
// and a switch is several steps (flip, end the epoch, clear, reset state), so
|
|
// two interleaved switches could clear what the later one just admitted.
|
|
var mockModeSwitchMu sync.Mutex
|
|
|
|
// SetMockMode switches between mock data and real infrastructure data at runtime.
|
|
// It must not be called from inside an alert evaluation, whose completion it
|
|
// may wait for.
|
|
func (m *Monitor) SetMockMode(enable bool) error {
|
|
mockModeSwitchMu.Lock()
|
|
defer mockModeSwitchMu.Unlock()
|
|
|
|
current := mock.IsMockEnabled()
|
|
if current == enable {
|
|
log.Info().Bool("mockMode", enable).Msg("mock mode already in desired state")
|
|
return nil
|
|
}
|
|
|
|
// Every evaluation of mode-dependent data that started before the flip
|
|
// must finish or be refused before the clear, or it reopens alerts for
|
|
// the side the monitor just left (see mockModeFence).
|
|
if enable {
|
|
m.stopMockMetricsSampler()
|
|
if err := mock.SetEnabled(true); err != nil {
|
|
return err
|
|
}
|
|
m.mockModeFence.advance()
|
|
m.alertManager.ClearActiveAlerts()
|
|
m.mu.Lock()
|
|
m.resetStateLocked()
|
|
m.metricsHistory.Reset()
|
|
m.mu.Unlock()
|
|
m.StopDiscoveryService()
|
|
m.mu.RLock()
|
|
ctx := m.runtimeCtx
|
|
m.mu.RUnlock()
|
|
if ctx != nil {
|
|
m.startMockMetricsSampler(ctx)
|
|
}
|
|
log.Info().Msg("switched monitor to mock mode")
|
|
} else {
|
|
m.stopMockMetricsSampler()
|
|
if err := mock.SetEnabled(false); err != nil {
|
|
return err
|
|
}
|
|
m.mockModeFence.advance()
|
|
m.alertManager.ClearActiveAlerts()
|
|
m.forgetMockHostAgents()
|
|
m.mu.Lock()
|
|
m.resetStateLocked()
|
|
m.metricsHistory.Reset()
|
|
m.mu.Unlock()
|
|
log.Info().Msg("switched monitor to real data mode")
|
|
}
|
|
|
|
m.mu.RLock()
|
|
ctx := m.runtimeCtx
|
|
hub := m.wsHub
|
|
m.mu.RUnlock()
|
|
|
|
if hub != nil {
|
|
m.broadcastCurrentState(hub)
|
|
}
|
|
|
|
if enable && ctx != nil && keepRealPollingInMockMode() {
|
|
// Keep real metrics flowing while mock mode is enabled.
|
|
go m.poll(ctx, hub)
|
|
}
|
|
|
|
if !enable && ctx != nil {
|
|
// Kick off an immediate poll to repopulate state with live data.
|
|
go m.poll(ctx, hub)
|
|
if hub != nil && m.config.DiscoveryEnabled {
|
|
go m.StartDiscoveryService(ctx, hub, m.config.DiscoverySubnet)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (m *Monitor) resetStateLocked() {
|
|
m.state = models.NewState()
|
|
m.state.Stats = models.Stats{
|
|
StartTime: m.startTime,
|
|
Version: "2.0.0-go",
|
|
}
|
|
}
|
|
|
|
// GetStartTime returns the monitor start time
|
|
func (m *Monitor) GetStartTime() time.Time {
|
|
return m.startTime
|
|
}
|
|
|
|
// GetDiscoveryService returns the discovery service
|
|
func (m *Monitor) GetDiscoveryService() *discovery.Service {
|
|
return m.discoveryService
|
|
}
|
|
|
|
// StartDiscoveryService starts the discovery service if not already running
|
|
func (m *Monitor) StartDiscoveryService(ctx context.Context, wsHub *websocket.Hub, subnet string) {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if m.discoveryService != nil {
|
|
log.Debug().Msg("discovery service already running")
|
|
return
|
|
}
|
|
|
|
if subnet == "" {
|
|
subnet = "auto"
|
|
}
|
|
|
|
cfgProvider := func() config.DiscoveryConfig {
|
|
return m.discoveryConfigSnapshot()
|
|
}
|
|
|
|
m.discoveryService = discovery.NewService(wsHub, 5*time.Minute, subnet, cfgProvider)
|
|
if m.discoveryService != nil {
|
|
m.discoveryService.Start(ctx)
|
|
log.Info().Str("subnet", subnet).Msg("discovery service started")
|
|
} else {
|
|
log.Error().Msg("failed to create discovery service")
|
|
}
|
|
}
|
|
|
|
// StopDiscoveryService stops the discovery service if running
|
|
func (m *Monitor) StopDiscoveryService() {
|
|
m.mu.Lock()
|
|
defer m.mu.Unlock()
|
|
|
|
if m.discoveryService != nil {
|
|
m.discoveryService.Stop()
|
|
m.discoveryService = nil
|
|
log.Info().Msg("discovery service stopped")
|
|
}
|
|
}
|
|
|
|
// EnableTemperatureMonitoring enables temperature data collection
|
|
func (m *Monitor) EnableTemperatureMonitoring() {
|
|
// Temperature collection is always enabled when tempCollector is initialized
|
|
// This method exists for interface compatibility
|
|
log.Info().Msg("temperature monitoring enabled")
|
|
}
|
|
|
|
// DisableTemperatureMonitoring disables temperature data collection
|
|
func (m *Monitor) DisableTemperatureMonitoring() {
|
|
// Temperature collection is always enabled when tempCollector is initialized
|
|
// This method exists for interface compatibility
|
|
log.Info().Msg("temperature monitoring disabled")
|
|
}
|
|
|
|
// SetResourceStore sets the resource store for polling optimization.
|
|
// When set, the monitor will check if it should reduce polling frequency
|
|
// for nodes that have host agents providing data.
|
|
func (m *Monitor) SetResourceStore(store ResourceStoreInterface) {
|
|
if thresholdStore, ok := store.(StaleThresholdResourceStore); ok {
|
|
thresholdStore.SetStaleThresholds(m.resourceStaleThresholds())
|
|
}
|
|
|
|
m.mu.Lock()
|
|
m.resourceStore = store
|
|
incidentStore := m.incidentStore
|
|
m.mu.Unlock()
|
|
m.installOperatorIntentResolver(store)
|
|
log.Info().Msg("resource store set for polling optimization")
|
|
|
|
timelineAttached := false
|
|
if incidentStore != nil {
|
|
if timelineStore, ok := store.(memory.IncidentTimelineStore); ok {
|
|
incidentStore.SetResourceTimelineStore(timelineStore)
|
|
timelineAttached = true
|
|
} else {
|
|
incidentStore.SetResourceTimelineStore(nil)
|
|
}
|
|
}
|
|
if timelineAttached {
|
|
// The canonical boundary: with the durable resource store attached,
|
|
// restart repair can reach resource history as well as the incident
|
|
// fallback cache, and the replay watermark may advance. Run it in the
|
|
// background — router construction and health serving must not wait
|
|
// behind an un-projected backlog.
|
|
m.scheduleAlertProjectionCatchUp()
|
|
}
|
|
|
|
// Immediately backfill the store from current state so ReadState
|
|
// consumers have data as soon as the store is wired.
|
|
// Guard against minimally initialized monitors (e.g., test fixtures
|
|
// with bare &Monitor{}) where m.state may be nil.
|
|
if store != nil && m.state != nil {
|
|
m.updateResourceStore(m.currentStateWithScope())
|
|
}
|
|
}
|
|
|
|
// SetSupplementalRecordsProvider configures source-native resource providers
|
|
// that ingest alongside the legacy state snapshot path.
|
|
func (m *Monitor) SetSupplementalRecordsProvider(source unifiedresources.DataSource, provider MonitorSupplementalRecordsProvider) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
|
|
normalized := unifiedresources.DataSource(strings.ToLower(strings.TrimSpace(string(source))))
|
|
if normalized == "" {
|
|
return
|
|
}
|
|
|
|
m.mu.Lock()
|
|
if m.supplementalProviders == nil {
|
|
m.supplementalProviders = make(map[unifiedresources.DataSource]MonitorSupplementalRecordsProvider)
|
|
}
|
|
if provider == nil {
|
|
delete(m.supplementalProviders, normalized)
|
|
} else {
|
|
m.supplementalProviders[normalized] = provider
|
|
}
|
|
m.mu.Unlock()
|
|
|
|
m.updateResourceStore(m.currentStateWithScope())
|
|
}
|
|
|
|
// SetLicenseChecker wires the commercial feature gate used by monitoring-owned
|
|
// Pro behaviour. It mirrors the alert manager's checker so entitlement lookups
|
|
// stay request-free inside the polling loop.
|
|
func (m *Monitor) SetLicenseChecker(checker func(feature string) bool) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.licenseCheckerMu.Lock()
|
|
m.licenseChecker = checker
|
|
m.licenseCheckerMu.Unlock()
|
|
}
|
|
|
|
func (m *Monitor) hasLicensedFeature(feature string) bool {
|
|
if m == nil {
|
|
return false
|
|
}
|
|
m.licenseCheckerMu.RLock()
|
|
checker := m.licenseChecker
|
|
m.licenseCheckerMu.RUnlock()
|
|
if checker == nil {
|
|
return false
|
|
}
|
|
return checker(feature)
|
|
}
|
|
|
|
// SetRecoveryManager wires the recovery store manager for best-effort ingestion of
|
|
// recovery points derived from polled backup/snapshot data.
|
|
func (m *Monitor) SetRecoveryManager(manager *recoverymanager.Manager) {
|
|
m.mu.Lock()
|
|
m.recoveryManager = manager
|
|
m.mu.Unlock()
|
|
|
|
// Try cleanup during wiring so monitors that are already running still get
|
|
// the migration once a recovery manager becomes available.
|
|
go m.purgeStalePVEPBSBackupsBestEffort(context.Background())
|
|
}
|
|
|
|
// GetNotificationManager returns the notification manager
|
|
func (m *Monitor) GetNotificationManager() *notifications.NotificationManager {
|
|
return m.notificationMgr
|
|
}
|
|
|
|
// GetConfigPersistence returns the config persistence manager
|
|
func (m *Monitor) GetConfigPersistence() *config.ConfigPersistence {
|
|
return m.configPersist
|
|
}
|
|
|
|
// GetMetricsStore returns the persistent metrics store
|
|
func (m *Monitor) GetMetricsStore() *metrics.Store {
|
|
return m.metricsStore
|
|
}
|
|
|
|
// GetMetricsHistory returns the in-memory metrics history for trend analysis
|
|
// This is used by the AI context builder to compute trends and predictions
|
|
func (m *Monitor) GetMetricsHistory() *MetricsHistory {
|
|
return m.metricsHistory
|
|
}
|
|
|
|
// GetUnifiedResources returns the current unified resource view for this monitor.
|
|
// Returns nil when no resource store is configured.
|
|
func (m *Monitor) GetUnifiedResources() []unifiedresources.Resource {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
if store == nil {
|
|
return nil
|
|
}
|
|
|
|
return store.GetAll()
|
|
}
|
|
|
|
// MetricsTargetForResource resolves the metrics-store target for a canonical
|
|
// unified resource ID. Resolution goes through the canonical unified view
|
|
// (the same re-ingested registry the UI and /api/state read) because
|
|
// callers hold IDs from that view: the raw resource store's canonical IDs
|
|
// depend on per-boot ingest order for merged-source hosts, so the view and
|
|
// the store can disagree about the same machine's ID. The raw store stays
|
|
// as a fallback. Returns nil when nothing can resolve a target.
|
|
func (m *Monitor) MetricsTargetForResource(resourceID string) *unifiedresources.MetricsTarget {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
|
|
if view := m.GetUnifiedReadStateOrSnapshot(); view != nil {
|
|
if resolver, ok := view.(MetricsTargetResourceStore); ok {
|
|
if target := resolver.MetricsTargetForResource(resourceID); target != nil {
|
|
return target
|
|
}
|
|
}
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
if store == nil {
|
|
return nil
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return resolver.MetricsTargetForResource(resourceID)
|
|
}
|
|
|
|
// resourceChangeTimeline is the slice of the resource store needed to read
|
|
// the recorded state timeline for a canonical resource ID.
|
|
type resourceChangeTimeline interface {
|
|
GetRecentChanges(canonicalID string, since time.Time, limit int) ([]unifiedresources.ResourceChange, error)
|
|
}
|
|
|
|
// RecentResourceChanges returns the recorded change timeline for a canonical
|
|
// unified resource ID since the given time, newest first (store order).
|
|
// Returns nil when no resource store is wired or it does not record changes.
|
|
func (m *Monitor) RecentResourceChanges(resourceID string, since time.Time, limit int) []unifiedresources.ResourceChange {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
if store == nil {
|
|
return nil
|
|
}
|
|
|
|
timeline, ok := store.(resourceChangeTimeline)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
changes, err := timeline.GetRecentChanges(resourceID, since, limit)
|
|
if err != nil {
|
|
log.Warn().Err(err).Str("resourceID", resourceID).Msg("failed to read resource change timeline")
|
|
return nil
|
|
}
|
|
return changes
|
|
}
|
|
|
|
type monitorUnifiedStateView struct {
|
|
resources []unifiedresources.Resource
|
|
readState unifiedresources.ReadState
|
|
freshness time.Time
|
|
metricsTargets MetricsTargetResourceStore
|
|
}
|
|
|
|
type unifiedResourceReadStateLister interface {
|
|
GetAll() []unifiedresources.Resource
|
|
}
|
|
|
|
type unifiedResourceProjectionLister interface {
|
|
GetAllWithMetricsTargets() ([]unifiedresources.Resource, map[string]unifiedresources.MetricsTarget)
|
|
}
|
|
|
|
// This resolver never falls through to the live store, including for an
|
|
// unknown ID. A missing target belongs to this captured generation too.
|
|
type projectionMetricsTargets map[string]unifiedresources.MetricsTarget
|
|
|
|
func (targets projectionMetricsTargets) MetricsTargetForResource(id string) *unifiedresources.MetricsTarget {
|
|
target, ok := targets[unifiedresources.CanonicalResourceID(id)]
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return &target
|
|
}
|
|
|
|
func unifiedProjectionResources(lister unifiedResourceReadStateLister) ([]unifiedresources.Resource, MetricsTargetResourceStore) {
|
|
if projection, ok := lister.(unifiedResourceProjectionLister); ok {
|
|
resources, targets := projection.GetAllWithMetricsTargets()
|
|
return resources, projectionMetricsTargets(targets)
|
|
}
|
|
return lister.GetAll(), nil
|
|
}
|
|
|
|
func monitorUnifiedStateViewFromSnapshot(snapshot models.StateSnapshot) monitorUnifiedStateView {
|
|
registry := unifiedresources.NewRegistry(nil)
|
|
registry.IngestSnapshot(snapshot)
|
|
adapter := unifiedresources.NewMonitorAdapter(registry)
|
|
resources, targets := unifiedProjectionResources(adapter)
|
|
return monitorUnifiedStateView{
|
|
resources: resources,
|
|
readState: adapter,
|
|
freshness: snapshot.LastUpdate,
|
|
metricsTargets: targets,
|
|
}
|
|
}
|
|
|
|
func monitorUnifiedStateViewFromResources(resources []unifiedresources.Resource, freshness time.Time) monitorUnifiedStateView {
|
|
registry := unifiedresources.NewRegistry(nil)
|
|
registry.IngestResources(resources)
|
|
adapter := unifiedresources.NewMonitorAdapter(registry)
|
|
resources, targets := unifiedProjectionResources(adapter)
|
|
return monitorUnifiedStateView{
|
|
resources: resources,
|
|
readState: adapter,
|
|
freshness: freshness,
|
|
metricsTargets: targets,
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) unifiedStateViewWithStandaloneHostContinuity(view monitorUnifiedStateView) monitorUnifiedStateView {
|
|
if m == nil || view.readState == nil {
|
|
return view
|
|
}
|
|
|
|
readState := m.readStateWithStandaloneHostContinuity(view.readState)
|
|
view.readState = readState
|
|
|
|
lister, ok := readState.(unifiedResourceReadStateLister)
|
|
if !ok {
|
|
return view
|
|
}
|
|
|
|
resources, targets := unifiedProjectionResources(lister)
|
|
view.resources = resources
|
|
view.metricsTargets = targets
|
|
if view.freshness.IsZero() {
|
|
view.freshness = latestUnifiedResourceLastSeen(resources)
|
|
}
|
|
return view
|
|
}
|
|
|
|
func latestUnifiedResourceLastSeen(resources []unifiedresources.Resource) time.Time {
|
|
var latest time.Time
|
|
for _, resource := range resources {
|
|
if resource.LastSeen.After(latest) {
|
|
latest = resource.LastSeen
|
|
}
|
|
}
|
|
return latest
|
|
}
|
|
|
|
func (m *Monitor) currentUnifiedStateView() monitorUnifiedStateView {
|
|
if m == nil {
|
|
return monitorUnifiedStateView{}
|
|
}
|
|
|
|
if mock.IsMockEnabled() {
|
|
// Read the version before the snapshot so a tick landing in between
|
|
// caches newer data under an older token (harmless rebuild next
|
|
// call) rather than ever serving stale data under a newer one.
|
|
version := mock.FixtureDataVersion()
|
|
m.mockUnifiedViewMu.Lock()
|
|
if m.mockUnifiedViewValid && m.mockUnifiedViewVersion == version {
|
|
view := m.mockUnifiedView
|
|
m.mockUnifiedViewMu.Unlock()
|
|
return view
|
|
}
|
|
m.mockUnifiedViewMu.Unlock()
|
|
|
|
resources, freshness := mock.UnifiedResourceSnapshot()
|
|
if len(resources) > 0 || !freshness.IsZero() {
|
|
// Consumers share this view between ticks, mirroring the
|
|
// sharing semantics the persistent-store ReadState path has
|
|
// always had in real mode: views are read-only.
|
|
view := monitorUnifiedStateViewFromResources(resources, freshness)
|
|
m.mockUnifiedViewMu.Lock()
|
|
m.mockUnifiedView = view
|
|
m.mockUnifiedViewVersion = version
|
|
m.mockUnifiedViewValid = true
|
|
m.mockUnifiedViewMu.Unlock()
|
|
return view
|
|
}
|
|
return monitorUnifiedStateViewFromSnapshot(m.GetState())
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
state := m.state
|
|
m.mu.RUnlock()
|
|
|
|
if store == nil {
|
|
return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateViewFromSnapshot(m.GetState()))
|
|
}
|
|
|
|
freshness := unifiedResourceFreshness(store, state)
|
|
|
|
if readState, ok := store.(unifiedresources.ReadState); ok {
|
|
// The continuity view lists this same store (or its overlay) below.
|
|
// Cloning here too discarded a complete registry on every broadcast.
|
|
return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateView{
|
|
readState: readState,
|
|
freshness: freshness,
|
|
})
|
|
}
|
|
|
|
resources := store.GetAll()
|
|
if len(resources) > 0 || state == nil {
|
|
return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateViewFromResources(resources, freshness))
|
|
}
|
|
|
|
return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateViewFromSnapshot(m.GetState()))
|
|
}
|
|
|
|
func (m *Monitor) currentUnifiedResourceFreshness() time.Time {
|
|
if m == nil {
|
|
return time.Time{}
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
state := m.state
|
|
m.mu.RUnlock()
|
|
return unifiedResourceFreshness(store, state)
|
|
}
|
|
|
|
func unifiedResourceFreshness(store ResourceStoreInterface, state *models.State) time.Time {
|
|
freshness := time.Time{}
|
|
if freshnessStore, ok := store.(UnifiedResourceFreshnessStore); ok {
|
|
freshness = freshnessStore.UnifiedResourceFreshness()
|
|
}
|
|
if freshness.IsZero() && state != nil {
|
|
freshness = state.GetLastUpdate()
|
|
}
|
|
return freshness
|
|
}
|
|
|
|
// UnifiedResourceSnapshot returns a canonical unified-resource seed plus the
|
|
// associated freshness marker. In mock mode it returns the shared mock
|
|
// unified-resource fixture graph rather than the live resource store.
|
|
func (m *Monitor) UnifiedResourceSnapshot() ([]unifiedresources.Resource, time.Time) {
|
|
view := m.currentUnifiedStateView()
|
|
// REST /api/resources seeds its registry from this snapshot. Apply the
|
|
// same user-metadata hydration (container customUrl) as the websocket
|
|
// broadcast path, so the two payload shapes cannot drift.
|
|
return m.applyPersistedMetadataToUnifiedResources(view.resources), view.freshness
|
|
}
|
|
|
|
// GetUnifiedReadState returns a typed unified read-state provider when the
|
|
// configured resource store supports it.
|
|
func (m *Monitor) GetUnifiedReadState() unifiedresources.ReadState {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
if store == nil {
|
|
return nil
|
|
}
|
|
|
|
readState, ok := store.(unifiedresources.ReadState)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return readState
|
|
}
|
|
|
|
// GetUnifiedReadStateOrSnapshot returns unified read-state when available.
|
|
// If the monitor has not been wired with a resource store yet, it creates an
|
|
// ephemeral snapshot-backed adapter to preserve read access without exposing
|
|
// direct state reads to consumer packages.
|
|
func (m *Monitor) GetUnifiedReadStateOrSnapshot() unifiedresources.ReadState {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
// A store that is itself the read state needs no view. Building one
|
|
// deep-clones every resource, twice, only for this caller to discard the
|
|
// copies, and alert evaluation reaches here once per resource per poll
|
|
// through the default CPU evaluation window, which made every poll cost
|
|
// grow with the square of the estate (#2199).
|
|
if !mock.IsMockEnabled() {
|
|
if readState := m.GetUnifiedReadState(); readState != nil {
|
|
return m.readStateWithStandaloneHostContinuity(readState)
|
|
}
|
|
}
|
|
return m.currentUnifiedStateView().readState
|
|
}
|
|
|
|
// currentModeReadState is GetUnifiedReadStateOrSnapshot for callers that act
|
|
// on the inventory it lists: those that open or remove alerts from it (Docker
|
|
// pruning, backup and snapshot guest lookups) and the pollers' previous-state
|
|
// carry, which writes it back into live state. After leaving mock mode the
|
|
// registry may still hold the fixture estate until an undisturbed rebuild in
|
|
// the new epoch replaces it, so they get a view of current state meanwhile.
|
|
// The view lists the state snapshot without provider-owned supplemental
|
|
// resources or persisted manual links and copies the whole estate, so it is
|
|
// only for callers outside per-resource loops.
|
|
func (m *Monitor) currentModeReadState() unifiedresources.ReadState {
|
|
if m == nil {
|
|
return nil
|
|
}
|
|
if !mock.IsMockEnabled() && m.GetUnifiedReadState() != nil && !m.mockModeFence.registryCurrent() {
|
|
return m.unifiedStateViewWithStandaloneHostContinuity(monitorUnifiedStateViewFromSnapshot(m.GetState())).readState
|
|
}
|
|
return m.GetUnifiedReadStateOrSnapshot()
|
|
}
|
|
|
|
// shouldSkipNodeMetrics returns true if we should skip detailed metric polling
|
|
// for the given node because a host agent is providing richer data.
|
|
// This helps reduce API load when agents are active.
|
|
func (m *Monitor) shouldSkipNodeMetrics(nodeName string) bool {
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
|
|
if store == nil {
|
|
return false
|
|
}
|
|
|
|
should := store.ShouldSkipAPIPolling(nodeName)
|
|
if should {
|
|
log.Debug().
|
|
Str("node", nodeName).
|
|
Msg("Skipping detailed node metrics - host agent provides data")
|
|
}
|
|
return should
|
|
}
|
|
|
|
// readPathRegistryFreshness bounds how stale the unified registry may be
|
|
// when a read path (state API, websocket hydrate) serves it without
|
|
// rebuilding. It only needs to be wide enough to collapse same-cycle rebuild
|
|
// storms (agent report + broadcast + state request); it must stay well under
|
|
// the poll cadence so every poll-cycle broadcast still publishes a fresh
|
|
// registry.
|
|
const readPathRegistryFreshness = 2 * time.Second
|
|
|
|
// readRefreshResourceStore is implemented by stores whose read paths can
|
|
// refresh without queueing behind ingest rebuilds (see MonitorAdapter).
|
|
type readRefreshResourceStore interface {
|
|
TryReplaceRegistryForRead(snapshot models.StateSnapshot, maxAge time.Duration, supply func() map[unifiedresources.DataSource][]unifiedresources.IngestRecord) bool
|
|
}
|
|
|
|
// updateResourceStoreForRead keeps the resource store fresh on behalf of a
|
|
// read path. Ingest boundaries (agent reports) call updateResourceStore and
|
|
// always rebuild; serving state must not redo work an ingest just did, and
|
|
// above all must not queue behind an in-flight rebuild whose change-record
|
|
// and identity-pin persistence can take seconds per transaction on slow
|
|
// volumes (#1665: /api/state stuck for minutes with SQLite on NFS).
|
|
func (m *Monitor) updateResourceStoreForRead(state models.StateSnapshot, scope mockModeScope) {
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
|
|
if store == nil {
|
|
return
|
|
}
|
|
readStore, ok := store.(readRefreshResourceStore)
|
|
if !ok {
|
|
m.updateResourceStore(state, scope)
|
|
return
|
|
}
|
|
|
|
if thresholdStore, ok := store.(StaleThresholdResourceStore); ok {
|
|
thresholdStore.SetStaleThresholds(m.resourceStaleThresholds())
|
|
}
|
|
snapshotForStore := state
|
|
if ownedSources := m.providerOwnedSnapshotSources(); len(ownedSources) > 0 {
|
|
snapshotForStore = unifiedresources.SnapshotWithoutSources(state, ownedSources)
|
|
}
|
|
rebuilt := false
|
|
mark, published := scope.publish(func() bool {
|
|
rebuilt = readStore.TryReplaceRegistryForRead(snapshotForStore, readPathRegistryFreshness, m.collectSupplementalRecordsBySource)
|
|
if rebuilt {
|
|
recordSupplementalResourceChanges(store, m.collectSupplementalChanges())
|
|
}
|
|
return rebuilt
|
|
})
|
|
if !published || !rebuilt {
|
|
return
|
|
}
|
|
m.syncPublishedResourceStore(store, scope, mark)
|
|
}
|
|
|
|
// syncPublishedResourceStore runs the metric and alert syncs over the registry
|
|
// a refresh just published under scope. A rebuild from an ended mock-mode
|
|
// epoch that overlapped the refresh may have replaced that registry with the
|
|
// mode the monitor left, so the alert sync runs only if none did.
|
|
func (m *Monitor) syncPublishedResourceStore(store ResourceStoreInterface, scope mockModeScope, mark uint64) {
|
|
store = newResourceSnapshotStore(store)
|
|
resources := store.GetAll()
|
|
m.syncAllUnifiedMetrics(store)
|
|
if scope.undisturbedSince(mark) {
|
|
m.syncUnifiedResourceAlertsToState(resources, scope)
|
|
}
|
|
}
|
|
|
|
// currentStateWithScope reads the monitor state together with the mock-mode
|
|
// scope it was read in. GetState serves the fixture graph in mock mode, so a
|
|
// refresh that evaluates alerts from it must take the scope first.
|
|
func (m *Monitor) currentStateWithScope() (models.StateSnapshot, mockModeScope) {
|
|
scope := m.mockModeFence.begin()
|
|
return m.GetState(), scope
|
|
}
|
|
|
|
// updateResourceStore populates the canonical resource store from current
|
|
// monitoring state. Callers use it at accepted-ingest boundaries and before
|
|
// broadcast hydration so every ReadState consumer observes the same snapshot.
|
|
// The scope must predate the read of state (see currentStateWithScope).
|
|
func (m *Monitor) updateResourceStore(state models.StateSnapshot, scope mockModeScope) {
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
|
|
if store == nil {
|
|
log.Debug().Msg("[Resources] No resource store configured, skipping population")
|
|
return
|
|
}
|
|
|
|
if thresholdStore, ok := store.(StaleThresholdResourceStore); ok {
|
|
thresholdStore.SetStaleThresholds(m.resourceStaleThresholds())
|
|
}
|
|
|
|
log.Debug().
|
|
Int("nodes", len(state.Nodes)).
|
|
Int("vms", len(state.VMs)).
|
|
Int("containers", len(state.Containers)).
|
|
Int("hosts", len(state.Hosts)).
|
|
Int("dockerHosts", len(state.DockerHosts)).
|
|
Msg("[Resources] Populating resource store from state snapshot")
|
|
|
|
snapshotForStore := state
|
|
ownedSources := m.providerOwnedSnapshotSources()
|
|
if len(ownedSources) > 0 {
|
|
snapshotForStore = unifiedresources.SnapshotWithoutSources(state, ownedSources)
|
|
sourceNames := make([]string, 0, len(ownedSources))
|
|
for _, source := range ownedSources {
|
|
sourceNames = append(sourceNames, string(source))
|
|
}
|
|
log.Debug().
|
|
Strs("sources", sourceNames).
|
|
Msg("[Resources] Suppressing legacy snapshot slices for provider-owned sources")
|
|
}
|
|
|
|
mark, published := scope.publish(func() bool {
|
|
recordsBySource := m.collectSupplementalRecordsBySource()
|
|
supplementalChanges := m.collectSupplementalChanges()
|
|
if atomicStore, ok := store.(AtomicSnapshotResourceStore); ok {
|
|
atomicStore.PopulateSnapshotAndSupplemental(snapshotForStore, recordsBySource)
|
|
recordSupplementalResourceChanges(store, supplementalChanges)
|
|
for source, records := range recordsBySource {
|
|
if len(records) == 0 {
|
|
continue
|
|
}
|
|
log.Debug().
|
|
Str("source", string(source)).
|
|
Int("records", len(records)).
|
|
Msg("[Resources] Atomically ingested supplemental records")
|
|
}
|
|
return true
|
|
}
|
|
|
|
store.PopulateFromSnapshot(snapshotForStore)
|
|
|
|
supplementalStore, ok := store.(SupplementalRecordStore)
|
|
if ok {
|
|
for source, records := range recordsBySource {
|
|
if len(records) == 0 {
|
|
continue
|
|
}
|
|
supplementalStore.PopulateSupplementalRecords(source, records)
|
|
log.Debug().
|
|
Str("source", string(source)).
|
|
Int("records", len(records)).
|
|
Msg("[Resources] Ingested supplemental records")
|
|
}
|
|
}
|
|
|
|
recordSupplementalResourceChanges(store, supplementalChanges)
|
|
return true
|
|
})
|
|
if !published {
|
|
// The snapshot belongs to the mode the monitor just left.
|
|
return
|
|
}
|
|
m.syncPublishedResourceStore(store, scope, mark)
|
|
}
|
|
|
|
// resourceSnapshotStore serves one GetAll clone to every consumer of a single
|
|
// store-refresh pass. The metric syncs and the alert sync each cloned the
|
|
// whole registry for themselves, up to six clones for every accepted agent
|
|
// report (#2199). Consumers of a pass only read, so they can share one
|
|
// generation.
|
|
type resourceSnapshotStore struct {
|
|
ResourceStoreInterface
|
|
targets MetricsTargetResourceStore
|
|
resources []unifiedresources.Resource
|
|
listed bool
|
|
}
|
|
|
|
// newResourceSnapshotStore wraps store for one refresh pass. A store that
|
|
// cannot resolve metrics targets is returned as is, so the metric syncs'
|
|
// capability checks see exactly what they would without the wrapper.
|
|
func newResourceSnapshotStore(store ResourceStoreInterface) ResourceStoreInterface {
|
|
if store == nil {
|
|
return nil
|
|
}
|
|
if _, ok := store.(*resourceSnapshotStore); ok {
|
|
return store
|
|
}
|
|
targets, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return store
|
|
}
|
|
return &resourceSnapshotStore{ResourceStoreInterface: store, targets: targets}
|
|
}
|
|
|
|
func (s *resourceSnapshotStore) GetAll() []unifiedresources.Resource {
|
|
if !s.listed {
|
|
s.resources = s.ResourceStoreInterface.GetAll()
|
|
s.listed = true
|
|
}
|
|
return s.resources
|
|
}
|
|
|
|
func (s *resourceSnapshotStore) MetricsTargetForResource(resourceID string) *unifiedresources.MetricsTarget {
|
|
return s.targets.MetricsTargetForResource(resourceID)
|
|
}
|
|
|
|
// refreshUnifiedResourceStoreAfterAgentStateChange makes agent removal and
|
|
// host-agent evaluation immediately visible to canonical ReadState consumers;
|
|
// accepted reports go through refreshUnifiedResourceStoreAfterAgentReport.
|
|
// WebSocket broadcasts may also rebuild the store for their own hydrate path,
|
|
// but client presence must never be the trigger that publishes agent-backed
|
|
// runtime truth or retires removed inventory.
|
|
func (m *Monitor) refreshUnifiedResourceStoreAfterAgentStateChange() {
|
|
if m == nil || m.state == nil {
|
|
return
|
|
}
|
|
m.updateResourceStore(m.currentStateWithScope())
|
|
}
|
|
|
|
// agentReportRefreshInterval bounds how often accepted agent reports refresh
|
|
// the canonical store. It matches readPathRegistryFreshness, the staleness the
|
|
// read paths already accept.
|
|
const agentReportRefreshInterval = readPathRegistryFreshness
|
|
|
|
// agentReportRefreshState throttles report-driven store refreshes to a leading
|
|
// refresh plus at most one trailing refresh per window.
|
|
type agentReportRefreshState struct {
|
|
mu sync.Mutex
|
|
lastRun time.Time
|
|
running bool
|
|
pending *time.Timer
|
|
stopped bool
|
|
inFlight sync.WaitGroup
|
|
}
|
|
|
|
// refreshUnifiedResourceStoreAfterAgentReport publishes an accepted agent
|
|
// report to the canonical store. Every refresh is estate-wide, so refreshing
|
|
// on each report made total cost grow with agents times resources (#2199).
|
|
// The first report after a quiet window still refreshes before the handler
|
|
// returns. Reports within agentReportRefreshInterval of the previous refresh
|
|
// fold into one trailing refresh at the window's end, which reads the latest
|
|
// state and broadcasts it. Agent removal and host-agent evaluation keep
|
|
// refreshing immediately through refreshUnifiedResourceStoreAfterAgentStateChange.
|
|
func (m *Monitor) refreshUnifiedResourceStoreAfterAgentReport() {
|
|
if m == nil || m.state == nil {
|
|
return
|
|
}
|
|
window := m.agentReportRefreshWindow
|
|
if window <= 0 {
|
|
m.updateResourceStore(m.currentStateWithScope())
|
|
return
|
|
}
|
|
|
|
r := &m.agentReportRefresh
|
|
r.mu.Lock()
|
|
if r.stopped || r.pending != nil {
|
|
// A due trailing refresh reads state after this report was applied.
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
now := time.Now()
|
|
if !r.running && now.Sub(r.lastRun) >= window {
|
|
r.running = true
|
|
r.lastRun = now
|
|
r.mu.Unlock()
|
|
m.updateResourceStore(m.currentStateWithScope())
|
|
r.mu.Lock()
|
|
r.running = false
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
r.inFlight.Add(1)
|
|
r.pending = time.AfterFunc(max(window-now.Sub(r.lastRun), 0), m.runTrailingAgentReportRefresh)
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
func (m *Monitor) runTrailingAgentReportRefresh() {
|
|
r := &m.agentReportRefresh
|
|
defer r.inFlight.Done()
|
|
r.mu.Lock()
|
|
r.pending = nil
|
|
if r.stopped {
|
|
r.mu.Unlock()
|
|
return
|
|
}
|
|
r.running = true
|
|
r.lastRun = time.Now()
|
|
r.mu.Unlock()
|
|
|
|
m.updateResourceStore(m.currentStateWithScope())
|
|
|
|
r.mu.Lock()
|
|
r.running = false
|
|
r.mu.Unlock()
|
|
|
|
// No handler follows a trailing refresh, so publish it to clients here.
|
|
m.mu.RLock()
|
|
hub := m.wsHub
|
|
m.mu.RUnlock()
|
|
if hub != nil {
|
|
m.broadcastCurrentState(hub)
|
|
}
|
|
}
|
|
|
|
// stopAgentReportRefresh cancels a pending trailing refresh and waits for one
|
|
// already running, so none touches stores Stop is about to close.
|
|
func (m *Monitor) stopAgentReportRefresh() {
|
|
r := &m.agentReportRefresh
|
|
r.mu.Lock()
|
|
r.stopped = true
|
|
if r.pending != nil && r.pending.Stop() {
|
|
r.pending = nil
|
|
r.inFlight.Done()
|
|
}
|
|
r.mu.Unlock()
|
|
r.inFlight.Wait()
|
|
}
|
|
|
|
func recordSupplementalResourceChanges(store ResourceStoreInterface, changes []unifiedresources.ResourceChange) {
|
|
if store == nil || len(changes) == 0 {
|
|
return
|
|
}
|
|
|
|
recorder, ok := store.(canonicalResourceChangeRecorder)
|
|
if !ok || recorder == nil {
|
|
return
|
|
}
|
|
|
|
for _, change := range changes {
|
|
if err := recorder.RecordChange(change); err != nil {
|
|
log.Warn().
|
|
Err(err).
|
|
Str("resource_id", change.ResourceID).
|
|
Str("change_id", change.ID).
|
|
Str("kind", string(change.Kind)).
|
|
Msg("failed to record supplemental canonical resource change")
|
|
}
|
|
}
|
|
}
|
|
|
|
// unifiedMetricSampleKey identifies one persisted unified-metric series. The
|
|
// metrics store keys rows by (resource type, resource id, metric type, tier,
|
|
// timestamp), so the replay guard must match at least those coordinates.
|
|
type unifiedMetricSampleKey struct {
|
|
resourceType string
|
|
resourceID string
|
|
metricType string
|
|
}
|
|
|
|
// unifiedMetricSample is the last sample handed to the metrics store for a
|
|
// series. Timestamps are compared at the store's second resolution so a
|
|
// sub-second replay of the same observation is still recognised as redundant.
|
|
type unifiedMetricSample struct {
|
|
timestampUnix int64
|
|
value float64
|
|
}
|
|
|
|
// unifiedMetricSyncMaxSeries bounds the replay guard so long-lived monitors
|
|
// with churning resource IDs cannot pin memory. The map is reset wholesale
|
|
// when the cap is crossed; the worst case is one extra write per series.
|
|
const unifiedMetricSyncMaxSeries = 100000
|
|
|
|
// dedupeUnifiedMetricWrites drops exact repeats of a sample already handed to
|
|
// the metrics store. The unified sync runs on ingest boundaries and on
|
|
// read-side registry rebuilds (which fire as often as every two seconds), so
|
|
// the same observation is replayed many times between polls. Each replay would
|
|
// otherwise UPDATE the same row and commit a fresh transaction, churning the
|
|
// SQLite WAL with no new data (#1966). A changed value at the same timestamp is
|
|
// still written so corrections are not lost.
|
|
func (m *Monitor) dedupeUnifiedMetricWrites(writes []metrics.WriteMetric) []metrics.WriteMetric {
|
|
if len(writes) == 0 {
|
|
return writes
|
|
}
|
|
|
|
m.unifiedMetricSyncMu.Lock()
|
|
defer m.unifiedMetricSyncMu.Unlock()
|
|
if m.unifiedMetricSyncLast == nil || len(m.unifiedMetricSyncLast) > unifiedMetricSyncMaxSeries {
|
|
m.unifiedMetricSyncLast = make(map[unifiedMetricSampleKey]unifiedMetricSample, len(writes))
|
|
}
|
|
|
|
out := writes[:0]
|
|
for _, write := range writes {
|
|
key := unifiedMetricSampleKey{
|
|
resourceType: write.ResourceType,
|
|
resourceID: write.ResourceID,
|
|
metricType: write.MetricType,
|
|
}
|
|
sample := unifiedMetricSample{timestampUnix: write.Timestamp.Unix(), value: write.Value}
|
|
if previous, ok := m.unifiedMetricSyncLast[key]; ok && previous == sample {
|
|
continue
|
|
}
|
|
m.unifiedMetricSyncLast[key] = sample
|
|
out = append(out, write)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// syncAllUnifiedMetrics runs the per-resource-type unified syncs and commits
|
|
// the surviving store writes of the batched syncs as one transaction. Each sync
|
|
// still runs the per-series replay guard, so this only changes the transaction
|
|
// boundary, not the series written, their values or their observation times.
|
|
// The syncs run serially and each previously waited for its own commit, so a
|
|
// rebuild that carried new data for several resource types opened one SQLite
|
|
// transaction per type; batching collapses that to one (#1966). The physical
|
|
// disk sync keeps its own transaction because its SMART path writes directly.
|
|
func (m *Monitor) syncAllUnifiedMetrics(store ResourceStoreInterface) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
if m.metricsStore == nil {
|
|
// The in-memory history path still needs to run; there is no store
|
|
// batch to collect.
|
|
m.syncUnifiedAgentMetrics(store)
|
|
m.syncUnifiedVMMetrics(store)
|
|
m.syncUnifiedStorageMetrics(store)
|
|
m.syncUnifiedPhysicalDiskMetrics(store)
|
|
m.syncUnifiedAppContainerMetrics(store)
|
|
return
|
|
}
|
|
var batch []metrics.WriteMetric
|
|
sink := &batch
|
|
m.syncUnifiedAgentMetrics(store, sink)
|
|
m.syncUnifiedVMMetrics(store, sink)
|
|
m.syncUnifiedStorageMetrics(store, sink)
|
|
m.syncUnifiedAppContainerMetrics(store, sink)
|
|
m.syncUnifiedPhysicalDiskMetrics(store)
|
|
if len(batch) > 0 {
|
|
m.metricsStore.WriteBatchBounded(batch)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedAgentMetrics(store ResourceStoreInterface, sinks ...*[]metrics.WriteMetric) {
|
|
if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) {
|
|
return
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
now := time.Now()
|
|
storeWrites := make([]metrics.WriteMetric, 0)
|
|
appendStoreWrite := func(resourceType, resourceID, metricType string, value float64, observedAt time.Time) {
|
|
if m.metricsStore == nil {
|
|
return
|
|
}
|
|
storeWrites = append(storeWrites, metrics.WriteMetric{
|
|
ResourceType: resourceType,
|
|
ResourceID: resourceID,
|
|
MetricType: metricType,
|
|
Value: value,
|
|
Timestamp: observedAt,
|
|
Tier: metrics.TierRaw,
|
|
})
|
|
}
|
|
seenTargets := make(map[string]struct{})
|
|
for _, resource := range store.GetAll() {
|
|
if resource.Type != unifiedresources.ResourceTypeAgent || resource.Metrics == nil {
|
|
continue
|
|
}
|
|
if shouldSkipMockOwnedUnifiedMetricSync(resource) {
|
|
continue
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceAgent) ||
|
|
monitorHasSource(resource.Sources, unifiedresources.SourceProxmox) ||
|
|
monitorHasSource(resource.Sources, unifiedresources.SourceDocker) {
|
|
continue
|
|
}
|
|
|
|
target := resolver.MetricsTargetForResource(resource.ID)
|
|
if target == nil || target.ResourceType != "agent" || strings.TrimSpace(target.ResourceID) == "" {
|
|
continue
|
|
}
|
|
targetID := strings.TrimSpace(target.ResourceID)
|
|
if _, ok := seenTargets[targetID]; ok {
|
|
continue
|
|
}
|
|
seenTargets[targetID] = struct{}{}
|
|
metricKey := fmt.Sprintf("agent:%s", targetID)
|
|
observedAt := unifiedResourceObservedAt(resource, now)
|
|
|
|
// Native TrueNAS temperature must also survive the local History path.
|
|
// A sufficiently covered ring/store read need not query native fallback.
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceTrueNAS) && resource.Temperature != nil {
|
|
value := *resource.Temperature
|
|
if value > 0 && !math.IsNaN(value) && !math.IsInf(value, 0) {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "temperature", value, observedAt)
|
|
}
|
|
appendStoreWrite("agent", targetID, "temperature", value, observedAt)
|
|
}
|
|
}
|
|
|
|
if metric := resource.Metrics.CPU; metric != nil {
|
|
value := metric.Percent
|
|
if value == 0 {
|
|
value = metric.Value
|
|
}
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "cpu", value, observedAt)
|
|
}
|
|
appendStoreWrite("agent", targetID, "cpu", value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.Memory; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "memory", value, observedAt)
|
|
}
|
|
appendStoreWrite("agent", targetID, "memory", value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.Disk; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "disk", value, observedAt)
|
|
}
|
|
appendStoreWrite("agent", targetID, "disk", value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetIn; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "netin", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("agent", targetID, "netin", metric.Value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetOut; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "netout", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("agent", targetID, "netout", metric.Value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskRead; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "diskread", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("agent", targetID, "diskread", metric.Value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskWrite; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "diskwrite", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("agent", targetID, "diskwrite", metric.Value, observedAt)
|
|
}
|
|
}
|
|
storeWrites = m.dedupeUnifiedMetricWrites(storeWrites)
|
|
if len(sinks) > 0 && sinks[0] != nil {
|
|
*sinks[0] = append(*sinks[0], storeWrites...)
|
|
} else if len(storeWrites) > 0 {
|
|
m.metricsStore.WriteBatchBounded(storeWrites)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedVMMetrics(store ResourceStoreInterface, sinks ...*[]metrics.WriteMetric) {
|
|
if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) {
|
|
return
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
now := time.Now()
|
|
storeWrites := make([]metrics.WriteMetric, 0)
|
|
appendStoreWrite := func(resourceType, resourceID, metricType string, value float64, observedAt time.Time) {
|
|
if m.metricsStore == nil {
|
|
return
|
|
}
|
|
storeWrites = append(storeWrites, metrics.WriteMetric{
|
|
ResourceType: resourceType,
|
|
ResourceID: resourceID,
|
|
MetricType: metricType,
|
|
Value: value,
|
|
Timestamp: observedAt,
|
|
Tier: metrics.TierRaw,
|
|
})
|
|
}
|
|
seenTargets := make(map[string]struct{})
|
|
for _, resource := range store.GetAll() {
|
|
if resource.Type != unifiedresources.ResourceTypeVM || resource.Metrics == nil {
|
|
continue
|
|
}
|
|
if shouldSkipMockOwnedUnifiedMetricSync(resource) {
|
|
continue
|
|
}
|
|
|
|
// Proxmox and libvirt both have a native history writer
|
|
// (monitor_pve_guest_helpers and monitor_libvirt), so the unified sync
|
|
// must not add a second timeline for the same series.
|
|
hasNativeVMWriter := strings.EqualFold(strings.TrimSpace(resource.Technology), "libvirt")
|
|
for _, source := range resource.Sources {
|
|
if source == unifiedresources.SourceProxmox {
|
|
hasNativeVMWriter = true
|
|
break
|
|
}
|
|
}
|
|
if hasNativeVMWriter {
|
|
continue
|
|
}
|
|
|
|
target := resolver.MetricsTargetForResource(resource.ID)
|
|
if target == nil || target.ResourceType != "vm" || strings.TrimSpace(target.ResourceID) == "" {
|
|
continue
|
|
}
|
|
targetID := strings.TrimSpace(target.ResourceID)
|
|
if _, ok := seenTargets[targetID]; ok {
|
|
continue
|
|
}
|
|
seenTargets[targetID] = struct{}{}
|
|
observedAt := unifiedResourceObservedAt(resource, now)
|
|
|
|
if metric := resource.Metrics.CPU; metric != nil {
|
|
value := metric.Percent
|
|
if value == 0 {
|
|
value = metric.Value
|
|
}
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "cpu", value, observedAt)
|
|
}
|
|
appendStoreWrite("vm", targetID, "cpu", value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.Memory; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "memory", value, observedAt)
|
|
}
|
|
appendStoreWrite("vm", targetID, "memory", value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.Disk; metric != nil && (metric.Total != nil || metric.Percent > 0 || metric.Used != nil) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "disk", value, observedAt)
|
|
}
|
|
appendStoreWrite("vm", targetID, "disk", value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetIn; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "netin", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("vm", targetID, "netin", metric.Value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetOut; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "netout", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("vm", targetID, "netout", metric.Value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskRead; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "diskread", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("vm", targetID, "diskread", metric.Value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskWrite; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(targetID, "diskwrite", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("vm", targetID, "diskwrite", metric.Value, observedAt)
|
|
}
|
|
}
|
|
storeWrites = m.dedupeUnifiedMetricWrites(storeWrites)
|
|
if len(sinks) > 0 && sinks[0] != nil {
|
|
*sinks[0] = append(*sinks[0], storeWrites...)
|
|
} else if len(storeWrites) > 0 {
|
|
m.metricsStore.WriteBatchBounded(storeWrites)
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedStorageMetrics(store ResourceStoreInterface, sinks ...*[]metrics.WriteMetric) {
|
|
if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) {
|
|
return
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
now := time.Now()
|
|
storeWrites := make([]metrics.WriteMetric, 0)
|
|
appendStoreWrite := func(resourceType, resourceID, metricType string, value float64, observedAt time.Time) {
|
|
if m.metricsStore == nil {
|
|
return
|
|
}
|
|
storeWrites = append(storeWrites, metrics.WriteMetric{
|
|
ResourceType: resourceType,
|
|
ResourceID: resourceID,
|
|
MetricType: metricType,
|
|
Value: value,
|
|
Timestamp: observedAt,
|
|
Tier: metrics.TierRaw,
|
|
})
|
|
}
|
|
seenTargets := make(map[string]struct{})
|
|
for _, resource := range store.GetAll() {
|
|
if resource.Type != unifiedresources.ResourceTypeStorage || resource.Metrics == nil || resource.Metrics.Disk == nil {
|
|
continue
|
|
}
|
|
if shouldSkipMockOwnedUnifiedMetricSync(resource) {
|
|
continue
|
|
}
|
|
|
|
// Native Proxmox storage already writes to history during the storage poller.
|
|
if resource.Storage != nil && resource.Storage.Platform == "" {
|
|
hasProxmoxSource := false
|
|
for _, source := range resource.Sources {
|
|
if source == unifiedresources.SourceProxmox {
|
|
hasProxmoxSource = true
|
|
break
|
|
}
|
|
}
|
|
if hasProxmoxSource {
|
|
continue
|
|
}
|
|
}
|
|
|
|
target := resolver.MetricsTargetForResource(resource.ID)
|
|
var targetID string
|
|
if target != nil && target.ResourceType == "storage" && strings.TrimSpace(target.ResourceID) != "" {
|
|
targetID = strings.TrimSpace(target.ResourceID)
|
|
} else {
|
|
targetID = resource.ID
|
|
}
|
|
if _, ok := seenTargets[targetID]; ok {
|
|
continue
|
|
}
|
|
seenTargets[targetID] = struct{}{}
|
|
|
|
disk := resource.Metrics.Disk
|
|
observedAt := unifiedMetricObservedAt(resource, disk, now)
|
|
usage := disk.Percent
|
|
used := int64(0)
|
|
total := int64(0)
|
|
free := int64(0)
|
|
if disk.Used != nil {
|
|
used = *disk.Used
|
|
}
|
|
if disk.Total != nil {
|
|
total = *disk.Total
|
|
}
|
|
if total > 0 {
|
|
free = total - used
|
|
if usage == 0 && used > 0 {
|
|
usage = (float64(used) / float64(total)) * 100
|
|
}
|
|
}
|
|
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddStorageMetric(targetID, "usage", usage, observedAt)
|
|
if total > 0 {
|
|
m.metricsHistory.AddStorageMetric(targetID, "used", float64(used), observedAt)
|
|
m.metricsHistory.AddStorageMetric(targetID, "total", float64(total), observedAt)
|
|
m.metricsHistory.AddStorageMetric(targetID, "avail", float64(free), observedAt)
|
|
}
|
|
}
|
|
appendStoreWrite("storage", targetID, "usage", usage, observedAt)
|
|
if total > 0 {
|
|
appendStoreWrite("storage", targetID, "used", float64(used), observedAt)
|
|
appendStoreWrite("storage", targetID, "total", float64(total), observedAt)
|
|
appendStoreWrite("storage", targetID, "avail", float64(free), observedAt)
|
|
}
|
|
}
|
|
storeWrites = m.dedupeUnifiedMetricWrites(storeWrites)
|
|
if len(sinks) > 0 && sinks[0] != nil {
|
|
*sinks[0] = append(*sinks[0], storeWrites...)
|
|
} else if len(storeWrites) > 0 {
|
|
m.metricsStore.WriteBatchBounded(storeWrites)
|
|
}
|
|
}
|
|
|
|
// unifiedMetricObservedAt returns the source observation time represented by a
|
|
// canonical metric. Registry rebuilds can happen several times during one
|
|
// polling cycle; using rebuild time here would turn one upstream observation
|
|
// into several distinct history samples.
|
|
func unifiedMetricObservedAt(resource unifiedresources.Resource, metric *unifiedresources.MetricValue, fallback time.Time) time.Time {
|
|
if metric != nil && metric.Source != "" {
|
|
if status, ok := resource.SourceStatus[metric.Source]; ok && !status.LastSeen.IsZero() {
|
|
return status.LastSeen
|
|
}
|
|
}
|
|
if !resource.LastSeen.IsZero() {
|
|
return resource.LastSeen
|
|
}
|
|
return fallback
|
|
}
|
|
|
|
// unifiedResourceObservedAt resolves the source observation time for a
|
|
// canonical resource from whichever metric carries a source attribution.
|
|
// Unified syncs use it as the history timestamp so a read-side registry rebuild
|
|
// re-issuing the same observation maps to one timestamp instead of inventing a
|
|
// fresh sample each rebuild (see #1966).
|
|
func unifiedResourceObservedAt(resource unifiedresources.Resource, fallback time.Time) time.Time {
|
|
if resource.Metrics != nil {
|
|
for _, metric := range []*unifiedresources.MetricValue{
|
|
resource.Metrics.CPU,
|
|
resource.Metrics.Memory,
|
|
resource.Metrics.Disk,
|
|
resource.Metrics.NetIn,
|
|
resource.Metrics.NetOut,
|
|
} {
|
|
if metric != nil && metric.Source != "" {
|
|
return unifiedMetricObservedAt(resource, metric, fallback)
|
|
}
|
|
}
|
|
}
|
|
return unifiedMetricObservedAt(resource, nil, fallback)
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedPhysicalDiskMetrics(store ResourceStoreInterface) {
|
|
if store == nil || m.metricsStore == nil {
|
|
return
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
now := time.Now()
|
|
seenTargets := make(map[string]struct{})
|
|
for _, resource := range store.GetAll() {
|
|
if resource.Type != unifiedresources.ResourceTypePhysicalDisk || resource.PhysicalDisk == nil {
|
|
continue
|
|
}
|
|
if shouldSkipMockOwnedUnifiedMetricSync(resource) {
|
|
continue
|
|
}
|
|
|
|
hasNativeWriter := false
|
|
for _, source := range resource.Sources {
|
|
if source == unifiedresources.SourceProxmox || source == unifiedresources.SourceAgent {
|
|
hasNativeWriter = true
|
|
break
|
|
}
|
|
}
|
|
if hasNativeWriter {
|
|
continue
|
|
}
|
|
|
|
target := resolver.MetricsTargetForResource(resource.ID)
|
|
if target == nil || target.ResourceType != "disk" || strings.TrimSpace(target.ResourceID) == "" {
|
|
continue
|
|
}
|
|
targetID := strings.TrimSpace(target.ResourceID)
|
|
if _, ok := seenTargets[targetID]; ok {
|
|
continue
|
|
}
|
|
seenTargets[targetID] = struct{}{}
|
|
|
|
nodeName := ""
|
|
for _, hostname := range resource.Identity.Hostnames {
|
|
if hostname = strings.TrimSpace(hostname); hostname != "" {
|
|
nodeName = hostname
|
|
break
|
|
}
|
|
}
|
|
if nodeName == "" {
|
|
nodeName = firstNonEmptyString(strings.TrimSpace(resource.ParentName), strings.TrimSpace(resource.Name))
|
|
}
|
|
|
|
disk := models.PhysicalDisk{
|
|
ID: resource.ID,
|
|
Node: nodeName,
|
|
DevPath: resource.PhysicalDisk.DevPath,
|
|
Model: resource.PhysicalDisk.Model,
|
|
Serial: resource.PhysicalDisk.Serial,
|
|
WWN: resource.PhysicalDisk.WWN,
|
|
Type: resource.PhysicalDisk.DiskType,
|
|
Size: resource.PhysicalDisk.SizeBytes,
|
|
Health: resource.PhysicalDisk.Health,
|
|
Wearout: resource.PhysicalDisk.Wearout,
|
|
Temperature: resource.PhysicalDisk.Temperature,
|
|
RPM: resource.PhysicalDisk.RPM,
|
|
Used: resource.PhysicalDisk.Used,
|
|
SmartAttributes: smartAttributesFromUnifiedMeta(resource.PhysicalDisk.SMART),
|
|
Collection: diskinventory.CloneStatus(resource.PhysicalDisk.Collection),
|
|
LastChecked: resource.LastSeen,
|
|
}
|
|
if disk.Serial == "" {
|
|
disk.ID = targetID
|
|
}
|
|
// Anchor to the source observation time so a read-side registry rebuild
|
|
// re-issuing the same snapshot maps to one sample instead of a fresh
|
|
// wall-clock row (#1966).
|
|
m.writeSMARTMetrics(disk, unifiedResourceObservedAt(resource, now))
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) syncUnifiedAppContainerMetrics(store ResourceStoreInterface, sinks ...*[]metrics.WriteMetric) {
|
|
if store == nil || (m.metricsHistory == nil && m.metricsStore == nil) {
|
|
return
|
|
}
|
|
|
|
resolver, ok := store.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return
|
|
}
|
|
|
|
now := time.Now()
|
|
storeWrites := make([]metrics.WriteMetric, 0)
|
|
appendStoreWrite := func(resourceType, resourceID, metricType string, value float64, observedAt time.Time) {
|
|
if m.metricsStore == nil {
|
|
return
|
|
}
|
|
storeWrites = append(storeWrites, metrics.WriteMetric{
|
|
ResourceType: resourceType,
|
|
ResourceID: resourceID,
|
|
MetricType: metricType,
|
|
Value: value,
|
|
Timestamp: observedAt,
|
|
Tier: metrics.TierRaw,
|
|
})
|
|
}
|
|
seenTargets := make(map[string]struct{})
|
|
for _, resource := range store.GetAll() {
|
|
if resource.Type != unifiedresources.ResourceTypeAppContainer || resource.Metrics == nil {
|
|
continue
|
|
}
|
|
if shouldSkipMockOwnedUnifiedMetricSync(resource) {
|
|
continue
|
|
}
|
|
hasDockerSource := false
|
|
for _, source := range resource.Sources {
|
|
if source == unifiedresources.SourceDocker {
|
|
hasDockerSource = true
|
|
break
|
|
}
|
|
}
|
|
if hasDockerSource {
|
|
continue
|
|
}
|
|
|
|
target := resolver.MetricsTargetForResource(resource.ID)
|
|
if target == nil || target.ResourceType != "app-container" || strings.TrimSpace(target.ResourceID) == "" {
|
|
continue
|
|
}
|
|
targetID := strings.TrimSpace(target.ResourceID)
|
|
if _, ok := seenTargets[targetID]; ok {
|
|
continue
|
|
}
|
|
seenTargets[targetID] = struct{}{}
|
|
metricKey := fmt.Sprintf("docker:%s", targetID)
|
|
observedAt := unifiedResourceObservedAt(resource, now)
|
|
|
|
if metric := resource.Metrics.CPU; metric != nil {
|
|
value := metric.Percent
|
|
if value == 0 {
|
|
value = metric.Value
|
|
}
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "cpu", value, observedAt)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "cpu", value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.Memory; metric != nil && (metric.Total != nil || metric.Percent > 0) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "memory", value, observedAt)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "memory", value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.Disk; metric != nil && (metric.Total != nil || metric.Percent > 0) {
|
|
value := metric.Percent
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "disk", value, observedAt)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "disk", value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetIn; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "netin", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "netin", metric.Value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.NetOut; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "netout", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "netout", metric.Value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskRead; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "diskread", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "diskread", metric.Value, observedAt)
|
|
}
|
|
|
|
if metric := resource.Metrics.DiskWrite; metric != nil {
|
|
if m.metricsHistory != nil {
|
|
m.metricsHistory.AddGuestMetric(metricKey, "diskwrite", metric.Value, observedAt)
|
|
}
|
|
appendStoreWrite("dockerContainer", targetID, "diskwrite", metric.Value, observedAt)
|
|
}
|
|
}
|
|
storeWrites = m.dedupeUnifiedMetricWrites(storeWrites)
|
|
if len(sinks) > 0 && sinks[0] != nil {
|
|
*sinks[0] = append(*sinks[0], storeWrites...)
|
|
} else if len(storeWrites) > 0 {
|
|
m.metricsStore.WriteBatchBounded(storeWrites)
|
|
}
|
|
}
|
|
|
|
func shouldSkipMockOwnedUnifiedMetricSync(resource unifiedresources.Resource) bool {
|
|
if !mock.IsMockEnabled() {
|
|
return false
|
|
}
|
|
|
|
// In mock mode the canonical mock sampler owns chart/history continuity for
|
|
// the entire demo estate. Unified-resource sync must not append a second
|
|
// live timeline on top of seeded mock history for any resource class.
|
|
_ = resource
|
|
return true
|
|
}
|
|
|
|
func shouldSkipNativeMockStateMetricWrites() bool {
|
|
return mock.IsMockEnabled()
|
|
}
|
|
|
|
// getUnifiedResourcesForBroadcast retrieves all resources from the store.
|
|
// Returns nil if no resource store is configured.
|
|
func (m *Monitor) getUnifiedResourcesForBroadcast() []unifiedresources.Resource {
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
|
|
if store == nil {
|
|
log.Debug().Msg("[Resources] No store for broadcast")
|
|
return nil
|
|
}
|
|
|
|
allResources := store.GetAll()
|
|
log.Debug().Int("count", len(allResources)).Msg("[Resources] Got resources for broadcast")
|
|
if len(allResources) == 0 {
|
|
return nil
|
|
}
|
|
return allResources
|
|
}
|
|
|
|
// getResourcesForBroadcast retrieves all resources from the store and converts
|
|
// them to frontend format.
|
|
func (m *Monitor) getResourcesForBroadcast() []models.ResourceFrontend {
|
|
m.mu.RLock()
|
|
store := m.resourceStore
|
|
m.mu.RUnlock()
|
|
resourcesForBroadcast := m.applyPersistedMetadataToUnifiedResources(m.getUnifiedResourcesForBroadcast())
|
|
resourcesForBroadcast = unifiedresources.AttachResourceHealth(
|
|
resourcesForBroadcast,
|
|
resourceHealthAlerts(m.activeAlertsSnapshot()),
|
|
time.Now().UTC(),
|
|
)
|
|
resources, _ := convertResourcesForBroadcast(
|
|
resourcesForBroadcast,
|
|
broadcastMetricsTargetResolver(store),
|
|
)
|
|
return resources
|
|
}
|
|
|
|
func resourceHealthAlerts(active []models.Alert) []unifiedresources.ResourceHealthAlert {
|
|
if len(active) == 0 {
|
|
return nil
|
|
}
|
|
out := make([]unifiedresources.ResourceHealthAlert, 0, len(active))
|
|
for _, alert := range active {
|
|
out = append(out, unifiedresources.ResourceHealthAlert{
|
|
ResourceID: alert.ResourceID,
|
|
Level: alert.Level,
|
|
Type: alert.Type,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (m *Monitor) applyPersistedMetadataToUnifiedResources(resources []unifiedresources.Resource) []unifiedresources.Resource {
|
|
if len(resources) == 0 || m == nil {
|
|
return resources
|
|
}
|
|
|
|
out := make([]unifiedresources.Resource, len(resources))
|
|
copy(out, resources)
|
|
for i := range out {
|
|
m.applyPersistedMetadataToUnifiedResource(&out[i])
|
|
}
|
|
return out
|
|
}
|
|
|
|
// applyPersistedMetadataToUnifiedResource changes only a caller-owned resource
|
|
// value, never its nested registry state. Empty persisted values clear stale URLs.
|
|
func (m *Monitor) applyPersistedMetadataToUnifiedResource(resource *unifiedresources.Resource) {
|
|
switch unifiedresources.ContractResourceType(*resource) {
|
|
case unifiedresources.ResourceTypeAppContainer:
|
|
if resource.Docker == nil {
|
|
return
|
|
}
|
|
hostID := strings.TrimSpace(resource.Docker.HostSourceID)
|
|
containerID := strings.TrimSpace(resource.Docker.ContainerID)
|
|
if hostID == "" {
|
|
return
|
|
}
|
|
if customURL, ok := m.dockerAppContainerCustomURL(*resource, hostID, containerID); ok {
|
|
// The metadata record is authoritative even when empty: an
|
|
// explicit clear must remove a stale URL still carried by an
|
|
// older unified-resource snapshot.
|
|
resource.CustomURL = strings.TrimSpace(customURL)
|
|
}
|
|
case unifiedresources.ResourceTypePod,
|
|
unifiedresources.ResourceTypeK8sDeployment,
|
|
unifiedresources.ResourceTypeK8sService:
|
|
if customURL, ok := m.kubernetesWorkloadCustomURL(*resource); ok {
|
|
resource.CustomURL = strings.TrimSpace(customURL)
|
|
}
|
|
case unifiedresources.ResourceTypeAgent,
|
|
unifiedresources.ResourceType("docker-host"),
|
|
unifiedresources.ResourceTypePBS,
|
|
unifiedresources.ResourceTypePMG,
|
|
unifiedresources.ResourceTypeK8sCluster,
|
|
unifiedresources.ResourceTypeK8sNode:
|
|
if customURL, ok := m.hostResourceCustomURL(*resource); ok {
|
|
resource.CustomURL = customURL
|
|
}
|
|
}
|
|
}
|
|
|
|
func appendUniqueMetadataCandidate(candidates []string, seen map[string]struct{}, value string) []string {
|
|
value = strings.TrimSpace(value)
|
|
if value == "" {
|
|
return candidates
|
|
}
|
|
if _, exists := seen[value]; exists {
|
|
return candidates
|
|
}
|
|
seen[value] = struct{}{}
|
|
return append(candidates, value)
|
|
}
|
|
|
|
func hostResourceMetadataCandidates(resource unifiedresources.Resource) []string {
|
|
candidates := make([]string, 0, 20)
|
|
seen := make(map[string]struct{}, 20)
|
|
add := func(value string) {
|
|
candidates = appendUniqueMetadataCandidate(candidates, seen, value)
|
|
}
|
|
|
|
if resource.Docker != nil {
|
|
add(resource.Docker.HostSourceID)
|
|
add(resource.Docker.AgentID)
|
|
}
|
|
if resource.Agent != nil {
|
|
add(resource.Agent.AgentID)
|
|
}
|
|
if resource.PBS != nil {
|
|
add(resource.PBS.InstanceID)
|
|
}
|
|
if resource.PMG != nil {
|
|
add(resource.PMG.InstanceID)
|
|
}
|
|
if resource.Kubernetes != nil {
|
|
if unifiedresources.ContractResourceType(resource) == unifiedresources.ResourceTypeK8sNode {
|
|
clusterID := strings.TrimSpace(resource.Kubernetes.ClusterID)
|
|
nodeName := strings.TrimSpace(resource.Kubernetes.NodeName)
|
|
if nodeName == "" {
|
|
nodeName = strings.TrimSpace(resource.Name)
|
|
}
|
|
if clusterID != "" && nodeName != "" {
|
|
add(clusterID + ":node:" + nodeName)
|
|
}
|
|
add(resource.Kubernetes.NodeUID)
|
|
}
|
|
add(resource.Kubernetes.ClusterID)
|
|
add(resource.Kubernetes.AgentID)
|
|
}
|
|
if resource.Proxmox != nil {
|
|
add(resource.Proxmox.SourceID)
|
|
add(resource.Proxmox.NodeName)
|
|
}
|
|
if resource.DiscoveryTarget != nil {
|
|
add(resource.DiscoveryTarget.AgentID)
|
|
add(resource.DiscoveryTarget.ResourceID)
|
|
}
|
|
if resource.MetricsTarget != nil {
|
|
add(resource.MetricsTarget.ResourceID)
|
|
}
|
|
if resource.Canonical != nil {
|
|
add(resource.Canonical.PrimaryID)
|
|
if prefixAt := strings.IndexByte(resource.Canonical.PrimaryID, ':'); prefixAt >= 0 {
|
|
add(resource.Canonical.PrimaryID[prefixAt+1:])
|
|
}
|
|
for _, alias := range resource.Canonical.Aliases {
|
|
add(alias)
|
|
}
|
|
for _, supersededID := range resource.Canonical.SupersededIDs {
|
|
add(supersededID)
|
|
}
|
|
}
|
|
add(resource.Identity.MachineID)
|
|
add(resource.ID)
|
|
add(resource.Name)
|
|
return candidates
|
|
}
|
|
|
|
func (m *Monitor) hostResourceCustomURL(resource unifiedresources.Resource) (string, bool) {
|
|
if m == nil {
|
|
return "", false
|
|
}
|
|
|
|
candidates := hostResourceMetadataCandidates(resource)
|
|
if unifiedresources.ContractResourceType(resource) == unifiedresources.ResourceType("docker-host") &&
|
|
m.dockerMetadataStore != nil {
|
|
for _, candidate := range candidates {
|
|
if meta := m.dockerMetadataStore.GetHostMetadata(candidate); meta != nil {
|
|
if customURL := strings.TrimSpace(meta.CustomURL); customURL != "" {
|
|
return customURL, true
|
|
}
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
if m.hostMetadataStore != nil {
|
|
for _, candidate := range candidates {
|
|
if meta := m.hostMetadataStore.Get(candidate); meta != nil {
|
|
if customURL := strings.TrimSpace(meta.CustomURL); customURL != "" {
|
|
return customURL, true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return "", false
|
|
}
|
|
|
|
func (m *Monitor) dockerAppContainerCustomURL(
|
|
resource unifiedresources.Resource,
|
|
hostID,
|
|
containerID string,
|
|
) (string, bool) {
|
|
if m == nil {
|
|
return "", false
|
|
}
|
|
|
|
if m.guestMetadataStore != nil {
|
|
if stableKey := dockerAppContainerMetadataKey(hostID, resource.Name); stableKey != "" {
|
|
if meta := m.guestMetadataStore.Get(stableKey); meta != nil {
|
|
return strings.TrimSpace(meta.CustomURL), true
|
|
}
|
|
}
|
|
for _, key := range append(
|
|
[]string{strings.TrimSpace(resource.ID)},
|
|
dockerAppContainerGuestMetadataLegacyKeys(hostID, containerID)...,
|
|
) {
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if meta := m.guestMetadataStore.Get(key); meta != nil {
|
|
return strings.TrimSpace(meta.CustomURL), true
|
|
}
|
|
}
|
|
}
|
|
|
|
if m.dockerMetadataStore != nil {
|
|
// The runtime key is where drawer saves used to land, so it is always at
|
|
// least as fresh as the stable name key, which only receives
|
|
// copy-if-missing snapshots of it. Prefer the runtime record while it
|
|
// exists; the stable name key covers recreated containers whose old
|
|
// runtime key no longer matches.
|
|
if meta := m.dockerMetadataStore.Get(dockerContainerRuntimeMetadataKey(hostID, containerID)); meta != nil {
|
|
return strings.TrimSpace(meta.CustomURL), true
|
|
}
|
|
if stableKey := dockerContainerNameMetadataKey(hostID, resource.Name); stableKey != "" {
|
|
if meta := m.dockerMetadataStore.Get(stableKey); meta != nil {
|
|
return strings.TrimSpace(meta.CustomURL), true
|
|
}
|
|
}
|
|
}
|
|
|
|
return "", false
|
|
}
|
|
|
|
// broadcastResourceCatalogs carries the content-addressed dedupe catalogs the
|
|
// broadcast conversion extracts from per-resource static metadata.
|
|
type broadcastResourceCatalogs struct {
|
|
capabilities map[string]json.RawMessage
|
|
policies map[string]json.RawMessage
|
|
aiSafeSummaries map[string]string
|
|
}
|
|
|
|
// convertResourcesForBroadcast converts unified resources into the frontend
|
|
// payload shape. Distinct capability blobs are deduped into the returned
|
|
// catalog and referenced per resource via capabilitiesRef.
|
|
func convertResourcesForBroadcast(
|
|
allResources []unifiedresources.Resource,
|
|
metricsTargetResolvers ...MetricsTargetResourceStore,
|
|
) ([]models.ResourceFrontend, broadcastResourceCatalogs) {
|
|
if len(allResources) == 0 {
|
|
return []models.ResourceFrontend{}, broadcastResourceCatalogs{}
|
|
}
|
|
allResources = attachBroadcastMetricsTargets(
|
|
allResources,
|
|
firstBroadcastMetricsTargetResolver(metricsTargetResolvers),
|
|
)
|
|
allResources = unifiedresources.CoalescePresentationHostResources(allResources)
|
|
return convertPresentationResourcesForBroadcast(allResources)
|
|
}
|
|
|
|
// convertPresentationResourcesForBroadcast consumes an already-coalesced
|
|
// projection. Sorting the final rows avoids retaining a second estate-sized
|
|
// array of conversion inputs and coalescing the same hosts a second time.
|
|
func convertPresentationResourcesForBroadcast(allResources []unifiedresources.Resource) ([]models.ResourceFrontend, broadcastResourceCatalogs) {
|
|
if len(allResources) == 0 {
|
|
return []models.ResourceFrontend{}, broadcastResourceCatalogs{}
|
|
}
|
|
|
|
result := make([]models.ResourceFrontend, len(allResources))
|
|
sortKeys := make([]string, len(allResources))
|
|
catalogs := broadcastResourceCatalogs{
|
|
capabilities: make(map[string]json.RawMessage),
|
|
policies: make(map[string]json.RawMessage),
|
|
aiSafeSummaries: make(map[string]string),
|
|
}
|
|
for i, r := range allResources {
|
|
input := monitorResourceToConvertInput(r)
|
|
if len(input.Capabilities) > 0 {
|
|
id := capabilityCatalogID(input.Capabilities)
|
|
catalogs.capabilities[id] = input.Capabilities
|
|
input.CapabilitiesRef = id
|
|
input.Capabilities = nil
|
|
}
|
|
// Non-default policies and AI-safe summaries dedupe the same way:
|
|
// estates carry a handful of distinct postures and templated summary
|
|
// strings, so refs replace per-resource inline duplication.
|
|
if len(input.Policy) > 0 {
|
|
id := capabilityCatalogID(input.Policy)
|
|
catalogs.policies[id] = input.Policy
|
|
input.PolicyRef = id
|
|
input.Policy = nil
|
|
}
|
|
if input.AISafeSummary != "" {
|
|
id := capabilityCatalogID(json.RawMessage(input.AISafeSummary))
|
|
catalogs.aiSafeSummaries[id] = input.AISafeSummary
|
|
input.AISafeSummaryRef = id
|
|
input.AISafeSummary = ""
|
|
}
|
|
sortKey := strings.ToLower(input.DisplayName)
|
|
if sortKey == "" {
|
|
sortKey = strings.ToLower(input.Name)
|
|
}
|
|
sortKeys[i] = sortKey
|
|
result[i] = models.ConvertResourceToFrontend(input)
|
|
}
|
|
|
|
sort.Sort(broadcastFrontendSort{resources: result, keys: sortKeys})
|
|
if len(catalogs.capabilities) == 0 {
|
|
catalogs.capabilities = nil
|
|
}
|
|
if len(catalogs.policies) == 0 {
|
|
catalogs.policies = nil
|
|
}
|
|
if len(catalogs.aiSafeSummaries) == 0 {
|
|
catalogs.aiSafeSummaries = nil
|
|
}
|
|
return result, catalogs
|
|
}
|
|
|
|
// Keep each precomputed sort key beside its row while sorting in place.
|
|
type broadcastFrontendSort struct {
|
|
resources []models.ResourceFrontend
|
|
keys []string
|
|
}
|
|
|
|
func (s broadcastFrontendSort) Len() int { return len(s.resources) }
|
|
func (s broadcastFrontendSort) Less(i, j int) bool {
|
|
if s.keys[i] == s.keys[j] {
|
|
return s.resources[i].ID < s.resources[j].ID
|
|
}
|
|
return s.keys[i] < s.keys[j]
|
|
}
|
|
func (s broadcastFrontendSort) Swap(i, j int) {
|
|
s.resources[i], s.resources[j] = s.resources[j], s.resources[i]
|
|
s.keys[i], s.keys[j] = s.keys[j], s.keys[i]
|
|
}
|
|
|
|
func broadcastMetricsTargetResolver(source interface{}) MetricsTargetResourceStore {
|
|
resolver, ok := source.(MetricsTargetResourceStore)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
return resolver
|
|
}
|
|
|
|
func firstBroadcastMetricsTargetResolver(
|
|
resolvers []MetricsTargetResourceStore,
|
|
) MetricsTargetResourceStore {
|
|
for _, resolver := range resolvers {
|
|
if resolver != nil {
|
|
return resolver
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func attachBroadcastMetricsTargets(
|
|
resources []unifiedresources.Resource,
|
|
resolver MetricsTargetResourceStore,
|
|
) []unifiedresources.Resource {
|
|
if len(resources) == 0 || resolver == nil {
|
|
return resources
|
|
}
|
|
|
|
out := make([]unifiedresources.Resource, len(resources))
|
|
copy(out, resources)
|
|
attachBroadcastMetricsTargetsInPlace(out, resolver)
|
|
return out
|
|
}
|
|
|
|
// Only call with the owned outer slice of a coalesced broadcast projection.
|
|
// Nested resource data stays read-only, and the resolver returns owned targets.
|
|
func attachBroadcastMetricsTargetsInPlace(resources []unifiedresources.Resource, resolver MetricsTargetResourceStore) {
|
|
if resolver == nil {
|
|
return
|
|
}
|
|
for i := range resources {
|
|
if resources[i].MetricsTarget == nil {
|
|
resources[i].MetricsTarget = validBroadcastMetricsTarget(resolver.MetricsTargetForResource(resources[i].ID))
|
|
}
|
|
}
|
|
}
|
|
|
|
func validBroadcastMetricsTarget(
|
|
target *unifiedresources.MetricsTarget,
|
|
) *unifiedresources.MetricsTarget {
|
|
if target == nil {
|
|
return nil
|
|
}
|
|
resourceType := strings.TrimSpace(target.ResourceType)
|
|
resourceID := strings.TrimSpace(target.ResourceID)
|
|
if resourceType == "" || resourceID == "" {
|
|
return nil
|
|
}
|
|
return &unifiedresources.MetricsTarget{
|
|
ResourceType: resourceType,
|
|
ResourceID: resourceID,
|
|
}
|
|
}
|
|
|
|
func monitorResourceToConvertInput(resource unifiedresources.Resource) models.ResourceConvertInput {
|
|
resourceType := monitorFrontendResourceType(&resource)
|
|
if resource.DiscoveryTarget == nil {
|
|
resource.DiscoveryTarget = monitorDiscoveryTarget(&resource, resourceType)
|
|
}
|
|
if resource.MetricsTarget == nil {
|
|
resource.MetricsTarget = monitorMetricsTarget(&resource, resourceType)
|
|
}
|
|
unifiedresources.RefreshCanonicalMetadata(&resource)
|
|
slimResourceForBroadcast(&resource)
|
|
name, displayName := monitorFrontendNames(&resource, resourceType)
|
|
platformID := monitorPlatformID(&resource, resourceType)
|
|
|
|
input := models.ResourceConvertInput{
|
|
ID: resource.ID,
|
|
Type: resourceType,
|
|
Technology: resource.Technology,
|
|
Name: name,
|
|
DisplayName: displayName,
|
|
PlatformID: platformID,
|
|
PlatformType: monitorPlatformType(&resource, resourceType),
|
|
SourceType: monitorSourceType(resource.Sources),
|
|
Sources: monitorSourceKeys(resource.Sources),
|
|
ParentID: monitorStringValue(resource.ParentID),
|
|
ParentName: resource.ParentName,
|
|
ChildCount: resource.ChildCount,
|
|
ClusterID: monitorClusterID(&resource),
|
|
Status: monitorFrontendStatus(&resource, resourceType),
|
|
CPU: monitorMetricInput(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.CPU })),
|
|
Memory: monitorMetricInput(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Memory })),
|
|
Disk: monitorMetricInput(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Disk })),
|
|
Temperature: monitorTemperature(&resource),
|
|
Uptime: monitorUptime(&resource),
|
|
Tags: append([]string(nil), resource.Tags...),
|
|
Labels: monitorLabels(&resource),
|
|
CustomURL: strings.TrimSpace(resource.CustomURL),
|
|
LastSeenUnix: monitorLastSeenUnix(resource.LastSeen),
|
|
Health: monitorRawJSON(resource.Health),
|
|
IncidentCount: resource.IncidentCount,
|
|
IncidentCode: resource.IncidentCode,
|
|
IncidentSeverity: string(resource.IncidentSeverity),
|
|
IncidentSummary: resource.IncidentSummary,
|
|
IncidentCategory: resource.IncidentCategory,
|
|
IncidentLabel: resource.IncidentLabel,
|
|
IncidentPriority: resource.IncidentPriority,
|
|
IncidentImpactSummary: resource.IncidentImpactSummary,
|
|
IncidentUrgency: resource.IncidentUrgency,
|
|
IncidentAction: resource.IncidentAction,
|
|
Identity: monitorIdentity(&resource, name),
|
|
DiscoveryTarget: monitorRawJSON(resource.DiscoveryTarget),
|
|
MetricsTarget: monitorRawJSON(resource.MetricsTarget),
|
|
Canonical: monitorRawJSON(resource.Canonical),
|
|
Policy: monitorRawJSON(resource.Policy),
|
|
AISafeSummary: resource.AISafeSummary,
|
|
Capabilities: monitorRawJSON(resource.Capabilities),
|
|
Relationships: monitorRawJSON(resource.Relationships),
|
|
RecentChanges: monitorRawJSON(resource.RecentChanges),
|
|
FacetCounts: monitorRawJSON(resource.FacetCounts),
|
|
Incidents: monitorRawJSON(resource.Incidents),
|
|
Proxmox: monitorRawJSON(resource.Proxmox),
|
|
Storage: monitorRawJSON(resource.Storage),
|
|
Agent: monitorRawJSON(resource.Agent),
|
|
Docker: monitorRawJSON(resource.Docker),
|
|
PBS: monitorRawJSON(resource.PBS),
|
|
PMG: monitorRawJSON(resource.PMG),
|
|
Kubernetes: monitorRawJSON(resource.Kubernetes),
|
|
PhysicalDisk: monitorRawJSON(resource.PhysicalDisk),
|
|
Ceph: monitorRawJSON(resource.Ceph),
|
|
TrueNAS: monitorRawJSON(resource.TrueNAS),
|
|
VMware: monitorRawJSON(resource.VMware),
|
|
Availability: monitorRawJSON(resource.Availability),
|
|
AvailabilityChecks: monitorRawJSON(resource.AvailabilityChecks),
|
|
PlatformData: monitorPlatformData(&resource, resourceType, platformID),
|
|
}
|
|
|
|
hasNetwork, rx, tx := monitorNetworkMetricInput(resource.Metrics)
|
|
input.HasNetwork = hasNetwork
|
|
input.NetworkRX = rx
|
|
input.NetworkTX = tx
|
|
input.DiskReadRate, input.DiskWriteRate = monitorDiskIOMetricInput(resource.Metrics)
|
|
|
|
return input
|
|
}
|
|
|
|
func monitorRawJSON(value interface{}) json.RawMessage {
|
|
if value == nil {
|
|
return nil
|
|
}
|
|
// Optional facets arrive as typed nil pointers/slices in an interface.
|
|
// Marshaling "null" only to discard it costs one encode and allocation
|
|
// per absent facet, on every row and frame. Named slice/map marshalers
|
|
// still get their normal chance to emit a non-null representation.
|
|
v := reflect.ValueOf(value)
|
|
switch v.Kind() {
|
|
case reflect.Ptr:
|
|
if v.IsNil() {
|
|
return nil
|
|
}
|
|
case reflect.Slice, reflect.Map:
|
|
if v.IsNil() {
|
|
if _, custom := value.(json.Marshaler); !custom {
|
|
if _, customText := value.(encoding.TextMarshaler); !customText {
|
|
return nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
encoded, err := json.Marshal(value)
|
|
if err != nil || string(encoded) == "null" {
|
|
return nil
|
|
}
|
|
return encoded
|
|
}
|
|
|
|
// slimResourceForBroadcast trims static metadata the client-facing stream was
|
|
// re-shipping on every resource (governed gap resource-payload-static-metadata).
|
|
// It only edits the freshly refreshed per-broadcast copy, never stored state:
|
|
// - Aliases duplicating superseded canonical ids are dropped; the ids remain
|
|
// available in canonicalIdentity.supersededIds, which identity consumers
|
|
// also resolve against.
|
|
// - Default-posture policy and its AI-safe summary are omitted; consumers
|
|
// treat a missing policy as the default posture, and the summary is only
|
|
// rendered for non-default postures.
|
|
func slimResourceForBroadcast(resource *unifiedresources.Resource) {
|
|
if resource == nil {
|
|
return
|
|
}
|
|
if canonical := resource.Canonical; canonical != nil && len(canonical.Aliases) > 0 {
|
|
duplicated := make(map[string]struct{}, len(canonical.SupersededIDs)+2)
|
|
for _, id := range canonical.SupersededIDs {
|
|
duplicated[strings.ToLower(strings.TrimSpace(id))] = struct{}{}
|
|
}
|
|
// The row's own wire id is a self-reference: every identity consumer
|
|
// unions resource.id beside the alias spread, so the wire never needs
|
|
// it inside aliases. The primary id stays: aliases remain the complete
|
|
// live vocabulary a host is reachable by (see
|
|
// TestBroadcastSlimmingKeepsHostAgentSupersededIdentityResolvable).
|
|
if resource.ID != "" {
|
|
duplicated[strings.ToLower(strings.TrimSpace(resource.ID))] = struct{}{}
|
|
}
|
|
aliases := make([]string, 0, len(canonical.Aliases))
|
|
for _, alias := range canonical.Aliases {
|
|
if _, dup := duplicated[strings.ToLower(strings.TrimSpace(alias))]; dup {
|
|
continue
|
|
}
|
|
aliases = append(aliases, alias)
|
|
}
|
|
canonical.Aliases = aliases
|
|
}
|
|
if isDefaultBroadcastResourcePolicy(resource.Policy) {
|
|
resource.Policy = nil
|
|
resource.AISafeSummary = ""
|
|
}
|
|
}
|
|
|
|
// isDefaultBroadcastResourcePolicy mirrors the frontend's
|
|
// hasDefaultResourcePolicyPosture: internal sensitivity, cloud-summary
|
|
// routing, and no redactions is the posture consumers assume when the
|
|
// broadcast omits the policy entirely.
|
|
func isDefaultBroadcastResourcePolicy(policy *unifiedresources.ResourcePolicy) bool {
|
|
return policy != nil &&
|
|
policy.Sensitivity == unifiedresources.ResourceSensitivityInternal &&
|
|
policy.Routing.Scope == unifiedresources.ResourceRoutingScopeCloudSummary &&
|
|
len(policy.Routing.Redact) == 0
|
|
}
|
|
|
|
// capabilityCatalogID derives the stable catalog id for one marshaled
|
|
// capabilities blob. Content-addressed ids keep the catalog and refs stable
|
|
// across restarts and reconnects, so they never churn websocket deltas.
|
|
func capabilityCatalogID(encoded json.RawMessage) string {
|
|
digest := sha256.Sum256(encoded)
|
|
return hex.EncodeToString(digest[:6])
|
|
}
|
|
|
|
func monitorDiscoveryTarget(resource *unifiedresources.Resource, resourceType string) *unifiedresources.DiscoveryTarget {
|
|
switch resourceType {
|
|
case "agent", "docker-host":
|
|
agentID := monitorAgentTargetID(resource)
|
|
if agentID == "" {
|
|
return nil
|
|
}
|
|
return &unifiedresources.DiscoveryTarget{
|
|
ResourceType: string(unifiedresources.ResourceTypeAgent),
|
|
AgentID: agentID,
|
|
ResourceID: agentID,
|
|
Hostname: monitorTargetHostname(resource),
|
|
}
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func monitorMetricsTarget(resource *unifiedresources.Resource, resourceType string) *unifiedresources.MetricsTarget {
|
|
switch resourceType {
|
|
case "agent", "docker-host":
|
|
agentID := monitorAgentTargetID(resource)
|
|
if agentID == "" {
|
|
return nil
|
|
}
|
|
return &unifiedresources.MetricsTarget{
|
|
ResourceType: string(unifiedresources.ResourceTypeAgent),
|
|
ResourceID: agentID,
|
|
}
|
|
default:
|
|
return nil
|
|
}
|
|
}
|
|
|
|
func monitorAgentTargetID(resource *unifiedresources.Resource) string {
|
|
if resource.Agent != nil {
|
|
if id := strings.TrimSpace(resource.Agent.AgentID); id != "" {
|
|
return id
|
|
}
|
|
}
|
|
if resource.Docker != nil {
|
|
if id := strings.TrimSpace(resource.Docker.AgentID); id != "" {
|
|
return id
|
|
}
|
|
if id := strings.TrimSpace(resource.Docker.HostSourceID); id != "" {
|
|
return id
|
|
}
|
|
}
|
|
if resource.Kubernetes != nil {
|
|
if id := strings.TrimSpace(resource.Kubernetes.AgentID); id != "" {
|
|
return id
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func monitorTargetHostname(resource *unifiedresources.Resource) string {
|
|
if resource.Agent != nil {
|
|
if hostname := strings.TrimSpace(resource.Agent.Hostname); hostname != "" {
|
|
return hostname
|
|
}
|
|
}
|
|
if resource.Docker != nil {
|
|
if hostname := strings.TrimSpace(resource.Docker.Hostname); hostname != "" {
|
|
return hostname
|
|
}
|
|
}
|
|
if resource.PBS != nil {
|
|
if hostname := strings.TrimSpace(resource.PBS.Hostname); hostname != "" {
|
|
return hostname
|
|
}
|
|
}
|
|
if resource.PMG != nil {
|
|
if hostname := strings.TrimSpace(resource.PMG.Hostname); hostname != "" {
|
|
return hostname
|
|
}
|
|
}
|
|
return strings.TrimSpace(resource.Name)
|
|
}
|
|
|
|
func monitorFrontendResourceType(resource *unifiedresources.Resource) string {
|
|
return string(unifiedresources.ContractResourceType(*resource))
|
|
}
|
|
|
|
func monitorFrontendNames(resource *unifiedresources.Resource, resourceType string) (string, string) {
|
|
name := strings.TrimSpace(unifiedresources.ResourceDisplayName(*resource))
|
|
if name == "" {
|
|
name = resource.ID
|
|
}
|
|
return name, name
|
|
}
|
|
|
|
func monitorPlatformType(resource *unifiedresources.Resource, resourceType string) string {
|
|
if resource.Proxmox != nil {
|
|
return "proxmox-pve"
|
|
}
|
|
if resource.VMware != nil {
|
|
return "vmware-vsphere"
|
|
}
|
|
if resource.TrueNAS != nil {
|
|
return "truenas"
|
|
}
|
|
if resource.Availability != nil {
|
|
return "generic"
|
|
}
|
|
if storagePlatform := monitorStoragePlatformType(resource.Storage, resource.Sources); storagePlatform != "" {
|
|
return storagePlatform
|
|
}
|
|
switch resourceType {
|
|
case "vm", "system-container":
|
|
return "proxmox-pve"
|
|
case "storage", "pool":
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceProxmox) {
|
|
return "proxmox-pve"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourcePBS) {
|
|
return "proxmox-pbs"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceTrueNAS) {
|
|
return "truenas"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceVMware) {
|
|
return "vmware-vsphere"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceAgent) {
|
|
return "agent"
|
|
}
|
|
return "generic"
|
|
case "docker-host", "app-container":
|
|
return "docker"
|
|
case "k8s-cluster", "k8s-node", "pod", "k8s-deployment":
|
|
return "kubernetes"
|
|
case "pbs":
|
|
return "proxmox-pbs"
|
|
case "pmg":
|
|
return "proxmox-pmg"
|
|
case "agent":
|
|
return "agent"
|
|
default:
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceK8s) {
|
|
return "kubernetes"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceDocker) {
|
|
return "docker"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourcePBS) {
|
|
return "proxmox-pbs"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourcePMG) {
|
|
return "proxmox-pmg"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceAgent) {
|
|
return "agent"
|
|
}
|
|
if monitorHasSource(resource.Sources, unifiedresources.SourceProxmox) {
|
|
return "proxmox-pve"
|
|
}
|
|
for _, source := range resource.Sources {
|
|
if candidate := strings.TrimSpace(string(source)); candidate != "" {
|
|
return candidate
|
|
}
|
|
}
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
func monitorStoragePlatformType(storage *unifiedresources.StorageMeta, sources []unifiedresources.DataSource) string {
|
|
if storage == nil {
|
|
return ""
|
|
}
|
|
switch strings.ToLower(strings.TrimSpace(storage.Platform)) {
|
|
case "proxmox", "proxmox-pve", "pve":
|
|
return "proxmox-pve"
|
|
case "pbs", "proxmox-pbs":
|
|
return "proxmox-pbs"
|
|
case "truenas":
|
|
return "truenas"
|
|
case "vmware", "vmware-vsphere", "vsphere":
|
|
return "vmware-vsphere"
|
|
case "docker", "podman":
|
|
return "docker"
|
|
case "kubernetes", "k8s":
|
|
return "kubernetes"
|
|
case "unraid":
|
|
if monitorHasSource(sources, unifiedresources.SourceAgent) {
|
|
return "agent"
|
|
}
|
|
return "generic"
|
|
default:
|
|
return ""
|
|
}
|
|
}
|
|
|
|
func monitorPlatformID(resource *unifiedresources.Resource, resourceType string) string {
|
|
switch resourceType {
|
|
case "node", "vm", "system-container":
|
|
if resource.Proxmox != nil && strings.TrimSpace(resource.Proxmox.Instance) != "" {
|
|
return strings.TrimSpace(resource.Proxmox.Instance)
|
|
}
|
|
case "agent":
|
|
if resource.Agent != nil && strings.TrimSpace(resource.Agent.AgentID) != "" {
|
|
return strings.TrimSpace(resource.Agent.AgentID)
|
|
}
|
|
case "docker-host":
|
|
if resource.Docker != nil && strings.TrimSpace(resource.Docker.Hostname) != "" {
|
|
return strings.TrimSpace(resource.Docker.Hostname)
|
|
}
|
|
case "app-container":
|
|
if resource.Docker != nil && strings.TrimSpace(resource.Docker.Hostname) != "" {
|
|
return strings.TrimSpace(resource.Docker.Hostname)
|
|
}
|
|
if resource.ParentID != nil {
|
|
return strings.TrimSpace(*resource.ParentID)
|
|
}
|
|
case "k8s-cluster", "k8s-node", "pod", "k8s-deployment":
|
|
if resource.Kubernetes != nil && strings.TrimSpace(resource.Kubernetes.AgentID) != "" {
|
|
return strings.TrimSpace(resource.Kubernetes.AgentID)
|
|
}
|
|
case "pbs":
|
|
if resource.PBS != nil && strings.TrimSpace(resource.PBS.Hostname) != "" {
|
|
return strings.TrimSpace(resource.PBS.Hostname)
|
|
}
|
|
case "pmg":
|
|
if resource.PMG != nil && strings.TrimSpace(resource.PMG.Hostname) != "" {
|
|
return strings.TrimSpace(resource.PMG.Hostname)
|
|
}
|
|
case "network-endpoint":
|
|
if resource.Availability != nil {
|
|
if targetID := strings.TrimSpace(resource.Availability.TargetID); targetID != "" {
|
|
return targetID
|
|
}
|
|
if address := strings.TrimSpace(resource.Availability.Address); address != "" {
|
|
return address
|
|
}
|
|
}
|
|
}
|
|
return resource.ID
|
|
}
|
|
|
|
func monitorFrontendStatus(resource *unifiedresources.Resource, resourceType string) string {
|
|
switch resourceType {
|
|
case "app-container":
|
|
switch resource.Status {
|
|
case unifiedresources.StatusOnline:
|
|
return "running"
|
|
case unifiedresources.StatusOffline:
|
|
return "stopped"
|
|
case unifiedresources.StatusWarning:
|
|
return "degraded"
|
|
}
|
|
case "pod":
|
|
if resource.Kubernetes != nil {
|
|
phase := strings.ToLower(strings.TrimSpace(resource.Kubernetes.PodPhase))
|
|
switch phase {
|
|
case "pending", "unknown":
|
|
return "degraded"
|
|
case "succeeded", "failed":
|
|
return "stopped"
|
|
}
|
|
// Phase=Running is not enough — a pod can have Phase=Running with
|
|
// CrashLoopBackOff containers, which statusFromKubernetesPod
|
|
// correctly classifies as StatusWarning. Fall through to the
|
|
// unified-status mapping below so that surfaces as "degraded".
|
|
}
|
|
}
|
|
|
|
switch resource.Status {
|
|
case unifiedresources.StatusOnline:
|
|
if monitorIsWorkloadType(resourceType) || resourceType == "pod" {
|
|
return "running"
|
|
}
|
|
return "online"
|
|
case unifiedresources.StatusOffline:
|
|
if monitorIsWorkloadType(resourceType) || resourceType == "pod" {
|
|
return "stopped"
|
|
}
|
|
return "offline"
|
|
case unifiedresources.StatusWarning:
|
|
return "degraded"
|
|
default:
|
|
return "unknown"
|
|
}
|
|
}
|
|
|
|
func monitorIsWorkloadType(resourceType string) bool {
|
|
switch resourceType {
|
|
case "app-container", "system-container", "vm", "oci-container":
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func monitorClusterID(resource *unifiedresources.Resource) string {
|
|
if clusterID := strings.TrimSpace(unifiedresources.ResourceClusterName(*resource)); clusterID != "" {
|
|
return clusterID
|
|
}
|
|
|
|
if resource.Docker != nil && resource.Docker.Swarm != nil {
|
|
if name := strings.TrimSpace(resource.Docker.Swarm.ClusterName); name != "" {
|
|
return name
|
|
}
|
|
if id := strings.TrimSpace(resource.Docker.Swarm.ClusterID); id != "" {
|
|
return id
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func monitorMetricInput(metric *unifiedresources.MetricValue) *models.ResourceMetricInput {
|
|
if metric == nil {
|
|
return nil
|
|
}
|
|
|
|
current := metric.Percent
|
|
if current == 0 {
|
|
current = metric.Value
|
|
}
|
|
if metric.Percent != 0 && metric.Value != 0 {
|
|
current = math.Max(metric.Percent, metric.Value)
|
|
}
|
|
|
|
result := &models.ResourceMetricInput{Current: current, Observation: metric.Observation}
|
|
if metric.Total != nil {
|
|
total := *metric.Total
|
|
result.Total = &total
|
|
}
|
|
if metric.Used != nil {
|
|
used := *metric.Used
|
|
result.Used = &used
|
|
}
|
|
if result.Total != nil && result.Used != nil {
|
|
free := *result.Total - *result.Used
|
|
result.Free = &free
|
|
}
|
|
return result
|
|
}
|
|
|
|
func monitorNetworkMetricInput(metrics *unifiedresources.ResourceMetrics) (bool, int64, int64) {
|
|
if metrics == nil || (metrics.NetIn == nil && metrics.NetOut == nil) {
|
|
return false, 0, 0
|
|
}
|
|
|
|
var rx int64
|
|
var tx int64
|
|
if metrics.NetIn != nil {
|
|
rx = int64(math.Round(metrics.NetIn.Value))
|
|
}
|
|
if metrics.NetOut != nil {
|
|
tx = int64(math.Round(metrics.NetOut.Value))
|
|
}
|
|
return true, rx, tx
|
|
}
|
|
|
|
func monitorDiskIOMetricInput(metrics *unifiedresources.ResourceMetrics) (*int64, *int64) {
|
|
if metrics == nil || (metrics.DiskRead == nil && metrics.DiskWrite == nil) {
|
|
return nil, nil
|
|
}
|
|
var read, write *int64
|
|
if metrics.DiskRead != nil {
|
|
value := int64(math.Round(metrics.DiskRead.Value))
|
|
read = &value
|
|
}
|
|
if metrics.DiskWrite != nil {
|
|
value := int64(math.Round(metrics.DiskWrite.Value))
|
|
write = &value
|
|
}
|
|
return read, write
|
|
}
|
|
|
|
func monitorTemperature(resource *unifiedresources.Resource) *float64 {
|
|
if resource.Agent != nil && resource.Agent.Temperature != nil {
|
|
value := *resource.Agent.Temperature
|
|
return &value
|
|
}
|
|
if resource.Proxmox != nil && resource.Proxmox.Temperature != nil {
|
|
value := *resource.Proxmox.Temperature
|
|
return &value
|
|
}
|
|
if resource.Docker != nil && resource.Docker.Temperature != nil {
|
|
value := *resource.Docker.Temperature
|
|
return &value
|
|
}
|
|
if resource.Kubernetes != nil && resource.Kubernetes.Temperature != nil {
|
|
value := *resource.Kubernetes.Temperature
|
|
return &value
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func monitorUptime(resource *unifiedresources.Resource) *int64 {
|
|
if resource.Agent != nil && resource.Agent.UptimeSeconds > 0 {
|
|
value := resource.Agent.UptimeSeconds
|
|
return &value
|
|
}
|
|
if resource.Proxmox != nil && resource.Proxmox.Uptime > 0 {
|
|
value := resource.Proxmox.Uptime
|
|
return &value
|
|
}
|
|
if resource.Docker != nil && resource.Docker.UptimeSeconds > 0 {
|
|
value := resource.Docker.UptimeSeconds
|
|
return &value
|
|
}
|
|
if resource.Kubernetes != nil && resource.Kubernetes.UptimeSeconds > 0 {
|
|
value := resource.Kubernetes.UptimeSeconds
|
|
return &value
|
|
}
|
|
if resource.PBS != nil && resource.PBS.UptimeSeconds > 0 {
|
|
value := resource.PBS.UptimeSeconds
|
|
return &value
|
|
}
|
|
if resource.PMG != nil && resource.PMG.UptimeSeconds > 0 {
|
|
value := resource.PMG.UptimeSeconds
|
|
return &value
|
|
}
|
|
if resource.TrueNAS != nil && resource.TrueNAS.UptimeSeconds > 0 {
|
|
value := resource.TrueNAS.UptimeSeconds
|
|
return &value
|
|
}
|
|
// Canonical Resource.Uptime is the universal fallback. The vSphere
|
|
// adapter populates only this field for ESXi hosts and VMs (no
|
|
// vmware-specific UptimeSeconds carve-out), so without this fall the
|
|
// websocket broadcast layer would drop uptime for VMware-backed rows
|
|
// even though the REST `/api/resources` payload exposes it on
|
|
// `resource.uptime` directly.
|
|
if resource.Uptime > 0 {
|
|
value := resource.Uptime
|
|
return &value
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func monitorLabels(resource *unifiedresources.Resource) map[string]string {
|
|
if resource.Kubernetes == nil || len(resource.Kubernetes.Labels) == 0 {
|
|
return nil
|
|
}
|
|
labels := make(map[string]string, len(resource.Kubernetes.Labels))
|
|
for key, value := range resource.Kubernetes.Labels {
|
|
labels[key] = value
|
|
}
|
|
return labels
|
|
}
|
|
|
|
func monitorIdentity(resource *unifiedresources.Resource, fallbackName string) *models.ResourceIdentityInput {
|
|
hostname := ""
|
|
if resource.Agent != nil {
|
|
hostname = strings.TrimSpace(resource.Agent.Hostname)
|
|
}
|
|
if hostname == "" && resource.Docker != nil {
|
|
hostname = strings.TrimSpace(resource.Docker.Hostname)
|
|
}
|
|
if hostname == "" && resource.Proxmox != nil {
|
|
hostname = strings.TrimSpace(resource.Proxmox.NodeName)
|
|
}
|
|
if hostname == "" && resource.Availability != nil {
|
|
hostname = strings.TrimSpace(resource.Availability.Address)
|
|
}
|
|
if hostname == "" {
|
|
for _, candidate := range resource.Identity.Hostnames {
|
|
if trimmed := strings.TrimSpace(candidate); trimmed != "" {
|
|
hostname = trimmed
|
|
break
|
|
}
|
|
}
|
|
}
|
|
if hostname == "" {
|
|
hostname = fallbackName
|
|
}
|
|
|
|
ips := make([]string, 0, len(resource.Identity.IPAddresses))
|
|
for _, ip := range resource.Identity.IPAddresses {
|
|
trimmed := strings.TrimSpace(ip)
|
|
if trimmed == "" {
|
|
continue
|
|
}
|
|
ips = append(ips, trimmed)
|
|
}
|
|
|
|
machineID := strings.TrimSpace(resource.Identity.MachineID)
|
|
if hostname == "" && machineID == "" && len(ips) == 0 {
|
|
return nil
|
|
}
|
|
|
|
return &models.ResourceIdentityInput{
|
|
Hostname: hostname,
|
|
MachineID: machineID,
|
|
IPs: ips,
|
|
}
|
|
}
|
|
|
|
func monitorPlatformData(resource *unifiedresources.Resource, resourceType string, platformID string) json.RawMessage {
|
|
var payload interface{}
|
|
|
|
switch resourceType {
|
|
case "node":
|
|
if resource.Proxmox != nil {
|
|
payload = map[string]interface{}{
|
|
"instance": resource.Proxmox.Instance,
|
|
"host": "",
|
|
"guestURL": "",
|
|
"pveVersion": resource.Proxmox.PVEVersion,
|
|
"kernelVersion": resource.Proxmox.KernelVersion,
|
|
"cpuInfo": resource.Proxmox.CPUInfo,
|
|
"loadAverage": []float64{},
|
|
"isClusterMember": resource.Proxmox.ClusterName != "",
|
|
"clusterName": resource.Proxmox.ClusterName,
|
|
"connectionHealth": monitorSourceStatus(resource.SourceStatus, unifiedresources.SourceProxmox),
|
|
}
|
|
}
|
|
case "vm":
|
|
payload = buildProxmoxVMPayload(resource)
|
|
case "system-container", "oci-container":
|
|
payload = buildProxmoxVMPayload(resource)
|
|
case "agent":
|
|
if resource.Proxmox != nil {
|
|
payload = map[string]interface{}{
|
|
"instance": resource.Proxmox.Instance,
|
|
"host": "",
|
|
"guestURL": "",
|
|
"pveVersion": resource.Proxmox.PVEVersion,
|
|
"kernelVersion": resource.Proxmox.KernelVersion,
|
|
"cpuInfo": resource.Proxmox.CPUInfo,
|
|
"loadAverage": []float64{},
|
|
"isClusterMember": resource.Proxmox.ClusterName != "",
|
|
"clusterName": resource.Proxmox.ClusterName,
|
|
"connectionHealth": monitorSourceStatus(resource.SourceStatus, unifiedresources.SourceProxmox),
|
|
}
|
|
} else if resource.Agent != nil {
|
|
payload = map[string]interface{}{
|
|
"platform": resource.Agent.Platform,
|
|
"osName": resource.Agent.OSName,
|
|
"osVersion": resource.Agent.OSVersion,
|
|
"kernelVersion": resource.Agent.KernelVersion,
|
|
"architecture": resource.Agent.Architecture,
|
|
"agentVersion": resource.Agent.AgentVersion,
|
|
"interfaces": resource.Agent.NetworkInterfaces,
|
|
"disks": resource.Agent.Disks,
|
|
"memory": resource.Agent.Memory,
|
|
}
|
|
}
|
|
case "docker-host":
|
|
if resource.Docker != nil {
|
|
payload = map[string]interface{}{
|
|
"agentId": platformID,
|
|
"runtime": resource.Docker.Runtime,
|
|
"runtimeVersion": resource.Docker.RuntimeVersion,
|
|
"dockerVersion": resource.Docker.DockerVersion,
|
|
"os": resource.Docker.OS,
|
|
"kernelVersion": resource.Docker.KernelVersion,
|
|
"architecture": resource.Docker.Architecture,
|
|
"agentVersion": resource.Docker.AgentVersion,
|
|
"swarm": resource.Docker.Swarm,
|
|
"interfaces": resource.Docker.NetworkInterfaces,
|
|
"disks": resource.Docker.Disks,
|
|
}
|
|
}
|
|
case "app-container":
|
|
if resource.Docker != nil {
|
|
payload = map[string]interface{}{
|
|
"hostId": monitorStringValue(resource.ParentID),
|
|
"hostName": resource.Docker.Hostname,
|
|
"image": resource.Docker.Image,
|
|
"state": strings.ToLower(string(resource.Status)),
|
|
"status": strings.ToLower(string(resource.Status)),
|
|
"health": "",
|
|
"createdAt": time.Time{},
|
|
}
|
|
}
|
|
case "k8s-cluster":
|
|
if resource.Kubernetes != nil {
|
|
payload = map[string]interface{}{
|
|
"agentId": resource.Kubernetes.AgentID,
|
|
"server": resource.Kubernetes.Server,
|
|
"context": resource.Kubernetes.Context,
|
|
"version": resource.Kubernetes.Version,
|
|
"customDisplayName": "",
|
|
"hidden": false,
|
|
"pendingUninstall": resource.Kubernetes.PendingUninstall,
|
|
"nodeCount": resource.ChildCount,
|
|
}
|
|
}
|
|
case "k8s-node":
|
|
if resource.Kubernetes != nil {
|
|
payload = map[string]interface{}{
|
|
"clusterId": resource.Kubernetes.ClusterID,
|
|
"ready": resource.Kubernetes.Ready,
|
|
"unschedulable": resource.Kubernetes.Unschedulable,
|
|
"kubeletVersion": resource.Kubernetes.KubeletVersion,
|
|
"containerRuntimeVersion": resource.Kubernetes.ContainerRuntimeVersion,
|
|
"osImage": resource.Kubernetes.OSImage,
|
|
"kernelVersion": resource.Kubernetes.KernelVersion,
|
|
"architecture": resource.Kubernetes.Architecture,
|
|
"capacityCpuCores": resource.Kubernetes.CapacityCPU,
|
|
"capacityMemoryBytes": resource.Kubernetes.CapacityMemoryBytes,
|
|
"capacityPods": resource.Kubernetes.CapacityPods,
|
|
"allocatableCpuCores": resource.Kubernetes.AllocCPU,
|
|
"allocatableMemoryBytes": resource.Kubernetes.AllocMemoryBytes,
|
|
"allocatablePods": resource.Kubernetes.AllocPods,
|
|
"roles": append([]string(nil), resource.Kubernetes.Roles...),
|
|
}
|
|
}
|
|
case "pod":
|
|
if resource.Kubernetes != nil {
|
|
payload = map[string]interface{}{
|
|
"clusterId": resource.Kubernetes.ClusterID,
|
|
"namespace": resource.Kubernetes.Namespace,
|
|
"nodeName": resource.Kubernetes.NodeName,
|
|
"phase": resource.Kubernetes.PodPhase,
|
|
"restarts": resource.Kubernetes.Restarts,
|
|
"ownerKind": resource.Kubernetes.OwnerKind,
|
|
"ownerName": resource.Kubernetes.OwnerName,
|
|
}
|
|
}
|
|
case "k8s-deployment":
|
|
if resource.Kubernetes != nil {
|
|
payload = map[string]interface{}{
|
|
"clusterId": resource.Kubernetes.ClusterID,
|
|
"namespace": resource.Kubernetes.Namespace,
|
|
"desiredReplicas": resource.Kubernetes.DesiredReplicas,
|
|
"updatedReplicas": resource.Kubernetes.UpdatedReplicas,
|
|
"readyReplicas": resource.Kubernetes.ReadyReplicas,
|
|
"availableReplicas": resource.Kubernetes.AvailableReplicas,
|
|
}
|
|
}
|
|
case "pbs":
|
|
if resource.PBS != nil {
|
|
payload = map[string]interface{}{
|
|
"host": resource.PBS.Hostname,
|
|
"version": resource.PBS.Version,
|
|
"connectionHealth": resource.PBS.ConnectionHealth,
|
|
"memoryUsed": monitorMetricUsed(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Memory })),
|
|
"memoryTotal": monitorMetricTotal(monitorMetricValue(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.Memory })),
|
|
"numDatastores": resource.PBS.DatastoreCount,
|
|
}
|
|
}
|
|
case "pmg":
|
|
if resource.PMG != nil {
|
|
payload = map[string]interface{}{
|
|
"host": resource.PMG.Hostname,
|
|
"version": resource.PMG.Version,
|
|
"connectionHealth": resource.PMG.ConnectionHealth,
|
|
"nodeCount": resource.PMG.NodeCount,
|
|
"queueActive": resource.PMG.QueueActive,
|
|
"queueDeferred": resource.PMG.QueueDeferred,
|
|
"queueHold": resource.PMG.QueueHold,
|
|
"queueIncoming": resource.PMG.QueueIncoming,
|
|
"queueTotal": resource.PMG.QueueTotal,
|
|
"mailCountTotal": resource.PMG.MailCountTotal,
|
|
"spamIn": resource.PMG.SpamIn,
|
|
"virusIn": resource.PMG.VirusIn,
|
|
"lastUpdated": resource.PMG.LastUpdated,
|
|
}
|
|
}
|
|
case "storage", "pool":
|
|
payload = monitorStoragePlatformData(resource, platformID)
|
|
case "network-endpoint":
|
|
if resource.Availability != nil {
|
|
payload = map[string]interface{}{
|
|
"targetId": resource.Availability.TargetID,
|
|
"address": resource.Availability.Address,
|
|
"protocol": resource.Availability.Protocol,
|
|
"port": resource.Availability.Port,
|
|
"path": resource.Availability.Path,
|
|
"enabled": resource.Availability.Enabled,
|
|
"available": resource.Availability.Available,
|
|
"lastChecked": resource.Availability.LastChecked,
|
|
"lastSuccess": resource.Availability.LastSuccess,
|
|
"latencyMillis": resource.Availability.LatencyMillis,
|
|
"consecutiveFailures": resource.Availability.ConsecutiveFailures,
|
|
"lastError": resource.Availability.LastError,
|
|
"failureThreshold": resource.Availability.FailureThreshold,
|
|
"pollIntervalSeconds": resource.Availability.PollIntervalSeconds,
|
|
"timeoutMillis": resource.Availability.TimeoutMillis,
|
|
}
|
|
}
|
|
}
|
|
|
|
if payload == nil {
|
|
return nil
|
|
}
|
|
payload = monitorAttachSourceKeys(payload, resource.Sources)
|
|
if payload == nil {
|
|
return nil
|
|
}
|
|
|
|
encoded, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
return encoded
|
|
}
|
|
|
|
func monitorAttachSourceKeys(payload interface{}, sources []unifiedresources.DataSource) interface{} {
|
|
sourceKeys := monitorSourceKeys(sources)
|
|
if len(sourceKeys) == 0 {
|
|
return payload
|
|
}
|
|
if payloadMap, ok := payload.(map[string]interface{}); ok {
|
|
if payloadMap == nil {
|
|
return nil
|
|
}
|
|
payloadMap["sources"] = sourceKeys
|
|
return payloadMap
|
|
}
|
|
return payload
|
|
}
|
|
|
|
func monitorStoragePlatformData(resource *unifiedresources.Resource, platformID string) map[string]interface{} {
|
|
nodeLabel := resource.ParentName
|
|
if nodeLabel == "" {
|
|
nodeLabel = monitorStringValue(resource.ParentID)
|
|
}
|
|
payload := map[string]interface{}{
|
|
"instance": platformID,
|
|
"node": nodeLabel,
|
|
"active": resource.Status == unifiedresources.StatusOnline,
|
|
}
|
|
|
|
if resource.Proxmox != nil {
|
|
if strings.TrimSpace(resource.Proxmox.Instance) != "" {
|
|
payload["instance"] = strings.TrimSpace(resource.Proxmox.Instance)
|
|
}
|
|
if strings.TrimSpace(resource.Proxmox.NodeName) != "" {
|
|
payload["node"] = strings.TrimSpace(resource.Proxmox.NodeName)
|
|
}
|
|
}
|
|
|
|
if resource.Storage == nil {
|
|
payload["type"] = ""
|
|
payload["content"] = ""
|
|
payload["shared"] = false
|
|
payload["enabled"] = true
|
|
return payload
|
|
}
|
|
|
|
storage := resource.Storage
|
|
payload["type"] = storage.Type
|
|
payload["content"] = storage.Content
|
|
payload["contentTypes"] = append([]string(nil), storage.ContentTypes...)
|
|
payload["shared"] = storage.Shared
|
|
payload["enabled"] = storage.Enabled
|
|
payload["active"] = storage.Active
|
|
payload["isCeph"] = storage.IsCeph
|
|
payload["isZfs"] = storage.IsZFS
|
|
payload["platform"] = storage.Platform
|
|
payload["topology"] = storage.Topology
|
|
payload["protection"] = storage.Protection
|
|
payload["pool"] = storage.Pool
|
|
payload["path"] = storage.Path
|
|
payload["nodes"] = append([]string(nil), storage.Nodes...)
|
|
payload["risk"] = storage.Risk
|
|
payload["riskSummary"] = storage.RiskSummary
|
|
payload["postureSummary"] = storage.PostureSummary
|
|
payload["protectionReduced"] = storage.ProtectionReduced
|
|
payload["protectionSummary"] = storage.ProtectionSummary
|
|
payload["rebuildInProgress"] = storage.RebuildInProgress
|
|
payload["rebuildSummary"] = storage.RebuildSummary
|
|
payload["arrayState"] = storage.ArrayState
|
|
payload["syncAction"] = storage.SyncAction
|
|
payload["syncProgress"] = storage.SyncProgress
|
|
payload["numProtected"] = storage.NumProtected
|
|
payload["numDisabled"] = storage.NumDisabled
|
|
payload["numInvalid"] = storage.NumInvalid
|
|
payload["numMissing"] = storage.NumMissing
|
|
payload["zfsPoolState"] = storage.ZFSPoolState
|
|
payload["zfsReadErrors"] = storage.ZFSReadErrors
|
|
payload["zfsWriteErrors"] = storage.ZFSWriteErrors
|
|
payload["zfsChecksumErrors"] = storage.ZFSChecksumErrors
|
|
if storage.ZFSPool != nil {
|
|
payload["zfsPool"] = storage.ZFSPool
|
|
}
|
|
return payload
|
|
}
|
|
|
|
func convertProxmoxDisks(disks []unifiedresources.DiskInfo) []map[string]interface{} {
|
|
if len(disks) == 0 {
|
|
return nil
|
|
}
|
|
|
|
out := make([]map[string]interface{}, 0, len(disks))
|
|
for _, d := range disks {
|
|
usage := float64(0)
|
|
if d.Total > 0 {
|
|
usage = float64(d.Used) / float64(d.Total) * 100
|
|
}
|
|
out = append(out, map[string]interface{}{
|
|
"total": d.Total,
|
|
"used": d.Used,
|
|
"free": d.Free,
|
|
"usage": usage,
|
|
"mountpoint": d.Mountpoint,
|
|
"type": d.Filesystem,
|
|
"device": d.Device,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func buildProxmoxVMPayload(resource *unifiedresources.Resource) map[string]interface{} {
|
|
if resource.Proxmox == nil {
|
|
return nil
|
|
}
|
|
return map[string]interface{}{
|
|
"vmid": resource.Proxmox.VMID,
|
|
"node": resource.Proxmox.NodeName,
|
|
"instance": resource.Proxmox.Instance,
|
|
"cpus": resource.Proxmox.CPUs,
|
|
"template": resource.Proxmox.Template,
|
|
"networkIn": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.NetIn }),
|
|
"networkOut": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.NetOut }),
|
|
"diskRead": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue { return metrics.DiskRead }),
|
|
"diskWrite": monitorMetricInt64(resource.Metrics, func(metrics *unifiedresources.ResourceMetrics) *unifiedresources.MetricValue {
|
|
return metrics.DiskWrite
|
|
}),
|
|
"disks": convertProxmoxDisks(resource.Proxmox.Disks),
|
|
"swapUsed": resource.Proxmox.SwapUsed,
|
|
"swapTotal": resource.Proxmox.SwapTotal,
|
|
"balloon": resource.Proxmox.Balloon,
|
|
"lastBackup": resource.Proxmox.LastBackup,
|
|
// A backup of this guest is running right now; lastBackup still
|
|
// points at the latest completed backup.
|
|
"backupInProgress": resource.Proxmox.BackupInProgress,
|
|
"ipAddresses": append([]string(nil), resource.Identity.IPAddresses...),
|
|
}
|
|
}
|
|
|
|
func monitorMetricValue(metrics *unifiedresources.ResourceMetrics, pick func(*unifiedresources.ResourceMetrics) *unifiedresources.MetricValue) *unifiedresources.MetricValue {
|
|
if metrics == nil {
|
|
return nil
|
|
}
|
|
return pick(metrics)
|
|
}
|
|
|
|
func monitorMetricInt64(metrics *unifiedresources.ResourceMetrics, pick func(*unifiedresources.ResourceMetrics) *unifiedresources.MetricValue) int64 {
|
|
metric := monitorMetricValue(metrics, pick)
|
|
if metric == nil {
|
|
return 0
|
|
}
|
|
return int64(math.Round(metric.Value))
|
|
}
|
|
|
|
func monitorMetricUsed(metric *unifiedresources.MetricValue) int64 {
|
|
if metric == nil || metric.Used == nil {
|
|
return 0
|
|
}
|
|
return *metric.Used
|
|
}
|
|
|
|
func monitorMetricTotal(metric *unifiedresources.MetricValue) int64 {
|
|
if metric == nil || metric.Total == nil {
|
|
return 0
|
|
}
|
|
return *metric.Total
|
|
}
|
|
|
|
func monitorSourceStatus(statuses map[unifiedresources.DataSource]unifiedresources.SourceStatus, source unifiedresources.DataSource) string {
|
|
if statuses == nil {
|
|
return ""
|
|
}
|
|
status, ok := statuses[source]
|
|
if !ok {
|
|
return ""
|
|
}
|
|
return status.Status
|
|
}
|
|
|
|
func monitorHasSource(sources []unifiedresources.DataSource, source unifiedresources.DataSource) bool {
|
|
for _, candidate := range sources {
|
|
if candidate == source {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func monitorSourceKeys(sources []unifiedresources.DataSource) []string {
|
|
if len(sources) == 0 {
|
|
return nil
|
|
}
|
|
keys := make([]string, 0, len(sources))
|
|
seen := make(map[string]struct{}, len(sources))
|
|
for _, source := range sources {
|
|
key := strings.TrimSpace(string(source))
|
|
if key == "" {
|
|
continue
|
|
}
|
|
if _, ok := seen[key]; ok {
|
|
continue
|
|
}
|
|
seen[key] = struct{}{}
|
|
keys = append(keys, key)
|
|
}
|
|
return keys
|
|
}
|
|
|
|
func monitorSourceType(sources []unifiedresources.DataSource) string {
|
|
if len(sources) > 1 {
|
|
return "hybrid"
|
|
}
|
|
if len(sources) == 1 {
|
|
switch sources[0] {
|
|
case unifiedresources.SourceAgent, unifiedresources.SourceDocker, unifiedresources.SourceK8s:
|
|
return "agent"
|
|
default:
|
|
return "api"
|
|
}
|
|
}
|
|
return "api"
|
|
}
|
|
|
|
func monitorStringValue(value *string) string {
|
|
if value == nil {
|
|
return ""
|
|
}
|
|
return strings.TrimSpace(*value)
|
|
}
|
|
|
|
func monitorLastSeenUnix(value time.Time) int64 {
|
|
// Zero means the resource has never actually been sighted (synthesized
|
|
// offline placeholders). Report 0 so the frontend renders "never" instead
|
|
// of fabricating a fresh sighting.
|
|
if value.IsZero() {
|
|
return 0
|
|
}
|
|
return value.UnixMilli()
|
|
}
|
|
|
|
// pollStorageBackupsWithNodes polls backups using a provided nodes list to avoid duplicate GetNodes calls
|
|
// Stop gracefully stops the monitor
|
|
// guestMetadataDrainTimeout bounds how long Stop waits for queued guest
|
|
// metadata writes. It matches tenantMonitorShutdownTimeout so a wedged store
|
|
// cannot hold up tenant teardown indefinitely.
|
|
const guestMetadataDrainTimeout = 2 * time.Second
|
|
|
|
func (m *Monitor) Stop() {
|
|
if err := m.stop(); err != nil {
|
|
log.Error().Err(err).Msg("monitor stopped with incomplete guest metadata persistence; data directory must be retained")
|
|
}
|
|
}
|
|
|
|
func (m *Monitor) stop() error {
|
|
log.Info().Msg("stopping monitor")
|
|
|
|
// A trailing agent-report refresh must not run against stores closed below.
|
|
m.stopAgentReportRefresh()
|
|
|
|
if m.deadMan != nil {
|
|
m.deadMan.stop(time.Now().UTC(), m.alertManager)
|
|
}
|
|
|
|
// Stop the alert manager to save history
|
|
if m.alertManager != nil {
|
|
m.alertManager.Stop()
|
|
}
|
|
|
|
// Stop notification manager
|
|
if m.notificationMgr != nil {
|
|
m.notificationMgr.Stop()
|
|
}
|
|
|
|
// Drain background guest-metadata writes before the data directory can be
|
|
// torn down. Without this a queued write lands after shutdown and leaves a
|
|
// stray guest_metadata.json.tmp behind.
|
|
drainErr := m.guestMetadataStore.Close(guestMetadataDrainTimeout)
|
|
|
|
// Close persistent metrics store (flushes buffered data)
|
|
if m.metricsStore != nil {
|
|
if err := m.metricsStore.Close(); err != nil {
|
|
log.Error().Err(err).Msg("failed to close metrics store")
|
|
} else {
|
|
log.Info().Msg("metrics store closed successfully")
|
|
}
|
|
}
|
|
|
|
if drainErr != nil {
|
|
return drainErr
|
|
}
|
|
log.Info().Msg("monitor stopped")
|
|
return nil
|
|
}
|
|
|
|
// recordAuthFailure records an authentication failure for a node
|